unpack-objects: unpack large object in stream
PATCH 1/1 unpack-objects: unpack large object in stream
From: Han Xin <[email protected]>
When calling "unpack_non_delta_entry()", will allocate full memory for the whole size of the unpacked object and write the buffer to loose file on disk. This may lead to OOM for the git-unpack-objects process when unpacking a very large object.
In function "unpack_delta_entry()", will also allocate full memory to buffer the whole delta, but since there will be no delta for an object larger than "core.bigFileThreshold", this issue is moderate.
To resolve the OOM issue in "git-unpack-objects", we can unpack large object to file in stream, and use the setting of "core.bigFileThreshold" as the threshold for large object.
--- builtin/unpack-objects.c | 41 +++++++- object-file.c | 149 +++++++++++++++++++++++++++--- object-store.h | 9 ++ t/t5590-receive-unpack-objects.sh | 92 ++++++++++++++++++ 4 files changed, 279 insertions(+), 12 deletions(-) create mode 100755 t/t5590-receive-unpack-objects.sh
builtin/unpack-objects.c +40 -1
@@ -320,11 +320,50 @@ static void added_object(unsigned nr, enum object_type type, } } +static void fill_stream (struct git_zstream *stream) +{ + stream->next_in =fill (1 ); + stream->avail_in = len; +} + +static void use_stream (struct git_zstream *stream) +{ +use (len - stream->avail_in); +} + +static void write_stream_blob (unsigned nr,unsigned long size) +{ +struct git_zstream_reader reader; +struct object_id *oid = &obj_list[nr].oid; + + reader.fill = &fill_stream; + reader.use = &use_stream; + +if (write_stream_object_file (&reader, size,type_name (OBJ_BLOB ), + oid, dry_run)) +die ("failed to write object in stream" ); +if (strict && !dry_run) { +struct blob *blob =lookup_blob (the_repository, oid); +if (blob) + blob->object.flags |=FLAG_WRITTEN ; +else +die ("invalid blob object from stream" ); + } + obj_list[nr].obj =NULL ; +} +static void unpack_non_delta_entry (enum object_type type,unsigned long size,unsigned nr) { -void *buf =get_data (size); +void *buf; + +/* Write large blob in stream without allocating full buffer. */ +if (type ==OBJ_BLOB && size > big_file_threshold) { +write_stream_blob (nr, size); +return ; + } + buf =get_data (size);if (!dry_run && buf)write_object (nr, type, buf, size);else
object-file.c +138 -11
167 lines not on this page · raw message
object-store.h +9 -0
@@ -33,6 +33,11 @@ struct object_directory {char *path; }; +struct git_zstream_reader { +void (*fill)(struct git_zstream *); +void (*use)(struct git_zstream *); +}; + KHASH_INIT(odb_path_map, const char * /* key: odb_path */, struct object_directory *, 1, fspathhash, fspatheq)
@@ -225,6 +230,10 @@ int hash_object_file(const struct git_hash_algo *algo, const void *buf,int write_object_file (const void *buf,unsigned long len,const char *type,struct object_id *oid); +int write_stream_object_file (struct git_zstream_reader *reader, +unsigned long len,const char *type, +struct object_id *oid,int dry_run); +int hash_object_file_literally (const void *buf,unsigned long len,const char *type,struct object_id *oid,unsigned flags);
t/t5590-receive-unpack-objects.sh new file +92 -0
92 lines not on this page · raw message
Signature · 1 line
2.33.0.1.g09a6bb964f.dirty
Any suggestions?
Quoted 384 lines · Han Xin <[email protected]> 于2021年10月9日周六 下午4:21写道:
From: Han Xin <[email protected]>
When calling "
unpack_non_delta_entry()", will allocate full memory for the whole size of the unpacked object and write the buffer to loose file on disk. This may lead to OOM for the git-unpack-objects process when unpacking a very large object.In function "
unpack_delta_entry()", will also allocate full memory to buffer the whole delta, but since there will be no delta for an object larger than "core.bigFileThreshold", this issue is moderate.To resolve the OOM issue in "git-unpack-objects", we can unpack large object to file in stream, and use the setting of "core.bigFileThreshold" as the threshold for large object.
Reviewed-by Jiang Xin <[email protected]>Signed-off-by Han Xin <[email protected]>--- builtin/unpack-objects.c | 41 +++++++- object-file.c | 149 +++++++++++++++++++++++++++--- object-store.h | 9 ++ t/t5590-receive-unpack-objects.sh | 92 ++++++++++++++++++ 4 files changed, 279 insertions(+), 12 deletions(-) create mode 100755 t/t5590-receive-unpack-objects.shbuiltin/unpack-objects.c +40
-1@@ -320,11 +320,50 @@ static void added_object(unsigned nr, enum object_type type, } } +static void fill_stream (struct git_zstream *stream) +{ + stream->next_in =fill (1 ); + stream->avail_in = len; +} + +static void use_stream (struct git_zstream *stream) +{ +use (len - stream->avail_in); +} + +static void write_stream_blob (unsigned nr,unsigned long size) +{ +struct git_zstream_reader reader; +struct object_id *oid = &obj_list[nr].oid; + + reader.fill = &fill_stream; + reader.use = &use_stream; + +if (write_stream_object_file (&reader, size,type_name (OBJ_BLOB ), + oid, dry_run)) +die ("failed to write object in stream" ); +if (strict && !dry_run) { +struct blob *blob =lookup_blob (the_repository, oid); +if (blob) + blob->object.flags |=FLAG_WRITTEN ; +else +die ("invalid blob object from stream" ); + } + obj_list[nr].obj =NULL ; +} +static void unpack_non_delta_entry (enum object_type type,unsigned long size,unsigned nr) { -void *buf =get_data (size); +void *buf; + +/* Write large blob in stream without allocating full buffer. */ +if (type ==OBJ_BLOB && size > big_file_threshold) { +write_stream_blob (nr, size); +return ; + } + buf =get_data (size);if (!dry_run && buf)write_object (nr, type, buf, size);else object-file.c +138
-11@@ -1913,6 +1913,28 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)return fd; } +static int write_object_buffer (struct git_zstream *stream,git_hash_ctx *c, +int fd,unsigned char *compressed, +int compressed_len,const void *buf, +size_t len,int flush) +{ +int ret; + + stream->next_in = (void *)buf; + stream->avail_in = len; +do { +unsigned char *in0 = stream->next_in; + ret =git_deflate (stream, flush); + the_hash_algo->update_fn (c, in0, stream->next_in - in0); +if (write_buffer (fd, compressed, stream->next_out - compressed) <0 ) +die (_ ("unable to write loose object file" )); + stream->next_out = compressed; + stream->avail_out = compressed_len; + }while (ret ==Z_OK ); + +return ret; +} +static int write_loose_object (const struct object_id *oid,char *hdr,int hdrlen,const void *buf,unsigned long len,time_t mtime)@@ -1949,17 +1971,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr, the_hash_algo->update_fn (&c, hdr, hdrlen);/* Then the data itself.. */ - stream.next_in = (void *)buf; - stream.avail_in = len; -do { -unsigned char *in0 = stream.next_in; - ret =git_deflate (&stream,Z_FINISH ); - the_hash_algo->update_fn (&c, in0, stream.next_in - in0); -if (write_buffer (fd, compressed, stream.next_out - compressed) <0 ) -die (_ ("unable to write loose object file" )); - stream.next_out = compressed; - stream.avail_out =sizeof (compressed); - }while (ret ==Z_OK ); + ret =write_object_buffer (&stream, &c, fd, compressed, +sizeof (compressed), buf, len, +Z_FINISH );if (ret !=Z_STREAM_END ) die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),@@ -2020,6 +2034,119 @@ int write_object_file(const void *buf, unsigned long len, const char *type,return write_loose_object (oid, hdr, hdrlen, buf, len,0 ); } +int write_stream_object_file (struct git_zstream_reader *reader, +unsigned long len,const char *type, +struct object_id *oid, +int dry_run) +{ +git_zstream istream, ostream; +unsigned char buf[8192 ], compressed[4096 ]; +char hdr[MAX_HEADER_LEN ]; +int istatus, ostatus, fd =0 , hdrlen, dirlen, flush =0 ; +int ret =0 ; +git_hash_ctx c; +struct strbuf tmp_file =STRBUF_INIT ; +struct strbuf filename =STRBUF_INIT ; + +/* Write tmpfile in objects dir, because oid is unknown */ +if (!dry_run) { +strbuf_addstr (&filename, the_repository->objects->odb->path); +strbuf_addch (&filename,'/' ); + fd =create_tmpfile (&tmp_file, filename.buf); +if (fd <0 ) { +if (errno ==EACCES ) + ret =error (_ ("insufficient permission for adding an object to repository database %s" ), +get_object_directory ()); +else + ret =error_errno (_ ("unable to create temporary file" )); + goto cleanup; + } + } + +memset (&istream,0 ,sizeof (istream)); + istream.next_out = buf; + istream.avail_out =sizeof (buf); +git_inflate_init (&istream); + +if (!dry_run) { +/* Set it up */ +git_deflate_init (&ostream, zlib_compression_level); + ostream.next_out = compressed; + ostream.avail_out =sizeof (compressed); + the_hash_algo->init_fn (&c); + +/* First header */ + hdrlen =xsnprintf (hdr,sizeof (hdr),"%s %" PRIuMAX, type, + (uintmax_t )len) +1 ; + ostream.next_in = (unsigned char *)hdr; + ostream.avail_in = hdrlen; +while (git_deflate (&ostream,0 ) ==Z_OK ) + ;/* nothing */ + the_hash_algo->update_fn (&c, hdr, hdrlen); + } + +/* Then the data itself */ +do { +unsigned char *last_out = istream.next_out; + reader->fill (&istream); + istatus =git_inflate (&istream,0 ); +if (istatus ==Z_STREAM_END ) + flush =Z_FINISH ; + reader->use (&istream); +if (!dry_run) + ostatus =write_object_buffer (&ostream, &c, fd, compressed, +sizeof (compressed), last_out, + istream.next_out - last_out, + flush); + istream.next_out = buf; + istream.avail_out =sizeof (buf); + }while (istatus ==Z_OK ); + +if (istream.total_out != len || istatus !=Z_STREAM_END ) +die (_ ("inflate returned %d" ), istatus); +git_inflate_end (&istream); + +if (dry_run) + goto cleanup; + +if (ostatus !=Z_STREAM_END ) +die (_ ("unable to deflate new object (%d)" ), ostatus); + ostatus =git_deflate_end_gently (&ostream); +if (ostatus !=Z_OK ) +die (_ ("deflateEnd on object failed (%d)" ), ostatus); + the_hash_algo->final_fn (oid->hash, &c); +close_loose_object (fd); + +/* We get the oid now */ +loose_object_path (the_repository, &filename, oid); + + dirlen =directory_size (filename.buf); +if (dirlen) { +struct strbuf dir =STRBUF_INIT ; +/* +* Make sure the directory exists; note that the contents +* of the buffer are undefined after mkstemp returns an +* error, so we have to rewrite the whole buffer from +* scratch. +*/ +strbuf_add (&dir, filename.buf, dirlen -1 ); +if (mkdir (dir.buf,0777 ) && errno !=EEXIST ) { +unlink_or_warn (tmp_file.buf); +strbuf_release (&dir); + ret =-1 ; + goto cleanup; + } +strbuf_release (&dir); + } + + ret =finalize_object_file (tmp_file.buf, filename.buf); + +cleanup: +strbuf_release (&tmp_file); +strbuf_release (&filename); +return ret; +} +int hash_object_file_literally (const void *buf,unsigned long len,const char *type,struct object_id *oid,unsigned flags)object-store.h +9
-0@@ -33,6 +33,11 @@ struct object_directory {char *path; }; +struct git_zstream_reader { +void (*fill)(struct git_zstream *); +void (*use)(struct git_zstream *); +}; + KHASH_INIT(odb_path_map, const char * /* key: odb_path */, struct object_directory *, 1, fspathhash, fspatheq)@@ -225,6 +230,10 @@ int hash_object_file(const struct git_hash_algo *algo, const void *buf,int write_object_file (const void *buf,unsigned long len,const char *type,struct object_id *oid); +int write_stream_object_file (struct git_zstream_reader *reader, +unsigned long len,const char *type, +struct object_id *oid,int dry_run); +int hash_object_file_literally (const void *buf,unsigned long len,const char *type,struct object_id *oid,unsigned flags);t/t5590-receive-unpack-objects.sh new file +92
-1@@ -0,0 +1,92 @@ +#!/bin/sh +# +# Copyright (c) 2021 Han Xin +# + +test_description='Test unpack-objects when receive pack' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +. ./test-lib.sh + +test_expect_success "create commit with big blobs (1.5 MB)" ' + test-tool genrandom foo 1500000 >big-blob && + test_commit --append foo big-blob && + test-tool genrandom bar 1500000 >big-blob && + test_commit --append bar big-blob && + ( + cd .git && + find objects/?? -type f | sort + ) >expect && + git repack -ad +' + +test_expect_success 'setup GIT_ALLOC_LIMIT to 1MB' ' + GIT_ALLOC_LIMIT=1m && + export GIT_ALLOC_LIMIT +' + +test_expect_success 'prepare dest repository' ' + git init --bare dest.git && + git -C dest.git config core.bigFileThreshold 2m && + git -C dest.git config receive.unpacklimit 100 +' + +test_expect_success 'fail to push: cannot allocate' ' + test_must_fail git push dest.git HEAD 2>err && + test_i18ngrep "remote: fatal: attempting to allocate" err && + ( + cd dest.git && + find objects/?? -type f | sort + ) >actual && + ! test_cmp expect actual +' + +test_expect_success 'set a lower bigfile threshold' ' + git -C dest.git config core.bigFileThreshold 1m +' + +test_expect_success 'unpack big object in stream' ' + git push dest.git HEAD && + git -C dest.git fsck && + ( + cd dest.git && + find objects/?? -type f | sort + ) >actual && + test_cmp expect actual +' + +test_expect_success 'setup for unpack-objects dry-run test' ' + PACK=$(echo main | git pack-objects --progress --revs test) && + unset GIT_ALLOC_LIMIT && + git init --bare unpack-test.git +' + +test_expect_success 'unpack-objects dry-run with large threshold' ' + ( + cd unpack-test.git && + git config core.bigFileThreshold 2m && + git unpack-objects -n <../test-$PACK.pack + ) && + ( + cd unpack-test.git && + find objects/ -type f + ) >actual && + test_must_be_empty actual +' + +test_expect_success 'unpack-objects dry-run with small threshold' ' + ( + cd unpack-test.git && + git config core.bigFileThreshold 1m && + git unpack-objects -n <../test-$PACK.pack + ) && + ( + cd unpack-test.git && + find objects/ -type f + ) >actual && + test_must_be_empty actual +' + +test_done --2.33.0.1.g09a6bb964f.dirty
Quoted 35 lines · Han Xin
From: Han Xin <[email protected]>
When calling "
unpack_non_delta_entry()", will allocate full memory for the whole size of the unpacked object and write the buffer to loose file on disk. This may lead to OOM for the git-unpack-objects process when unpacking a very large object.In function "
unpack_delta_entry()", will also allocate full memory to buffer the whole delta, but since there will be no delta for an object larger than "core.bigFileThreshold", this issue is moderate.To resolve the OOM issue in "git-unpack-objects", we can unpack large object to file in stream, and use the setting of "core.bigFileThreshold" as the threshold for large object.
Reviewed-by Jiang Xin <[email protected]>Signed-off-by Han Xin <[email protected]>--- builtin/unpack-objects.c | 41 +++++++- object-file.c | 149 +++++++++++++++++++++++++++--- object-store.h | 9 ++ t/t5590-receive-unpack-objects.sh | 92 ++++++++++++++++++ 4 files changed, 279 insertions(+), 12 deletions(-) create mode 100755 t/t5590-receive-unpack-objects.shbuiltin/unpack-objects.c +2
-0@@ -320,11 +320,50 @@ static void added_object(unsigned nr, enum object_type type, } } +static void fill_stream (struct git_zstream *stream) +{
builtin/unpack-objects.c static void added_object(unsigned nr, enum object_type type, · line 334
Quoted 10 lines
builtin/unpack-objects.c +10
-0+ stream->next_in =fill (1 ); + stream->avail_in = len; +} + +static void use_stream (struct git_zstream *stream) +{ +use (len - stream->avail_in); +} + +static void write_stream_blob (unsigned nr,unsigned long size)
Can we use size_t for the size, and possibly nr, to improve compatibility with Windows systems where unsigned long is only 32 bits?
There has been some work in the past on providing large file support on Windows, which requires numerous long -> size_t changes.
Philip
Quoted 335 lines
builtin/unpack-objects.c +28
-1+{ +struct git_zstream_reader reader; +struct object_id *oid = &obj_list[nr].oid; + + reader.fill = &fill_stream; + reader.use = &use_stream; + +if (write_stream_object_file (&reader, size,type_name (OBJ_BLOB ), + oid, dry_run)) +die ("failed to write object in stream" ); +if (strict && !dry_run) { +struct blob *blob =lookup_blob (the_repository, oid); +if (blob) + blob->object.flags |=FLAG_WRITTEN ; +else +die ("invalid blob object from stream" ); + } + obj_list[nr].obj =NULL ; +} +static void unpack_non_delta_entry (enum object_type type,unsigned long size,unsigned nr) { -void *buf =get_data (size); +void *buf; + +/* Write large blob in stream without allocating full buffer. */ +if (type ==OBJ_BLOB && size > big_file_threshold) { +write_stream_blob (nr, size); +return ; + } + buf =get_data (size);if (!dry_run && buf)write_object (nr, type, buf, size);else object-file.c +138
-11@@ -1913,6 +1913,28 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)return fd; } +static int write_object_buffer (struct git_zstream *stream,git_hash_ctx *c, +int fd,unsigned char *compressed, +int compressed_len,const void *buf, +size_t len,int flush) +{ +int ret; + + stream->next_in = (void *)buf; + stream->avail_in = len; +do { +unsigned char *in0 = stream->next_in; + ret =git_deflate (stream, flush); + the_hash_algo->update_fn (c, in0, stream->next_in - in0); +if (write_buffer (fd, compressed, stream->next_out - compressed) <0 ) +die (_ ("unable to write loose object file" )); + stream->next_out = compressed; + stream->avail_out = compressed_len; + }while (ret ==Z_OK ); + +return ret; +} +static int write_loose_object (const struct object_id *oid,char *hdr,int hdrlen,const void *buf,unsigned long len,time_t mtime)@@ -1949,17 +1971,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr, the_hash_algo->update_fn (&c, hdr, hdrlen);/* Then the data itself.. */ - stream.next_in = (void *)buf; - stream.avail_in = len; -do { -unsigned char *in0 = stream.next_in; - ret =git_deflate (&stream,Z_FINISH ); - the_hash_algo->update_fn (&c, in0, stream.next_in - in0); -if (write_buffer (fd, compressed, stream.next_out - compressed) <0 ) -die (_ ("unable to write loose object file" )); - stream.next_out = compressed; - stream.avail_out =sizeof (compressed); - }while (ret ==Z_OK ); + ret =write_object_buffer (&stream, &c, fd, compressed, +sizeof (compressed), buf, len, +Z_FINISH );if (ret !=Z_STREAM_END ) die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),@@ -2020,6 +2034,119 @@ int write_object_file(const void *buf, unsigned long len, const char *type,return write_loose_object (oid, hdr, hdrlen, buf, len,0 ); } +int write_stream_object_file (struct git_zstream_reader *reader, +unsigned long len,const char *type, +struct object_id *oid, +int dry_run) +{ +git_zstream istream, ostream; +unsigned char buf[8192 ], compressed[4096 ]; +char hdr[MAX_HEADER_LEN ]; +int istatus, ostatus, fd =0 , hdrlen, dirlen, flush =0 ; +int ret =0 ; +git_hash_ctx c; +struct strbuf tmp_file =STRBUF_INIT ; +struct strbuf filename =STRBUF_INIT ; + +/* Write tmpfile in objects dir, because oid is unknown */ +if (!dry_run) { +strbuf_addstr (&filename, the_repository->objects->odb->path); +strbuf_addch (&filename,'/' ); + fd =create_tmpfile (&tmp_file, filename.buf); +if (fd <0 ) { +if (errno ==EACCES ) + ret =error (_ ("insufficient permission for adding an object to repository database %s" ), +get_object_directory ()); +else + ret =error_errno (_ ("unable to create temporary file" )); + goto cleanup; + } + } + +memset (&istream,0 ,sizeof (istream)); + istream.next_out = buf; + istream.avail_out =sizeof (buf); +git_inflate_init (&istream); + +if (!dry_run) { +/* Set it up */ +git_deflate_init (&ostream, zlib_compression_level); + ostream.next_out = compressed; + ostream.avail_out =sizeof (compressed); + the_hash_algo->init_fn (&c); + +/* First header */ + hdrlen =xsnprintf (hdr,sizeof (hdr),"%s %" PRIuMAX, type, + (uintmax_t )len) +1 ; + ostream.next_in = (unsigned char *)hdr; + ostream.avail_in = hdrlen; +while (git_deflate (&ostream,0 ) ==Z_OK ) + ;/* nothing */ + the_hash_algo->update_fn (&c, hdr, hdrlen); + } + +/* Then the data itself */ +do { +unsigned char *last_out = istream.next_out; + reader->fill (&istream); + istatus =git_inflate (&istream,0 ); +if (istatus ==Z_STREAM_END ) + flush =Z_FINISH ; + reader->use (&istream); +if (!dry_run) + ostatus =write_object_buffer (&ostream, &c, fd, compressed, +sizeof (compressed), last_out, + istream.next_out - last_out, + flush); + istream.next_out = buf; + istream.avail_out =sizeof (buf); + }while (istatus ==Z_OK ); + +if (istream.total_out != len || istatus !=Z_STREAM_END ) +die (_ ("inflate returned %d" ), istatus); +git_inflate_end (&istream); + +if (dry_run) + goto cleanup; + +if (ostatus !=Z_STREAM_END ) +die (_ ("unable to deflate new object (%d)" ), ostatus); + ostatus =git_deflate_end_gently (&ostream); +if (ostatus !=Z_OK ) +die (_ ("deflateEnd on object failed (%d)" ), ostatus); + the_hash_algo->final_fn (oid->hash, &c); +close_loose_object (fd); + +/* We get the oid now */ +loose_object_path (the_repository, &filename, oid); + + dirlen =directory_size (filename.buf); +if (dirlen) { +struct strbuf dir =STRBUF_INIT ; +/* +* Make sure the directory exists; note that the contents +* of the buffer are undefined after mkstemp returns an +* error, so we have to rewrite the whole buffer from +* scratch. +*/ +strbuf_add (&dir, filename.buf, dirlen -1 ); +if (mkdir (dir.buf,0777 ) && errno !=EEXIST ) { +unlink_or_warn (tmp_file.buf); +strbuf_release (&dir); + ret =-1 ; + goto cleanup; + } +strbuf_release (&dir); + } + + ret =finalize_object_file (tmp_file.buf, filename.buf); + +cleanup: +strbuf_release (&tmp_file); +strbuf_release (&filename); +return ret; +} +int hash_object_file_literally (const void *buf,unsigned long len,const char *type,struct object_id *oid,unsigned flags)object-store.h +9
-0@@ -33,6 +33,11 @@ struct object_directory {char *path; }; +struct git_zstream_reader { +void (*fill)(struct git_zstream *); +void (*use)(struct git_zstream *); +}; + KHASH_INIT(odb_path_map, const char * /* key: odb_path */, struct object_directory *, 1, fspathhash, fspatheq)@@ -225,6 +230,10 @@ int hash_object_file(const struct git_hash_algo *algo, const void *buf,int write_object_file (const void *buf,unsigned long len,const char *type,struct object_id *oid); +int write_stream_object_file (struct git_zstream_reader *reader, +unsigned long len,const char *type, +struct object_id *oid,int dry_run); +int hash_object_file_literally (const void *buf,unsigned long len,const char *type,struct object_id *oid,unsigned flags);t/t5590-receive-unpack-objects.sh new file +92
-0@@ -0,0 +1,92 @@ +#!/bin/sh +# +# Copyright (c) 2021 Han Xin +# + +test_description='Test unpack-objects when receive pack' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +. ./test-lib.sh + +test_expect_success "create commit with big blobs (1.5 MB)" ' + test-tool genrandom foo 1500000 >big-blob && + test_commit --append foo big-blob && + test-tool genrandom bar 1500000 >big-blob && + test_commit --append bar big-blob && + ( + cd .git && + find objects/?? -type f | sort + ) >expect && + git repack -ad +' + +test_expect_success 'setup GIT_ALLOC_LIMIT to 1MB' ' + GIT_ALLOC_LIMIT=1m && + export GIT_ALLOC_LIMIT +' + +test_expect_success 'prepare dest repository' ' + git init --bare dest.git && + git -C dest.git config core.bigFileThreshold 2m && + git -C dest.git config receive.unpacklimit 100 +' + +test_expect_success 'fail to push: cannot allocate' ' + test_must_fail git push dest.git HEAD 2>err && + test_i18ngrep "remote: fatal: attempting to allocate" err && + ( + cd dest.git && + find objects/?? -type f | sort + ) >actual && + ! test_cmp expect actual +' + +test_expect_success 'set a lower bigfile threshold' ' + git -C dest.git config core.bigFileThreshold 1m +' + +test_expect_success 'unpack big object in stream' ' + git push dest.git HEAD && + git -C dest.git fsck && + ( + cd dest.git && + find objects/?? -type f | sort + ) >actual && + test_cmp expect actual +' + +test_expect_success 'setup for unpack-objects dry-run test' ' + PACK=$(echo main | git pack-objects --progress --revs test) && + unset GIT_ALLOC_LIMIT && + git init --bare unpack-test.git +' + +test_expect_success 'unpack-objects dry-run with large threshold' ' + ( + cd unpack-test.git && + git config core.bigFileThreshold 2m && + git unpack-objects -n <../test-$PACK.pack + ) && + ( + cd unpack-test.git && + find objects/ -type f + ) >actual && + test_must_be_empty actual +' + +test_expect_success 'unpack-objects dry-run with small threshold' ' + ( + cd unpack-test.git && + git config core.bigFileThreshold 1m && + git unpack-objects -n <../test-$PACK.pack + ) && + ( + cd unpack-test.git && + find objects/ -type f + ) >actual && + test_must_be_empty actual +' + +test_done
Quoted 51 lines · Philip Oakley <[email protected]> 于2021年10月20日周三 下午10:43写道:
Quoted 35 lines · Han Xin
From: Han Xin <[email protected]>
When calling "
unpack_non_delta_entry()", will allocate full memory for the whole size of the unpacked object and write the buffer to loose file on disk. This may lead to OOM for the git-unpack-objects process when unpacking a very large object.In function "
unpack_delta_entry()", will also allocate full memory to buffer the whole delta, but since there will be no delta for an object larger than "core.bigFileThreshold", this issue is moderate.To resolve the OOM issue in "git-unpack-objects", we can unpack large object to file in stream, and use the setting of "core.bigFileThreshold" as the threshold for large object.
Reviewed-by Jiang Xin <[email protected]>Signed-off-by Han Xin <[email protected]>--- builtin/unpack-objects.c | 41 +++++++- object-file.c | 149 +++++++++++++++++++++++++++--- object-store.h | 9 ++ t/t5590-receive-unpack-objects.sh | 92 ++++++++++++++++++ 4 files changed, 279 insertions(+), 12 deletions(-) create mode 100755 t/t5590-receive-unpack-objects.shbuiltin/unpack-objects.c +2
-0@@ -320,11 +320,50 @@ static void added_object(unsigned nr, enum object_type type, } } +static void fill_stream (struct git_zstream *stream) +{builtin/unpack-objects.c static void added_object(unsigned nr, enum object_type type, · line 334
Quoted 10 lines
builtin/unpack-objects.c +10
-0+ stream->next_in =fill (1 ); + stream->avail_in = len; +} + +static void use_stream (struct git_zstream *stream) +{ +use (len - stream->avail_in); +} + +static void write_stream_blob (unsigned nr,unsigned long size)Can we use
size_tfor thesize, and possiblynr, to improve compatibility with Windows systems where unsigned long is only 32 bits?
Quoted 4 lines
There has been some work in the past on providing large file support on Windows, which requires numerous long ->
size_tchanges.Philip
Thanks for your review. I'm not sure if I should do this change in this patch, it will also change the type defined in unpack_one(),unpack_non_delta_entry, write_object() and many others.
Quoted 336 lines
Quoted 335 lines
builtin/unpack-objects.c +28
-1+{ +struct git_zstream_reader reader; +struct object_id *oid = &obj_list[nr].oid; + + reader.fill = &fill_stream; + reader.use = &use_stream; + +if (write_stream_object_file (&reader, size,type_name (OBJ_BLOB ), + oid, dry_run)) +die ("failed to write object in stream" ); +if (strict && !dry_run) { +struct blob *blob =lookup_blob (the_repository, oid); +if (blob) + blob->object.flags |=FLAG_WRITTEN ; +else +die ("invalid blob object from stream" ); + } + obj_list[nr].obj =NULL ; +} +static void unpack_non_delta_entry (enum object_type type,unsigned long size,unsigned nr) { -void *buf =get_data (size); +void *buf; + +/* Write large blob in stream without allocating full buffer. */ +if (type ==OBJ_BLOB && size > big_file_threshold) { +write_stream_blob (nr, size); +return ; + } + buf =get_data (size);if (!dry_run && buf)write_object (nr, type, buf, size);else object-file.c +138
-11@@ -1913,6 +1913,28 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)return fd; } +static int write_object_buffer (struct git_zstream *stream,git_hash_ctx *c, +int fd,unsigned char *compressed, +int compressed_len,const void *buf, +size_t len,int flush) +{ +int ret; + + stream->next_in = (void *)buf; + stream->avail_in = len; +do { +unsigned char *in0 = stream->next_in; + ret =git_deflate (stream, flush); + the_hash_algo->update_fn (c, in0, stream->next_in - in0); +if (write_buffer (fd, compressed, stream->next_out - compressed) <0 ) +die (_ ("unable to write loose object file" )); + stream->next_out = compressed; + stream->avail_out = compressed_len; + }while (ret ==Z_OK ); + +return ret; +} +static int write_loose_object (const struct object_id *oid,char *hdr,int hdrlen,const void *buf,unsigned long len,time_t mtime)@@ -1949,17 +1971,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr, the_hash_algo->update_fn (&c, hdr, hdrlen);/* Then the data itself.. */ - stream.next_in = (void *)buf; - stream.avail_in = len; -do { -unsigned char *in0 = stream.next_in; - ret =git_deflate (&stream,Z_FINISH ); - the_hash_algo->update_fn (&c, in0, stream.next_in - in0); -if (write_buffer (fd, compressed, stream.next_out - compressed) <0 ) -die (_ ("unable to write loose object file" )); - stream.next_out = compressed; - stream.avail_out =sizeof (compressed); - }while (ret ==Z_OK ); + ret =write_object_buffer (&stream, &c, fd, compressed, +sizeof (compressed), buf, len, +Z_FINISH );if (ret !=Z_STREAM_END ) die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),@@ -2020,6 +2034,119 @@ int write_object_file(const void *buf, unsigned long len, const char *type,return write_loose_object (oid, hdr, hdrlen, buf, len,0 ); } +int write_stream_object_file (struct git_zstream_reader *reader, +unsigned long len,const char *type, +struct object_id *oid, +int dry_run) +{ +git_zstream istream, ostream; +unsigned char buf[8192 ], compressed[4096 ]; +char hdr[MAX_HEADER_LEN ]; +int istatus, ostatus, fd =0 , hdrlen, dirlen, flush =0 ; +int ret =0 ; +git_hash_ctx c; +struct strbuf tmp_file =STRBUF_INIT ; +struct strbuf filename =STRBUF_INIT ; + +/* Write tmpfile in objects dir, because oid is unknown */ +if (!dry_run) { +strbuf_addstr (&filename, the_repository->objects->odb->path); +strbuf_addch (&filename,'/' ); + fd =create_tmpfile (&tmp_file, filename.buf); +if (fd <0 ) { +if (errno ==EACCES ) + ret =error (_ ("insufficient permission for adding an object to repository database %s" ), +get_object_directory ()); +else + ret =error_errno (_ ("unable to create temporary file" )); + goto cleanup; + } + } + +memset (&istream,0 ,sizeof (istream)); + istream.next_out = buf; + istream.avail_out =sizeof (buf); +git_inflate_init (&istream); + +if (!dry_run) { +/* Set it up */ +git_deflate_init (&ostream, zlib_compression_level); + ostream.next_out = compressed; + ostream.avail_out =sizeof (compressed); + the_hash_algo->init_fn (&c); + +/* First header */ + hdrlen =xsnprintf (hdr,sizeof (hdr),"%s %" PRIuMAX, type, + (uintmax_t )len) +1 ; + ostream.next_in = (unsigned char *)hdr; + ostream.avail_in = hdrlen; +while (git_deflate (&ostream,0 ) ==Z_OK ) + ;/* nothing */ + the_hash_algo->update_fn (&c, hdr, hdrlen); + } + +/* Then the data itself */ +do { +unsigned char *last_out = istream.next_out; + reader->fill (&istream); + istatus =git_inflate (&istream,0 ); +if (istatus ==Z_STREAM_END ) + flush =Z_FINISH ; + reader->use (&istream); +if (!dry_run) + ostatus =write_object_buffer (&ostream, &c, fd, compressed, +sizeof (compressed), last_out, + istream.next_out - last_out, + flush); + istream.next_out = buf; + istream.avail_out =sizeof (buf); + }while (istatus ==Z_OK ); + +if (istream.total_out != len || istatus !=Z_STREAM_END ) +die (_ ("inflate returned %d" ), istatus); +git_inflate_end (&istream); + +if (dry_run) + goto cleanup; + +if (ostatus !=Z_STREAM_END ) +die (_ ("unable to deflate new object (%d)" ), ostatus); + ostatus =git_deflate_end_gently (&ostream); +if (ostatus !=Z_OK ) +die (_ ("deflateEnd on object failed (%d)" ), ostatus); + the_hash_algo->final_fn (oid->hash, &c); +close_loose_object (fd); + +/* We get the oid now */ +loose_object_path (the_repository, &filename, oid); + + dirlen =directory_size (filename.buf); +if (dirlen) { +struct strbuf dir =STRBUF_INIT ; +/* +* Make sure the directory exists; note that the contents +* of the buffer are undefined after mkstemp returns an +* error, so we have to rewrite the whole buffer from +* scratch. +*/ +strbuf_add (&dir, filename.buf, dirlen -1 ); +if (mkdir (dir.buf,0777 ) && errno !=EEXIST ) { +unlink_or_warn (tmp_file.buf); +strbuf_release (&dir); + ret =-1 ; + goto cleanup; + } +strbuf_release (&dir); + } + + ret =finalize_object_file (tmp_file.buf, filename.buf); + +cleanup: +strbuf_release (&tmp_file); +strbuf_release (&filename); +return ret; +} +int hash_object_file_literally (const void *buf,unsigned long len,const char *type,struct object_id *oid,unsigned flags)object-store.h +9
-0@@ -33,6 +33,11 @@ struct object_directory {char *path; }; +struct git_zstream_reader { +void (*fill)(struct git_zstream *); +void (*use)(struct git_zstream *); +}; + KHASH_INIT(odb_path_map, const char * /* key: odb_path */, struct object_directory *, 1, fspathhash, fspatheq)@@ -225,6 +230,10 @@ int hash_object_file(const struct git_hash_algo *algo, const void *buf,int write_object_file (const void *buf,unsigned long len,const char *type,struct object_id *oid); +int write_stream_object_file (struct git_zstream_reader *reader, +unsigned long len,const char *type, +struct object_id *oid,int dry_run); +int hash_object_file_literally (const void *buf,unsigned long len,const char *type,struct object_id *oid,unsigned flags);t/t5590-receive-unpack-objects.sh new file +92
-0@@ -0,0 +1,92 @@ +#!/bin/sh +# +# Copyright (c) 2021 Han Xin +# + +test_description='Test unpack-objects when receive pack' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +. ./test-lib.sh + +test_expect_success "create commit with big blobs (1.5 MB)" ' + test-tool genrandom foo 1500000 >big-blob && + test_commit --append foo big-blob && + test-tool genrandom bar 1500000 >big-blob && + test_commit --append bar big-blob && + ( + cd .git && + find objects/?? -type f | sort + ) >expect && + git repack -ad +' + +test_expect_success 'setup GIT_ALLOC_LIMIT to 1MB' ' + GIT_ALLOC_LIMIT=1m && + export GIT_ALLOC_LIMIT +' + +test_expect_success 'prepare dest repository' ' + git init --bare dest.git && + git -C dest.git config core.bigFileThreshold 2m && + git -C dest.git config receive.unpacklimit 100 +' + +test_expect_success 'fail to push: cannot allocate' ' + test_must_fail git push dest.git HEAD 2>err && + test_i18ngrep "remote: fatal: attempting to allocate" err && + ( + cd dest.git && + find objects/?? -type f | sort + ) >actual && + ! test_cmp expect actual +' + +test_expect_success 'set a lower bigfile threshold' ' + git -C dest.git config core.bigFileThreshold 1m +' + +test_expect_success 'unpack big object in stream' ' + git push dest.git HEAD && + git -C dest.git fsck && + ( + cd dest.git && + find objects/?? -type f | sort + ) >actual && + test_cmp expect actual +' + +test_expect_success 'setup for unpack-objects dry-run test' ' + PACK=$(echo main | git pack-objects --progress --revs test) && + unset GIT_ALLOC_LIMIT && + git init --bare unpack-test.git +' + +test_expect_success 'unpack-objects dry-run with large threshold' ' + ( + cd unpack-test.git && + git config core.bigFileThreshold 2m && + git unpack-objects -n <../test-$PACK.pack + ) && + ( + cd unpack-test.git && + find objects/ -type f + ) >actual && + test_must_be_empty actual +' + +test_expect_success 'unpack-objects dry-run with small threshold' ' + ( + cd unpack-test.git && + git config core.bigFileThreshold 1m && + git unpack-objects -n <../test-$PACK.pack + ) && + ( + cd unpack-test.git && + find objects/ -type f + ) >actual && + test_must_be_empty actual +' + +test_done
Quoted 4 lines · Han Xin
Quoted 4 lines
Quoted 1 line
+static void
write_stream_blob(unsigned nr, unsigned long size)Can we use
size_tfor thesize, and possiblynr, to improve compatibility with Windows systems where unsigned long is only 32 bits?
Quoted 8 lines
Quoted 4 lines
There has been some work in the past on providing large file support on Windows, which requires numerous long ->
size_tchanges.Philip
Thanks for your review. I'm not sure if I should do this change in this patch, it will also change the type defined in
unpack_one(),unpack_non_delta_entry,write_object()and many others.
I was mainly raising the issue regarding the 4GB (sometime 2GB) limitations on Windows which has been a problem for many years.
I had been thinking of not changing the nr (number of objects limit) as 2G objects is hopefully already sufficient, even for thargest of repos (though IIUC their index file size did break the 32bit size limit).
Staying with the existing types won't make the situation any worse, so from that perspective the change isn't needed.
--
Philip
Any more suggestions?
Quoted 384 lines · Han Xin <[email protected]> 于2021年10月9日周六 下午4:21写道:
From: Han Xin <[email protected]>
When calling "
unpack_non_delta_entry()", will allocate full memory for the whole size of the unpacked object and write the buffer to loose file on disk. This may lead to OOM for the git-unpack-objects process when unpacking a very large object.In function "
unpack_delta_entry()", will also allocate full memory to buffer the whole delta, but since there will be no delta for an object larger than "core.bigFileThreshold", this issue is moderate.To resolve the OOM issue in "git-unpack-objects", we can unpack large object to file in stream, and use the setting of "core.bigFileThreshold" as the threshold for large object.
Reviewed-by Jiang Xin <[email protected]>Signed-off-by Han Xin <[email protected]>--- builtin/unpack-objects.c | 41 +++++++- object-file.c | 149 +++++++++++++++++++++++++++--- object-store.h | 9 ++ t/t5590-receive-unpack-objects.sh | 92 ++++++++++++++++++ 4 files changed, 279 insertions(+), 12 deletions(-) create mode 100755 t/t5590-receive-unpack-objects.shbuiltin/unpack-objects.c +40
-1@@ -320,11 +320,50 @@ static void added_object(unsigned nr, enum object_type type, } } +static void fill_stream (struct git_zstream *stream) +{ + stream->next_in =fill (1 ); + stream->avail_in = len; +} + +static void use_stream (struct git_zstream *stream) +{ +use (len - stream->avail_in); +} + +static void write_stream_blob (unsigned nr,unsigned long size) +{ +struct git_zstream_reader reader; +struct object_id *oid = &obj_list[nr].oid; + + reader.fill = &fill_stream; + reader.use = &use_stream; + +if (write_stream_object_file (&reader, size,type_name (OBJ_BLOB ), + oid, dry_run)) +die ("failed to write object in stream" ); +if (strict && !dry_run) { +struct blob *blob =lookup_blob (the_repository, oid); +if (blob) + blob->object.flags |=FLAG_WRITTEN ; +else +die ("invalid blob object from stream" ); + } + obj_list[nr].obj =NULL ; +} +static void unpack_non_delta_entry (enum object_type type,unsigned long size,unsigned nr) { -void *buf =get_data (size); +void *buf; + +/* Write large blob in stream without allocating full buffer. */ +if (type ==OBJ_BLOB && size > big_file_threshold) { +write_stream_blob (nr, size); +return ; + } + buf =get_data (size);if (!dry_run && buf)write_object (nr, type, buf, size);else object-file.c +138
-11@@ -1913,6 +1913,28 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)return fd; } +static int write_object_buffer (struct git_zstream *stream,git_hash_ctx *c, +int fd,unsigned char *compressed, +int compressed_len,const void *buf, +size_t len,int flush) +{ +int ret; + + stream->next_in = (void *)buf; + stream->avail_in = len; +do { +unsigned char *in0 = stream->next_in; + ret =git_deflate (stream, flush); + the_hash_algo->update_fn (c, in0, stream->next_in - in0); +if (write_buffer (fd, compressed, stream->next_out - compressed) <0 ) +die (_ ("unable to write loose object file" )); + stream->next_out = compressed; + stream->avail_out = compressed_len; + }while (ret ==Z_OK ); + +return ret; +} +static int write_loose_object (const struct object_id *oid,char *hdr,int hdrlen,const void *buf,unsigned long len,time_t mtime)@@ -1949,17 +1971,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr, the_hash_algo->update_fn (&c, hdr, hdrlen);/* Then the data itself.. */ - stream.next_in = (void *)buf; - stream.avail_in = len; -do { -unsigned char *in0 = stream.next_in; - ret =git_deflate (&stream,Z_FINISH ); - the_hash_algo->update_fn (&c, in0, stream.next_in - in0); -if (write_buffer (fd, compressed, stream.next_out - compressed) <0 ) -die (_ ("unable to write loose object file" )); - stream.next_out = compressed; - stream.avail_out =sizeof (compressed); - }while (ret ==Z_OK ); + ret =write_object_buffer (&stream, &c, fd, compressed, +sizeof (compressed), buf, len, +Z_FINISH );if (ret !=Z_STREAM_END ) die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),@@ -2020,6 +2034,119 @@ int write_object_file(const void *buf, unsigned long len, const char *type,return write_loose_object (oid, hdr, hdrlen, buf, len,0 ); } +int write_stream_object_file (struct git_zstream_reader *reader, +unsigned long len,const char *type, +struct object_id *oid, +int dry_run) +{ +git_zstream istream, ostream; +unsigned char buf[8192 ], compressed[4096 ]; +char hdr[MAX_HEADER_LEN ]; +int istatus, ostatus, fd =0 , hdrlen, dirlen, flush =0 ; +int ret =0 ; +git_hash_ctx c; +struct strbuf tmp_file =STRBUF_INIT ; +struct strbuf filename =STRBUF_INIT ; + +/* Write tmpfile in objects dir, because oid is unknown */ +if (!dry_run) { +strbuf_addstr (&filename, the_repository->objects->odb->path); +strbuf_addch (&filename,'/' ); + fd =create_tmpfile (&tmp_file, filename.buf); +if (fd <0 ) { +if (errno ==EACCES ) + ret =error (_ ("insufficient permission for adding an object to repository database %s" ), +get_object_directory ()); +else + ret =error_errno (_ ("unable to create temporary file" )); + goto cleanup; + } + } + +memset (&istream,0 ,sizeof (istream)); + istream.next_out = buf; + istream.avail_out =sizeof (buf); +git_inflate_init (&istream); + +if (!dry_run) { +/* Set it up */ +git_deflate_init (&ostream, zlib_compression_level); + ostream.next_out = compressed; + ostream.avail_out =sizeof (compressed); + the_hash_algo->init_fn (&c); + +/* First header */ + hdrlen =xsnprintf (hdr,sizeof (hdr),"%s %" PRIuMAX, type, + (uintmax_t )len) +1 ; + ostream.next_in = (unsigned char *)hdr; + ostream.avail_in = hdrlen; +while (git_deflate (&ostream,0 ) ==Z_OK ) + ;/* nothing */ + the_hash_algo->update_fn (&c, hdr, hdrlen); + } + +/* Then the data itself */ +do { +unsigned char *last_out = istream.next_out; + reader->fill (&istream); + istatus =git_inflate (&istream,0 ); +if (istatus ==Z_STREAM_END ) + flush =Z_FINISH ; + reader->use (&istream); +if (!dry_run) + ostatus =write_object_buffer (&ostream, &c, fd, compressed, +sizeof (compressed), last_out, + istream.next_out - last_out, + flush); + istream.next_out = buf; + istream.avail_out =sizeof (buf); + }while (istatus ==Z_OK ); + +if (istream.total_out != len || istatus !=Z_STREAM_END ) +die (_ ("inflate returned %d" ), istatus); +git_inflate_end (&istream); + +if (dry_run) + goto cleanup; + +if (ostatus !=Z_STREAM_END ) +die (_ ("unable to deflate new object (%d)" ), ostatus); + ostatus =git_deflate_end_gently (&ostream); +if (ostatus !=Z_OK ) +die (_ ("deflateEnd on object failed (%d)" ), ostatus); + the_hash_algo->final_fn (oid->hash, &c); +close_loose_object (fd); + +/* We get the oid now */ +loose_object_path (the_repository, &filename, oid); + + dirlen =directory_size (filename.buf); +if (dirlen) { +struct strbuf dir =STRBUF_INIT ; +/* +* Make sure the directory exists; note that the contents +* of the buffer are undefined after mkstemp returns an +* error, so we have to rewrite the whole buffer from +* scratch. +*/ +strbuf_add (&dir, filename.buf, dirlen -1 ); +if (mkdir (dir.buf,0777 ) && errno !=EEXIST ) { +unlink_or_warn (tmp_file.buf); +strbuf_release (&dir); + ret =-1 ; + goto cleanup; + } +strbuf_release (&dir); + } + + ret =finalize_object_file (tmp_file.buf, filename.buf); + +cleanup: +strbuf_release (&tmp_file); +strbuf_release (&filename); +return ret; +} +int hash_object_file_literally (const void *buf,unsigned long len,const char *type,struct object_id *oid,unsigned flags)object-store.h +9
-0@@ -33,6 +33,11 @@ struct object_directory {char *path; }; +struct git_zstream_reader { +void (*fill)(struct git_zstream *); +void (*use)(struct git_zstream *); +}; + KHASH_INIT(odb_path_map, const char * /* key: odb_path */, struct object_directory *, 1, fspathhash, fspatheq)@@ -225,6 +230,10 @@ int hash_object_file(const struct git_hash_algo *algo, const void *buf,int write_object_file (const void *buf,unsigned long len,const char *type,struct object_id *oid); +int write_stream_object_file (struct git_zstream_reader *reader, +unsigned long len,const char *type, +struct object_id *oid,int dry_run); +int hash_object_file_literally (const void *buf,unsigned long len,const char *type,struct object_id *oid,unsigned flags);t/t5590-receive-unpack-objects.sh new file +92
-1@@ -0,0 +1,92 @@ +#!/bin/sh +# +# Copyright (c) 2021 Han Xin +# + +test_description='Test unpack-objects when receive pack' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +. ./test-lib.sh + +test_expect_success "create commit with big blobs (1.5 MB)" ' + test-tool genrandom foo 1500000 >big-blob && + test_commit --append foo big-blob && + test-tool genrandom bar 1500000 >big-blob && + test_commit --append bar big-blob && + ( + cd .git && + find objects/?? -type f | sort + ) >expect && + git repack -ad +' + +test_expect_success 'setup GIT_ALLOC_LIMIT to 1MB' ' + GIT_ALLOC_LIMIT=1m && + export GIT_ALLOC_LIMIT +' + +test_expect_success 'prepare dest repository' ' + git init --bare dest.git && + git -C dest.git config core.bigFileThreshold 2m && + git -C dest.git config receive.unpacklimit 100 +' + +test_expect_success 'fail to push: cannot allocate' ' + test_must_fail git push dest.git HEAD 2>err && + test_i18ngrep "remote: fatal: attempting to allocate" err && + ( + cd dest.git && + find objects/?? -type f | sort + ) >actual && + ! test_cmp expect actual +' + +test_expect_success 'set a lower bigfile threshold' ' + git -C dest.git config core.bigFileThreshold 1m +' + +test_expect_success 'unpack big object in stream' ' + git push dest.git HEAD && + git -C dest.git fsck && + ( + cd dest.git && + find objects/?? -type f | sort + ) >actual && + test_cmp expect actual +' + +test_expect_success 'setup for unpack-objects dry-run test' ' + PACK=$(echo main | git pack-objects --progress --revs test) && + unset GIT_ALLOC_LIMIT && + git init --bare unpack-test.git +' + +test_expect_success 'unpack-objects dry-run with large threshold' ' + ( + cd unpack-test.git && + git config core.bigFileThreshold 2m && + git unpack-objects -n <../test-$PACK.pack + ) && + ( + cd unpack-test.git && + find objects/ -type f + ) >actual && + test_must_be_empty actual +' + +test_expect_success 'unpack-objects dry-run with small threshold' ' + ( + cd unpack-test.git && + git config core.bigFileThreshold 1m && + git unpack-objects -n <../test-$PACK.pack + ) && + ( + cd unpack-test.git && + find objects/ -type f + ) >actual && + test_must_be_empty actual +' + +test_done --2.33.0.1.g09a6bb964f.dirty
(replies to the alibaba-inc.com aren't getting through for me)
Quoted 9 lines · Han Xin
Any more suggestions?
Quoted 6 lines · Han Xin <[email protected]> 于2021年10月9日周六 下午4:21写道:
From: Han Xin <[email protected]>
When calling "
unpack_non_delta_entry()", will allocate full memory for the whole size of the unpacked object and write the buffer to loose file on disk. This may lead to OOM for the git-unpack-objects process when unpacking a very large object.
Is it possible to split the patch into smaller pieces, taking each item separately?
For large files (as above), it should be possible to stream the unpacking direct to disk, in the same way that the zlib reading is chunked. However having the same 'code' in two places would need to be addressed (the DRY principle).
At the moment on LLP64 systems (Windows) there is already a long (32bit) vs size_t (64bit) problem there (zlib stream), and the size_t problem then permeates the wider codebase.
The normal Git file operations does tend to memory map whole files, but here it looks like you can bypass that.
Quoted 4 lines
Quoted 4 lines
In function "
unpack_delta_entry()", will also allocate full memory to buffer the whole delta, but since there will be no delta for an object larger than "core.bigFileThreshold", this issue is moderate.
What does 'moderate' mean here? Does it mean there is a simple test that allows you to side step the whole problem?
Quoted 4 lines
Quoted 4 lines
To resolve the OOM issue in "git-unpack-objects", we can unpack large object to file in stream, and use the setting of "core.bigFileThreshold" as the threshold for large object.
Is this "core.bigFileThreshold" the core element? If so, it is too far down the commit message. The readers have already (potentially) misread the message and reacted too soon. Perhaps: "use core.bigFileThreshold to avoid mmap OOM limits when unpacking".
--
Philip
Quoted 369 lines
Quoted 369 lines
Reviewed-by Jiang Xin <[email protected]>Signed-off-by Han Xin <[email protected]>--- builtin/unpack-objects.c | 41 +++++++- object-file.c | 149 +++++++++++++++++++++++++++--- object-store.h | 9 ++ t/t5590-receive-unpack-objects.sh | 92 ++++++++++++++++++ 4 files changed, 279 insertions(+), 12 deletions(-) create mode 100755 t/t5590-receive-unpack-objects.shbuiltin/unpack-objects.c +40
-1@@ -320,11 +320,50 @@ static void added_object(unsigned nr, enum object_type type, } } +static void fill_stream (struct git_zstream *stream) +{ + stream->next_in =fill (1 ); + stream->avail_in = len; +} + +static void use_stream (struct git_zstream *stream) +{ +use (len - stream->avail_in); +} + +static void write_stream_blob (unsigned nr,unsigned long size) +{ +struct git_zstream_reader reader; +struct object_id *oid = &obj_list[nr].oid; + + reader.fill = &fill_stream; + reader.use = &use_stream; + +if (write_stream_object_file (&reader, size,type_name (OBJ_BLOB ), + oid, dry_run)) +die ("failed to write object in stream" ); +if (strict && !dry_run) { +struct blob *blob =lookup_blob (the_repository, oid); +if (blob) + blob->object.flags |=FLAG_WRITTEN ; +else +die ("invalid blob object from stream" ); + } + obj_list[nr].obj =NULL ; +} +static void unpack_non_delta_entry (enum object_type type,unsigned long size,unsigned nr) { -void *buf =get_data (size); +void *buf; + +/* Write large blob in stream without allocating full buffer. */ +if (type ==OBJ_BLOB && size > big_file_threshold) { +write_stream_blob (nr, size); +return ; + } + buf =get_data (size);if (!dry_run && buf)write_object (nr, type, buf, size);else object-file.c +138
-11@@ -1913,6 +1913,28 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)return fd; } +static int write_object_buffer (struct git_zstream *stream,git_hash_ctx *c, +int fd,unsigned char *compressed, +int compressed_len,const void *buf, +size_t len,int flush) +{ +int ret; + + stream->next_in = (void *)buf; + stream->avail_in = len; +do { +unsigned char *in0 = stream->next_in; + ret =git_deflate (stream, flush); + the_hash_algo->update_fn (c, in0, stream->next_in - in0); +if (write_buffer (fd, compressed, stream->next_out - compressed) <0 ) +die (_ ("unable to write loose object file" )); + stream->next_out = compressed; + stream->avail_out = compressed_len; + }while (ret ==Z_OK ); + +return ret; +} +static int write_loose_object (const struct object_id *oid,char *hdr,int hdrlen,const void *buf,unsigned long len,time_t mtime)@@ -1949,17 +1971,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr, the_hash_algo->update_fn (&c, hdr, hdrlen);/* Then the data itself.. */ - stream.next_in = (void *)buf; - stream.avail_in = len; -do { -unsigned char *in0 = stream.next_in; - ret =git_deflate (&stream,Z_FINISH ); - the_hash_algo->update_fn (&c, in0, stream.next_in - in0); -if (write_buffer (fd, compressed, stream.next_out - compressed) <0 ) -die (_ ("unable to write loose object file" )); - stream.next_out = compressed; - stream.avail_out =sizeof (compressed); - }while (ret ==Z_OK ); + ret =write_object_buffer (&stream, &c, fd, compressed, +sizeof (compressed), buf, len, +Z_FINISH );if (ret !=Z_STREAM_END ) die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),@@ -2020,6 +2034,119 @@ int write_object_file(const void *buf, unsigned long len, const char *type,return write_loose_object (oid, hdr, hdrlen, buf, len,0 ); } +int write_stream_object_file (struct git_zstream_reader *reader, +unsigned long len,const char *type, +struct object_id *oid, +int dry_run) +{ +git_zstream istream, ostream; +unsigned char buf[8192 ], compressed[4096 ]; +char hdr[MAX_HEADER_LEN ]; +int istatus, ostatus, fd =0 , hdrlen, dirlen, flush =0 ; +int ret =0 ; +git_hash_ctx c; +struct strbuf tmp_file =STRBUF_INIT ; +struct strbuf filename =STRBUF_INIT ; + +/* Write tmpfile in objects dir, because oid is unknown */ +if (!dry_run) { +strbuf_addstr (&filename, the_repository->objects->odb->path); +strbuf_addch (&filename,'/' ); + fd =create_tmpfile (&tmp_file, filename.buf); +if (fd <0 ) { +if (errno ==EACCES ) + ret =error (_ ("insufficient permission for adding an object to repository database %s" ), +get_object_directory ()); +else + ret =error_errno (_ ("unable to create temporary file" )); + goto cleanup; + } + } + +memset (&istream,0 ,sizeof (istream)); + istream.next_out = buf; + istream.avail_out =sizeof (buf); +git_inflate_init (&istream); + +if (!dry_run) { +/* Set it up */ +git_deflate_init (&ostream, zlib_compression_level); + ostream.next_out = compressed; + ostream.avail_out =sizeof (compressed); + the_hash_algo->init_fn (&c); + +/* First header */ + hdrlen =xsnprintf (hdr,sizeof (hdr),"%s %" PRIuMAX, type, + (uintmax_t )len) +1 ; + ostream.next_in = (unsigned char *)hdr; + ostream.avail_in = hdrlen; +while (git_deflate (&ostream,0 ) ==Z_OK ) + ;/* nothing */ + the_hash_algo->update_fn (&c, hdr, hdrlen); + } + +/* Then the data itself */ +do { +unsigned char *last_out = istream.next_out; + reader->fill (&istream); + istatus =git_inflate (&istream,0 ); +if (istatus ==Z_STREAM_END ) + flush =Z_FINISH ; + reader->use (&istream); +if (!dry_run) + ostatus =write_object_buffer (&ostream, &c, fd, compressed, +sizeof (compressed), last_out, + istream.next_out - last_out, + flush); + istream.next_out = buf; + istream.avail_out =sizeof (buf); + }while (istatus ==Z_OK ); + +if (istream.total_out != len || istatus !=Z_STREAM_END ) +die (_ ("inflate returned %d" ), istatus); +git_inflate_end (&istream); + +if (dry_run) + goto cleanup; + +if (ostatus !=Z_STREAM_END ) +die (_ ("unable to deflate new object (%d)" ), ostatus); + ostatus =git_deflate_end_gently (&ostream); +if (ostatus !=Z_OK ) +die (_ ("deflateEnd on object failed (%d)" ), ostatus); + the_hash_algo->final_fn (oid->hash, &c); +close_loose_object (fd); + +/* We get the oid now */ +loose_object_path (the_repository, &filename, oid); + + dirlen =directory_size (filename.buf); +if (dirlen) { +struct strbuf dir =STRBUF_INIT ; +/* +* Make sure the directory exists; note that the contents +* of the buffer are undefined after mkstemp returns an +* error, so we have to rewrite the whole buffer from +* scratch. +*/ +strbuf_add (&dir, filename.buf, dirlen -1 ); +if (mkdir (dir.buf,0777 ) && errno !=EEXIST ) { +unlink_or_warn (tmp_file.buf); +strbuf_release (&dir); + ret =-1 ; + goto cleanup; + } +strbuf_release (&dir); + } + + ret =finalize_object_file (tmp_file.buf, filename.buf); + +cleanup: +strbuf_release (&tmp_file); +strbuf_release (&filename); +return ret; +} +int hash_object_file_literally (const void *buf,unsigned long len,const char *type,struct object_id *oid,unsigned flags)object-store.h +9
-0@@ -33,6 +33,11 @@ struct object_directory {char *path; }; +struct git_zstream_reader { +void (*fill)(struct git_zstream *); +void (*use)(struct git_zstream *); +}; + KHASH_INIT(odb_path_map, const char * /* key: odb_path */, struct object_directory *, 1, fspathhash, fspatheq)@@ -225,6 +230,10 @@ int hash_object_file(const struct git_hash_algo *algo, const void *buf,int write_object_file (const void *buf,unsigned long len,const char *type,struct object_id *oid); +int write_stream_object_file (struct git_zstream_reader *reader, +unsigned long len,const char *type, +struct object_id *oid,int dry_run); +int hash_object_file_literally (const void *buf,unsigned long len,const char *type,struct object_id *oid,unsigned flags);t/t5590-receive-unpack-objects.sh new file +92
-1@@ -0,0 +1,92 @@ +#!/bin/sh +# +# Copyright (c) 2021 Han Xin +# + +test_description='Test unpack-objects when receive pack' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +. ./test-lib.sh + +test_expect_success "create commit with big blobs (1.5 MB)" ' + test-tool genrandom foo 1500000 >big-blob && + test_commit --append foo big-blob && + test-tool genrandom bar 1500000 >big-blob && + test_commit --append bar big-blob && + ( + cd .git && + find objects/?? -type f | sort + ) >expect && + git repack -ad +' + +test_expect_success 'setup GIT_ALLOC_LIMIT to 1MB' ' + GIT_ALLOC_LIMIT=1m && + export GIT_ALLOC_LIMIT +' + +test_expect_success 'prepare dest repository' ' + git init --bare dest.git && + git -C dest.git config core.bigFileThreshold 2m && + git -C dest.git config receive.unpacklimit 100 +' + +test_expect_success 'fail to push: cannot allocate' ' + test_must_fail git push dest.git HEAD 2>err && + test_i18ngrep "remote: fatal: attempting to allocate" err && + ( + cd dest.git && + find objects/?? -type f | sort + ) >actual && + ! test_cmp expect actual +' + +test_expect_success 'set a lower bigfile threshold' ' + git -C dest.git config core.bigFileThreshold 1m +' + +test_expect_success 'unpack big object in stream' ' + git push dest.git HEAD && + git -C dest.git fsck && + ( + cd dest.git && + find objects/?? -type f | sort + ) >actual && + test_cmp expect actual +' + +test_expect_success 'setup for unpack-objects dry-run test' ' + PACK=$(echo main | git pack-objects --progress --revs test) && + unset GIT_ALLOC_LIMIT && + git init --bare unpack-test.git +' + +test_expect_success 'unpack-objects dry-run with large threshold' ' + ( + cd unpack-test.git && + git config core.bigFileThreshold 2m && + git unpack-objects -n <../test-$PACK.pack + ) && + ( + cd unpack-test.git && + find objects/ -type f + ) >actual && + test_must_be_empty actual +' + +test_expect_success 'unpack-objects dry-run with small threshold' ' + ( + cd unpack-test.git && + git config core.bigFileThreshold 1m && + git unpack-objects -n <../test-$PACK.pack + ) && + ( + cd unpack-test.git && + find objects/ -type f + ) >actual && + test_must_be_empty actual +' + +test_done --2.33.0.1.g09a6bb964f.dirty
PATCH v2 1/6 object-file: refactor write_loose_object() to support inputstream
In reply to Han XinFrom: Han Xin <[email protected]>
Refactor write_loose_object() to support inputstream, in the same way that zlib reading is chunked.
Using "in_stream" instead of "void *buf", we needn't to allocate enough memory in advance, and only part of the contents will be read when called "in_stream.read()".
--- object-file.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++---- object-store.h | 5 +++++ 2 files changed, 51 insertions(+), 4 deletions(-)
object-file.c +46 -4
@@ -1860,8 +1860,26 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)return fd; } +struct input_data_from_buffer { +const char *buf; +unsigned long len; +}; + +static const char *read_input_stream_from_buffer (void *data,unsigned long *len) +{ +struct input_data_from_buffer *input = (struct input_data_from_buffer *)data; + +if (input->len ==0 ) { + *len =0 ; +return NULL ; + } + *len = input->len; + input->len =0 ; +return input->buf; +} +static int write_loose_object (const struct object_id *oid,char *hdr, -int hdrlen,const void *buf,unsigned long len, +int hdrlen,struct input_stream *in_stream,time_t mtime,unsigned flags) {int fd, ret;
@@ -1871,6 +1889,8 @@ static int write_loose_object(const struct object_id *oid, char *hdr,struct object_id parano_oid;static struct strbuf tmp_file =STRBUF_INIT ;static struct strbuf filename =STRBUF_INIT ; +const char *buf; +unsigned long len;loose_object_path (the_repository, &filename, oid);
@@ -1898,6 +1918,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr, the_hash_algo->update_fn (&c, hdr, hdrlen);/* Then the data itself.. */ + buf = in_stream->read (in_stream->data, &len); stream.next_in = (void *)buf; stream.avail_in = len; do {
@@ -1960,6 +1981,13 @@ int write_object_file_flags(const void *buf, unsigned long len, {char hdr[MAX_HEADER_LEN ];int hdrlen =sizeof (hdr); +struct input_stream in_stream = { + .read = read_input_stream_from_buffer, + .data = (void *)&(struct input_data_from_buffer ) { + .buf = buf, + .len = len, + }, + };/* Normally if we have it in the pack then we do not bother writing * it out into .git/objects/??/?{38} file.
@@ -1968,7 +1996,7 @@ int write_object_file_flags(const void *buf, unsigned long len, &hdrlen);if (freshen_packed_object (oid) ||freshen_loose_object (oid))return 0 ; -return write_loose_object (oid, hdr, hdrlen, buf, len,0 , flags); +return write_loose_object (oid, hdr, hdrlen, &in_stream,0 , flags); } int hash_object_file_literally(const void *buf, unsigned long len,
@@ -1977,6 +2005,13 @@ int hash_object_file_literally(const void *buf, unsigned long len, {char *header;int hdrlen, status =0 ; +struct input_stream in_stream = { + .read = read_input_stream_from_buffer, + .data = (void *)&(struct input_data_from_buffer ) { + .buf = buf, + .len = len, + }, + };/* type string, SP, %lu of the length plus NUL must fit this */ hdrlen =strlen (type) +MAX_HEADER_LEN ;
@@ -1988,7 +2023,7 @@ int hash_object_file_literally(const void *buf, unsigned long len, goto cleanup;if (freshen_packed_object (oid) ||freshen_loose_object (oid)) goto cleanup; - status =write_loose_object (oid, header, hdrlen, buf, len,0 ,0 ); + status =write_loose_object (oid, header, hdrlen, &in_stream,0 ,0 ); cleanup:free (header);
@@ -2003,14 +2038,21 @@ int force_object_loose(const struct object_id *oid, time_t mtime)char hdr[MAX_HEADER_LEN ];int hdrlen;int ret; +struct input_data_from_buffer data; +struct input_stream in_stream = { + .read = read_input_stream_from_buffer, + .data = &data, + };if (has_loose_object (oid))return 0 ; buf =read_object (the_repository, oid, &type, &len);if (!buf)return error (_ ("cannot read object for %s" ),oid_to_hex (oid)); + data.buf = buf; + data.len = len; hdrlen =xsnprintf (hdr,sizeof (hdr),"%s %" PRIuMAX ,type_name (type), (uintmax_t )len) +1 ; - ret =write_loose_object (oid, hdr, hdrlen, buf, len, mtime,0 ); + ret =write_loose_object (oid, hdr, hdrlen, &in_stream, mtime,0 );free (buf);return ret;
object-store.h +5 -0
@@ -34,6 +34,11 @@ struct object_directory {char *path; }; +struct input_stream { +const char *(*read)(void * data,unsigned long *len); +void *data; +}; + KHASH_INIT(odb_path_map, const char * /* key: odb_path */, struct object_directory *, 1, fspathhash, fspatheq)
Signature · 1 line
2.33.1.44.g9344627884.agit.6.5.4
Re: [PATCH v2 1/6] object-file: refactor write_loose_object() to support inputstream
Quoted 2 lines · Han Xin
From: Han Xin <[email protected]>
It would be better to provide a cover letter describing changes in v2, such as:
* Make "write_loose_object()" a public method, so we can reuse it in "unpack_non_delta_entry()".
(But I doubt we can use "write_object_file_flags()" public
function, without make this change.)
* Add an new interface "input_stream" as an argument for "write_loose_object()", so that we can feed data to "write_loose_object()" from buffer or from zlib stream.
Quoted 2 lines
Refactor
write_loose_object()to support inputstream, in the same way that zlib reading is chunked.
In the beginning of your commit log, you should describe the problem, such as:
We used to read the full content of a blob into buffer in "unpack_non_delta_entry()" by calling:
void *buf = get_data(size);
This will consume lots of memory for a very big blob object.
Quoted 16 lines
Using "
in_stream" instead of "void *buf", we needn't to allocate enough memory in advance, and only part of the contents will be read when called "in_stream.read()".Helped-by Jiang Xin <[email protected]>Signed-off-by Han Xin <[email protected]>--- object-file.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++---- object-store.h | 5 +++++ 2 files changed, 51 insertions(+), 4 deletions(-)object-file.c +0
-0
object-file.c static int create_tmpfile(struct strbuf *tmp, const char *filename) · line 1868
Quoted 9 lines
object-file.c +6
-0@@ -1860,8 +1860,26 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)return fd; } +struct input_data_from_buffer { +const char *buf; +unsigned long len; +}; + +static const char *read_input_stream_from_buffer (void *data,unsigned long *len)
Use "const void *" for the type of return variable, just like input argument for write_loose_object()?
Quoted 19 lines
object-file.c +13
-1+{ +struct input_data_from_buffer *input = (struct input_data_from_buffer *)data; + +if (input->len ==0 ) { + *len =0 ; +return NULL ; + } + *len = input->len; + input->len =0 ; +return input->buf; +} +static int write_loose_object (const struct object_id *oid,char *hdr, -int hdrlen,const void *buf,unsigned long len, +int hdrlen,struct input_stream *in_stream,time_t mtime,unsigned flags) {int fd, ret;
object-file.c static int write_loose_object(const struct object_id *oid, char *hdr, · line 1892
Quoted 4 lines
object-file.c +1
-0@@ -1871,6 +1889,8 @@ static int write_loose_object(const struct object_id *oid, char *hdr,struct object_id parano_oid;static struct strbuf tmp_file =STRBUF_INIT ;static struct strbuf filename =STRBUF_INIT ; +const char *buf;
Can we use the same prototype as the original: "const void *buf" ?
Quoted 100 lines
object-file.c +26
-3+unsigned long len;loose_object_path (the_repository, &filename, oid);@@ -1898,6 +1918,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr, the_hash_algo->update_fn (&c, hdr, hdrlen);/* Then the data itself.. */ + buf = in_stream->read (in_stream->data, &len); stream.next_in = (void *)buf; stream.avail_in = len; do {@@ -1960,6 +1981,13 @@ int write_object_file_flags(const void *buf, unsigned long len, {char hdr[MAX_HEADER_LEN ];int hdrlen =sizeof (hdr); +struct input_stream in_stream = { + .read = read_input_stream_from_buffer, + .data = (void *)&(struct input_data_from_buffer ) { + .buf = buf, + .len = len, + }, + };/* Normally if we have it in the pack then we do not bother writing * it out into .git/objects/??/?{38} file. @@ -1968,7 +1996,7 @@ int write_object_file_flags(const void *buf, unsigned long len, &hdrlen);if (freshen_packed_object (oid) ||freshen_loose_object (oid))return 0 ; -return write_loose_object (oid, hdr, hdrlen, buf, len,0 , flags); +return write_loose_object (oid, hdr, hdrlen, &in_stream,0 , flags); } int hash_object_file_literally(const void *buf, unsigned long len,@@ -1977,6 +2005,13 @@ int hash_object_file_literally(const void *buf, unsigned long len, {char *header;int hdrlen, status =0 ; +struct input_stream in_stream = { + .read = read_input_stream_from_buffer, + .data = (void *)&(struct input_data_from_buffer ) { + .buf = buf, + .len = len, + }, + };/* type string, SP, %lu of the length plus NUL must fit this */ hdrlen =strlen (type) +MAX_HEADER_LEN ;@@ -1988,7 +2023,7 @@ int hash_object_file_literally(const void *buf, unsigned long len, goto cleanup;if (freshen_packed_object (oid) ||freshen_loose_object (oid)) goto cleanup; - status =write_loose_object (oid, header, hdrlen, buf, len,0 ,0 ); + status =write_loose_object (oid, header, hdrlen, &in_stream,0 ,0 ); cleanup:free (header);@@ -2003,14 +2038,21 @@ int force_object_loose(const struct object_id *oid, time_t mtime)char hdr[MAX_HEADER_LEN ];int hdrlen;int ret; +struct input_data_from_buffer data; +struct input_stream in_stream = { + .read = read_input_stream_from_buffer, + .data = &data, + };if (has_loose_object (oid))return 0 ; buf =read_object (the_repository, oid, &type, &len);if (!buf)return error (_ ("cannot read object for %s" ),oid_to_hex (oid)); + data.buf = buf; + data.len = len; hdrlen =xsnprintf (hdr,sizeof (hdr),"%s %" PRIuMAX ,type_name (type), (uintmax_t )len) +1 ; - ret =write_loose_object (oid, hdr, hdrlen, buf, len, mtime,0 ); + ret =write_loose_object (oid, hdr, hdrlen, &in_stream, mtime,0 );free (buf);return ret;object-store.h +5
-1@@ -34,6 +34,11 @@ struct object_directory {char *path; }; +struct input_stream { +const char *(*read)(void * data,unsigned long *len); +void *data; +}; + KHASH_INIT(odb_path_map, const char * /* key: odb_path */, struct object_directory *, 1, fspathhash, fspatheq) --2.33.1.44.g9344627884.agit.6.5.4
Re: [PATCH v2 1/6] object-file: refactor write_loose_object() to support inputstream
Quoted 18 lines · Jiang Xin
Quoted 2 lines · Han Xin
From: Han Xin <[email protected]>
It would be better to provide a cover letter describing changes in v2, such as:
* Make "
write_loose_object()" a public method, so we can reuse it in "unpack_non_delta_entry()".
(But I doubt we can use "write_object_file_flags()" public
function, without make this change.)* Add an new interface "
input_stream" as an argument for "write_loose_object()", so that we can feed data to "write_loose_object()" from buffer or from zlib stream.Quoted 2 lines
Refactor
write_loose_object()to support inputstream, in the same way that zlib reading is chunked.
Quoted 8 lines
In the beginning of your commit log, you should describe the problem, such as:
We used to read the full content of a blob into buffer in "
unpack_non_delta_entry()" by calling:void *buf = get_data(size);This will consume lots of memory for a very big blob object.
I was not sure where "in_stream" came from---"use X insteads of Y", when X is what these patches invent and introduce, does not make a good explanation without explaining what X is, what problem X is attempting to solve and how.
Thanks for helping to clarify the proposed log message.
PATCH v2 2/6 object-file.c: add dry_run mode for write_loose_object()
In reply to Han XinFrom: Han Xin <[email protected]>
We will use "write_loose_object()" later to handle large blob object, which needs to work in dry_run mode.
--- object-file.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-)
object-file.c +19 -13
@@ -1880,9 +1880,10 @@ static const char *read_input_stream_from_buffer(void *data, unsigned long *len)static int write_loose_object (const struct object_id *oid,char *hdr,int hdrlen,struct input_stream *in_stream, +int dry_run,time_t mtime,unsigned flags) { -int fd, ret; +int fd, ret =0 ;unsigned char compressed[4096 ];git_zstream stream;git_hash_ctx c;
@@ -1894,14 +1895,16 @@ static int write_loose_object(const struct object_id *oid, char *hdr,loose_object_path (the_repository, &filename, oid); - fd =create_tmpfile (&tmp_file, filename.buf); -if (fd <0 ) { -if (flags &HASH_SILENT ) -return -1 ; -else if (errno ==EACCES ) -return error (_ ("insufficient permission for adding an object to repository database %s" ),get_object_directory ()); -else -return error_errno (_ ("unable to create temporary file" )); +if (!dry_run) { + fd =create_tmpfile (&tmp_file, filename.buf); +if (fd <0 ) { +if (flags &HASH_SILENT ) +return -1 ; +else if (errno ==EACCES ) +return error (_ ("insufficient permission for adding an object to repository database %s" ),get_object_directory ()); +else +return error_errno (_ ("unable to create temporary file" )); + } }/* Set it up */
@@ -1925,7 +1928,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr,unsigned char *in0 = stream.next_in; ret =git_deflate (&stream,Z_FINISH ); the_hash_algo->update_fn (&c, in0, stream.next_in - in0); -if (write_buffer (fd, compressed, stream.next_out - compressed) <0 ) +if (!dry_run &&write_buffer (fd, compressed, stream.next_out - compressed) <0 )die (_ ("unable to write loose object file" )); stream.next_out = compressed; stream.avail_out =sizeof (compressed);
@@ -1943,6 +1946,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr,die (_ ("confused by unstable object source data for %s" ),oid_to_hex (oid)); +if (dry_run) +return 0 ; +close_loose_object (fd);if (mtime) {
@@ -1996,7 +2002,7 @@ int write_object_file_flags(const void *buf, unsigned long len, &hdrlen);if (freshen_packed_object (oid) ||freshen_loose_object (oid))return 0 ; -return write_loose_object (oid, hdr, hdrlen, &in_stream,0 , flags); +return write_loose_object (oid, hdr, hdrlen, &in_stream,0 ,0 , flags); } int hash_object_file_literally(const void *buf, unsigned long len,
@@ -2023,7 +2029,7 @@ int hash_object_file_literally(const void *buf, unsigned long len, goto cleanup;if (freshen_packed_object (oid) ||freshen_loose_object (oid)) goto cleanup; - status =write_loose_object (oid, header, hdrlen, &in_stream,0 ,0 ); + status =write_loose_object (oid, header, hdrlen, &in_stream,0 ,0 ,0 ); cleanup:free (header);
@@ -2052,7 +2058,7 @@ int force_object_loose(const struct object_id *oid, time_t mtime) data.buf = buf; data.len = len; hdrlen =xsnprintf (hdr,sizeof (hdr),"%s %" PRIuMAX ,type_name (type), (uintmax_t )len) +1 ; - ret =write_loose_object (oid, hdr, hdrlen, &in_stream, mtime,0 ); + ret =write_loose_object (oid, hdr, hdrlen, &in_stream,0 , mtime,0 );free (buf);return ret;
Signature · 1 line
2.33.1.44.g9344627884.agit.6.5.4
Re: [PATCH v2 2/6] object-file.c: add dry_run mode for write_loose_object()
Quoted 5 lines · Han Xin
From: Han Xin <[email protected]>
We will use "
write_loose_object()" later to handle large blob object, which needs to work indry_runmode.
The dry_run mode comes from "builtin/unpack-object.c", throw the buffer read from "get_data()".
So why not add "dry_run" to "get_data()" instead?
If we have a dry_run version of get_data, such as "get_data(size, dry_run)", we do not have to add dry_run mode for ” write_loose_object()".
See: git grep -A5 get_data builtin/unpack-objects.cbuiltin/unpack-objects.c: void *buf = get_data(size);
builtin/unpack-objects.c-
builtin/unpack-objects.c- if (!dry_run && buf)
builtin/unpack-objects.c- write_object(nr, type, buf, size);
builtin/unpack-objects.c- else
builtin/unpack-objects.c- free(buf);
--builtin/unpack-objects.c: delta_data = get_data(delta_size);
builtin/unpack-objects.c- if (dry_run || !delta_data) {
builtin/unpack-objects.c- free(delta_data);
builtin/unpack-objects.c- return;
builtin/unpack-objects.c- }
--builtin/unpack-objects.c: delta_data = get_data(delta_size);
builtin/unpack-objects.c- if (dry_run || !delta_data) {
builtin/unpack-objects.c- free(delta_data);
builtin/unpack-objects.c- return;
builtin/unpack-objects.c- }
Quoted 96 lines
Helped-by Jiang Xin <[email protected]>Signed-off-by Han Xin <[email protected]>--- object-file.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-)object-file.c +19
-14@@ -1880,9 +1880,10 @@ static const char *read_input_stream_from_buffer(void *data, unsigned long *len)static int write_loose_object (const struct object_id *oid,char *hdr,int hdrlen,struct input_stream *in_stream, +int dry_run,time_t mtime,unsigned flags) { -int fd, ret; +int fd, ret =0 ;unsigned char compressed[4096 ];git_zstream stream;git_hash_ctx c;@@ -1894,14 +1895,16 @@ static int write_loose_object(const struct object_id *oid, char *hdr,loose_object_path (the_repository, &filename, oid); - fd =create_tmpfile (&tmp_file, filename.buf); -if (fd <0 ) { -if (flags &HASH_SILENT ) -return -1 ; -else if (errno ==EACCES ) -return error (_ ("insufficient permission for adding an object to repository database %s" ),get_object_directory ()); -else -return error_errno (_ ("unable to create temporary file" )); +if (!dry_run) { + fd =create_tmpfile (&tmp_file, filename.buf); +if (fd <0 ) { +if (flags &HASH_SILENT ) +return -1 ; +else if (errno ==EACCES ) +return error (_ ("insufficient permission for adding an object to repository database %s" ),get_object_directory ()); +else +return error_errno (_ ("unable to create temporary file" )); + } }/* Set it up */ @@ -1925,7 +1928,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr,unsigned char *in0 = stream.next_in; ret =git_deflate (&stream,Z_FINISH ); the_hash_algo->update_fn (&c, in0, stream.next_in - in0); -if (write_buffer (fd, compressed, stream.next_out - compressed) <0 ) +if (!dry_run &&write_buffer (fd, compressed, stream.next_out - compressed) <0 )die (_ ("unable to write loose object file" )); stream.next_out = compressed; stream.avail_out =sizeof (compressed);@@ -1943,6 +1946,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr,die (_ ("confused by unstable object source data for %s" ),oid_to_hex (oid)); +if (dry_run) +return 0 ; +close_loose_object (fd);if (mtime) {@@ -1996,7 +2002,7 @@ int write_object_file_flags(const void *buf, unsigned long len, &hdrlen);if (freshen_packed_object (oid) ||freshen_loose_object (oid))return 0 ; -return write_loose_object (oid, hdr, hdrlen, &in_stream,0 , flags); +return write_loose_object (oid, hdr, hdrlen, &in_stream,0 ,0 , flags); } int hash_object_file_literally(const void *buf, unsigned long len,@@ -2023,7 +2029,7 @@ int hash_object_file_literally(const void *buf, unsigned long len, goto cleanup;if (freshen_packed_object (oid) ||freshen_loose_object (oid)) goto cleanup; - status =write_loose_object (oid, header, hdrlen, &in_stream,0 ,0 ); + status =write_loose_object (oid, header, hdrlen, &in_stream,0 ,0 ,0 ); cleanup:free (header);@@ -2052,7 +2058,7 @@ int force_object_loose(const struct object_id *oid, time_t mtime) data.buf = buf; data.len = len; hdrlen =xsnprintf (hdr,sizeof (hdr),"%s %" PRIuMAX ,type_name (type), (uintmax_t )len) +1 ; - ret =write_loose_object (oid, hdr, hdrlen, &in_stream, mtime,0 ); + ret =write_loose_object (oid, hdr, hdrlen, &in_stream,0 , mtime,0 );free (buf);return ret; --2.33.1.44.g9344627884.agit.6.5.4
PATCH v2 3/6 object-file.c: handle nil oid in write_loose_object()
In reply to Han XinFrom: Han Xin <[email protected]>
When read input stream, oid can't get before reading all, and it will be filled after reading.
--- object-file.c | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-)
object-file.c +32 -2
@@ -1893,7 +1893,13 @@ static int write_loose_object(const struct object_id *oid, char *hdr,const char *buf;unsigned long len; -loose_object_path (the_repository, &filename, oid); +if (is_null_oid (oid)) { +/* When oid is not determined, save tmp file to odb path. */ +strbuf_reset (&filename); +strbuf_addstr (&filename, the_repository->objects->odb->path); +strbuf_addch (&filename,'/' ); + }else +loose_object_path (the_repository, &filename, oid);if (!dry_run) { fd =create_tmpfile (&tmp_file, filename.buf);
@@ -1942,7 +1948,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr,die (_ ("deflateEnd on object %s failed (%d)" ),oid_to_hex (oid), ret); the_hash_algo->final_oid_fn (¶no_oid, &c); -if (!oideq (oid, ¶no_oid)) +if (!is_null_oid (oid) && !oideq (oid, ¶no_oid))die (_ ("confused by unstable object source data for %s" ),oid_to_hex (oid));
@@ -1951,6 +1957,30 @@ static int write_loose_object(const struct object_id *oid, char *hdr,close_loose_object (fd); +if (is_null_oid (oid)) { +int dirlen; + +/* copy oid */ +oidcpy ((struct object_id *)oid, ¶no_oid); +/* We get the oid now */ +loose_object_path (the_repository, &filename, oid); + + dirlen =directory_size (filename.buf); +if (dirlen) { +struct strbuf dir =STRBUF_INIT ; +/* +* Make sure the directory exists; note that the +* contents of the buffer are undefined after mkstemp +* returns an error, so we have to rewrite the whole +* buffer from scratch. +*/ +strbuf_reset (&dir); +strbuf_add (&dir, filename.buf, dirlen -1 ); +if (mkdir (dir.buf,0777 ) && errno !=EEXIST ) +return -1 ; + } + } +if (mtime) {struct utimbuf utb; utb.actime = mtime;
Signature · 1 line
2.33.1.44.g9344627884.agit.6.5.4
Re: [PATCH v2 3/6] object-file.c: handle nil oid in write_loose_object()
Quoted 5 lines · Han Xin
From: Han Xin <[email protected]>
When read input stream, oid can't get before reading all, and it will be filled after reading.
Under what circumstances is the oid a null oid? Can we get the oid from “obj_list[nr].oid” ?
See unpack_non_delta_entry() of builtin/unpack-objects.c.
Quoted 68 lines
Helped-by Jiang Xin <[email protected]>Signed-off-by Han Xin <[email protected]>--- object-file.c | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-)object-file.c +32
-3@@ -1893,7 +1893,13 @@ static int write_loose_object(const struct object_id *oid, char *hdr,const char *buf;unsigned long len; -loose_object_path (the_repository, &filename, oid); +if (is_null_oid (oid)) { +/* When oid is not determined, save tmp file to odb path. */ +strbuf_reset (&filename); +strbuf_addstr (&filename, the_repository->objects->odb->path); +strbuf_addch (&filename,'/' ); + }else +loose_object_path (the_repository, &filename, oid);if (!dry_run) { fd =create_tmpfile (&tmp_file, filename.buf);@@ -1942,7 +1948,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr,die (_ ("deflateEnd on object %s failed (%d)" ),oid_to_hex (oid), ret); the_hash_algo->final_oid_fn (¶no_oid, &c); -if (!oideq (oid, ¶no_oid)) +if (!is_null_oid (oid) && !oideq (oid, ¶no_oid))die (_ ("confused by unstable object source data for %s" ),oid_to_hex (oid));@@ -1951,6 +1957,30 @@ static int write_loose_object(const struct object_id *oid, char *hdr,close_loose_object (fd); +if (is_null_oid (oid)) { +int dirlen; + +/* copy oid */ +oidcpy ((struct object_id *)oid, ¶no_oid); +/* We get the oid now */ +loose_object_path (the_repository, &filename, oid); + + dirlen =directory_size (filename.buf); +if (dirlen) { +struct strbuf dir =STRBUF_INIT ; +/* +* Make sure the directory exists; note that the +* contents of the buffer are undefined after mkstemp +* returns an error, so we have to rewrite the whole +* buffer from scratch. +*/ +strbuf_reset (&dir); +strbuf_add (&dir, filename.buf, dirlen -1 ); +if (mkdir (dir.buf,0777 ) && errno !=EEXIST ) +return -1 ; + } + } +if (mtime) {struct utimbuf utb; utb.actime = mtime; --2.33.1.44.g9344627884.agit.6.5.4
PATCH v2 4/6 object-file.c: read input stream repeatedly in write_loose_object()
In reply to Han XinFrom: Han Xin <[email protected]>
Read input stream repeatedly in write_loose_object() unless reach the end, so that we can divide the large blob write into many small blocks.
--- object-file.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-)
object-file.c +9 -5
@@ -1891,7 +1891,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr,static struct strbuf tmp_file =STRBUF_INIT ;static struct strbuf filename =STRBUF_INIT ;const char *buf; -unsigned long len; +int flush =0 ;if (is_null_oid (oid)) {/* When oid is not determined, save tmp file to odb path. */
@@ -1927,12 +1927,16 @@ static int write_loose_object(const struct object_id *oid, char *hdr, the_hash_algo->update_fn (&c, hdr, hdrlen);/* Then the data itself.. */ - buf = in_stream->read (in_stream->data, &len); - stream.next_in = (void *)buf; - stream.avail_in = len; do { unsigned char *in0 = stream.next_in; - ret = git_deflate(&stream, Z_FINISH); + if (!stream.avail_in) { + if ((buf = in_stream->read(in_stream->data, &stream.avail_in))) { + stream.next_in = (void *)buf; + in0 = (unsigned char *)buf; + } else + flush = Z_FINISH; + } + ret = git_deflate(&stream, flush); the_hash_algo->update_fn(&c, in0, stream.next_in - in0); if (!dry_run && write_buffer(fd, compressed, stream.next_out - compressed) < 0) die(_("unable to write loose object file"));
Signature · 1 line
2.33.1.44.g9344627884.agit.6.5.4
Re: [PATCH v2 4/6] object-file.c: read input stream repeatedly in write_loose_object()
Quoted 5 lines · Han Xin
From: Han Xin <[email protected]>
Read input stream repeatedly in
write_loose_object()unless reach the end, so that we can divide the large blob write into many small blocks.
In order to prepare the stream version of "write_loose_object()", we need ...
Quoted 21 lines
Signed-off-by Han Xin <[email protected]>--- object-file.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-)object-file.c +1
-1@@ -1891,7 +1891,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr,static struct strbuf tmp_file =STRBUF_INIT ;static struct strbuf filename =STRBUF_INIT ;const char *buf; -unsigned long len; +int flush =0 ;if (is_null_oid (oid)) {/* When oid is not determined, save tmp file to odb path. */ @@ -1927,12 +1927,16 @@ static int write_loose_object(const struct object_id *oid, char *hdr, the_hash_algo->update_fn (&c, hdr, hdrlen);
object-file.c static int write_loose_object(const struct object_id *oid, char *hdr, · line 1933
Quoted 10 lines
object-file.c +2
-4/* Then the data itself.. */ - buf = in_stream->read (in_stream->data, &len); - stream.next_in = (void *)buf; - stream.avail_in = len; do { unsigned char *in0 = stream.next_in; - ret = git_deflate(&stream, Z_FINISH); + if (!stream.avail_in) { + if ((buf = in_stream->read(in_stream->data, &stream.avail_in))) {
if ((buf = in_stream->read(in_stream->data, &stream.avail_in)) != NULL) {
Or split this long line into:
buf = in_stream->read(in_stream->data, &stream.avail_in);
if (buf) {object-file.c static int write_loose_object(const struct object_id *oid, char *hdr, · line 1937
Quoted 4 lines
object-file.c +4
-0+ stream.next_in = (void *)buf; + in0 = (unsigned char *)buf; + }else + flush =Z_FINISH ;
Add {} around this single line, see:
https://github.com/git/git/blob/master/Documentation/CodingGuidelines#L279-L289
Quoted 8 lines
object-file.c +2
-1+ } + ret =git_deflate (&stream, flush); the_hash_algo->update_fn (&c, in0, stream.next_in - in0);if (!dry_run &&write_buffer (fd, compressed, stream.next_out - compressed) <0 )die (_ ("unable to write loose object file" )); --2.33.1.44.g9344627884.agit.6.5.4
PATCH v2 5/6 object-store.h: add write_loose_object()
In reply to Han XinFrom: Han Xin <[email protected]>
For large loose object files, that should be possible to stream it direct to disk with "write_loose_object()".
Unlike "write_object_file()", you need to implement an "input_stream" instead of giving void *buf.
--- object-file.c | 8 ++++---- object-store.h | 5 +++++ 2 files changed, 9 insertions(+), 4 deletions(-)
object-file.c +4 -4
@@ -1878,10 +1878,10 @@ static const char *read_input_stream_from_buffer(void *data, unsigned long *len)return input->buf; } -static int write_loose_object (const struct object_id *oid,char *hdr, -int hdrlen,struct input_stream *in_stream, -int dry_run, -time_t mtime,unsigned flags) +int write_loose_object (const struct object_id *oid,char *hdr, +int hdrlen,struct input_stream *in_stream, +int dry_run, +time_t mtime,unsigned flags) {int fd, ret =0 ;unsigned char compressed[4096 ];
object-store.h +5 -0
@@ -228,6 +228,11 @@ int hash_object_file(const struct git_hash_algo *algo, const void *buf, unsigned long len, const char *type, struct object_id *oid); +int write_loose_object (const struct object_id *oid,char *hdr, +int hdrlen,struct input_stream *in_stream, +int dry_run, +time_t mtime,unsigned flags); +int write_object_file_flags (const void *buf,unsigned long len,const char *type,struct object_id *oid,unsigned flags);
Signature · 1 line
2.33.1.44.g9344627884.agit.6.5.4
PATCH v2 6/6 unpack-objects: unpack large object in stream
In reply to Han XinFrom: Han Xin <[email protected]>
When calling "unpack_non_delta_entry()", will allocate full memory for the whole size of the unpacked object and write the buffer to loose file on disk. This may lead to OOM for the git-unpack-objects process when unpacking a very large object.
In function "unpack_delta_entry()", will also allocate full memory to buffer the whole delta, but since there will be no delta for an object larger than "core.bigFileThreshold", this issue is moderate.
To resolve the OOM issue in "git-unpack-objects", we can unpack large object to file in stream, and use "core.bigFileThreshold" to avoid OOM limits when called "get_data()".
--- builtin/unpack-objects.c | 76 ++++++++++++++++++++++++- t/t5590-receive-unpack-objects.sh | 92 +++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 1 deletion(-) create mode 100755 t/t5590-receive-unpack-objects.sh
builtin/unpack-objects.c +75 -1
@@ -320,11 +320,85 @@ static void added_object(unsigned nr, enum object_type type, } } +struct input_data_from_zstream { +git_zstream *zstream; +unsigned char buf[4096 ]; +int status; +}; + +static const char *read_inflate_in_stream (void *data,unsigned long *readlen) +{ +struct input_data_from_zstream *input = data; +git_zstream *zstream = input->zstream; +void *in =fill (1 ); + +if (!len || input->status ==Z_STREAM_END ) { + *readlen =0 ; +return NULL ; + } + + zstream->next_out = input->buf; + zstream->avail_out =sizeof (input->buf); + zstream->next_in = in; + zstream->avail_in = len; + + input->status =git_inflate (zstream,0 ); +use (len - zstream->avail_in); + *readlen =sizeof (input->buf) - zstream->avail_out; + +return (const char *)input->buf; +} + +static void write_stream_blob (unsigned nr,unsigned long size) +{ +char hdr[32 ]; +int hdrlen; +git_zstream zstream; +struct input_data_from_zstream data; +struct input_stream in_stream = { + .read = read_inflate_in_stream, + .data = &data, + }; +struct object_id *oid = &obj_list[nr].oid; +int ret; + +memset (&zstream,0 ,sizeof (zstream)); +memset (&data,0 ,sizeof (data)); + data.zstream = &zstream; +git_inflate_init (&zstream); + +/* Generate the header */ + hdrlen =xsnprintf (hdr,sizeof (hdr),"%s %" PRIuMAX,type_name (OBJ_BLOB ), (uintmax_t )size) +1 ; + +if ((ret =write_loose_object (oid, hdr, hdrlen, &in_stream, dry_run,0 ,0 ))) +die (_ ("failed to write object in stream %d" ), ret); + +if (zstream.total_out != size || data.status !=Z_STREAM_END ) +die (_ ("inflate returned %d" ), data.status); +git_inflate_end (&zstream); + +if (strict && !dry_run) { +struct blob *blob =lookup_blob (the_repository, oid); +if (blob) + blob->object.flags |=FLAG_WRITTEN ; +else +die ("invalid blob object from stream" ); + } + obj_list[nr].obj =NULL ; +} +static void unpack_non_delta_entry (enum object_type type,unsigned long size,unsigned nr) { -void *buf =get_data (size); +void *buf; + +/* Write large blob in stream without allocating full buffer. */ +if (type ==OBJ_BLOB && size > big_file_threshold) { +write_stream_blob (nr, size); +return ; + } + buf =get_data (size);if (!dry_run && buf)write_object (nr, type, buf, size);else
t/t5590-receive-unpack-objects.sh new file +92 -0
92 lines not on this page · raw message
Signature · 1 line
2.33.1.44.g9344627884.agit.6.5.4
Quoted 96 lines · Han Xin
From: Han Xin <[email protected]>
When calling "
unpack_non_delta_entry()", will allocate full memory for the whole size of the unpacked object and write the buffer to loose file on disk. This may lead to OOM for the git-unpack-objects process when unpacking a very large object.In function "
unpack_delta_entry()", will also allocate full memory to buffer the whole delta, but since there will be no delta for an object larger than "core.bigFileThreshold", this issue is moderate.To resolve the OOM issue in "git-unpack-objects", we can unpack large object to file in stream, and use "core.bigFileThreshold" to avoid OOM limits when called "
get_data()".Signed-off-by Han Xin <[email protected]>--- builtin/unpack-objects.c | 76 ++++++++++++++++++++++++- t/t5590-receive-unpack-objects.sh | 92 +++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 1 deletion(-) create mode 100755 t/t5590-receive-unpack-objects.shbuiltin/unpack-objects.c +65
-0@@ -320,11 +320,85 @@ static void added_object(unsigned nr, enum object_type type, } } +struct input_data_from_zstream { +git_zstream *zstream; +unsigned char buf[4096 ]; +int status; +}; + +static const char *read_inflate_in_stream (void *data,unsigned long *readlen) +{ +struct input_data_from_zstream *input = data; +git_zstream *zstream = input->zstream; +void *in =fill (1 ); + +if (!len || input->status ==Z_STREAM_END ) { + *readlen =0 ; +return NULL ; + } + + zstream->next_out = input->buf; + zstream->avail_out =sizeof (input->buf); + zstream->next_in = in; + zstream->avail_in = len; + + input->status =git_inflate (zstream,0 ); +use (len - zstream->avail_in); + *readlen =sizeof (input->buf) - zstream->avail_out; + +return (const char *)input->buf; +} + +static void write_stream_blob (unsigned nr,unsigned long size) +{ +char hdr[32 ]; +int hdrlen; +git_zstream zstream; +struct input_data_from_zstream data; +struct input_stream in_stream = { + .read = read_inflate_in_stream, + .data = &data, + }; +struct object_id *oid = &obj_list[nr].oid; +int ret; + +memset (&zstream,0 ,sizeof (zstream)); +memset (&data,0 ,sizeof (data)); + data.zstream = &zstream; +git_inflate_init (&zstream); + +/* Generate the header */ + hdrlen =xsnprintf (hdr,sizeof (hdr),"%s %" PRIuMAX,type_name (OBJ_BLOB ), (uintmax_t )size) +1 ; + +if ((ret =write_loose_object (oid, hdr, hdrlen, &in_stream, dry_run,0 ,0 ))) +die (_ ("failed to write object in stream %d" ), ret); + +if (zstream.total_out != size || data.status !=Z_STREAM_END ) +die (_ ("inflate returned %d" ), data.status); +git_inflate_end (&zstream); + +if (strict && !dry_run) { +struct blob *blob =lookup_blob (the_repository, oid); +if (blob) + blob->object.flags |=FLAG_WRITTEN ; +else +die ("invalid blob object from stream" ); + } + obj_list[nr].obj =NULL ;
builtin/unpack-objects.c static void added_object(unsigned nr, enum object_type type, · line 396
Quoted 10 lines
builtin/unpack-objects.c +6
-1+} +static void unpack_non_delta_entry (enum object_type type,unsigned long size,unsigned nr) { -void *buf =get_data (size); +void *buf; + +/* Write large blob in stream without allocating full buffer. */ +if (type ==OBJ_BLOB && size > big_file_threshold) {
Default size of big_file_threshold is 512m. Can we use "write_stream_blob" for all objects? Can we get a more suitable threshold through some benchmark data?
builtin/unpack-objects.c static void added_object(unsigned nr, enum object_type type, · line 399
Quoted 3 lines
builtin/unpack-objects.c +3
-0+write_stream_blob (nr, size); +return ; + }
--
Jiang Xin
v3 cover unpack large objects in stream
In reply to Han XinFrom: Han Xin <[email protected]>
Although we do not recommend users push large binary files to the git repositories, it's difficult to prevent them from doing so. Once, we found a problem with a surge in memory usage on the server. The source of the problem is that a user submitted a single object with a size of 15GB. Once someone initiates a git push, the git process will immediately allocate 15G of memory, resulting in an OOM risk.
Through further analysis, we found that when we execute git unpack-objects, in unpack_non_delta_entry(), "void *buf = get_data(size);" will directly allocate memory equal to the size of the object. This is quite a scary thing, because the pre-receive hook has not been executed at this time, and we cannot avoid this by hooks.
I got inspiration from the deflate process of zlib, maybe it would be a good idea to change unpack-objects to stream deflate.
Changes since v2:
* Rewrite commit messages and make changes suggested by Jiang Xin.
* Remove the commit "object-file.c: add dry_run mode for write_loose_object()" and use a new commit "unpack-objects.c: add dry_run mode for get_data()" instead.
Han Xin (5):
object-file: refactor write_loose_object() to read buffer from stream object-file.c: handle undetermined oid in write_loose_object() object-file.c: read stream in a loop in write_loose_object() unpack-objects.c: add dry_run mode for get_data()
unpack-objects: unpack_non_delta_entry() read data in a stream
builtin/unpack-objects.c | 92 +++++++++++++++++++++++++-- object-file.c | 98 +++++++++++++++++++++++++---- object-store.h | 9 +++ t/t5590-unpack-non-delta-objects.sh | 76 ++++++++++++++++++++++ 4 files changed, 257 insertions(+), 18 deletions(-) create mode 100755 t/t5590-unpack-non-delta-objects.sh
Range-diff against v2:
1: 01672f50a0 ! 1: 8640b04f6d object-file: refactor write_loose_object() to support inputstream
@@ Metadata
Author: Han Xin <[email protected]>
## Commit message ##
- object-file: refactor write_loose_object() to support inputstream
+ object-file: refactor write_loose_object() to read buffer from stream
- Refactor write_loose_object() to support inputstream, in the same way
- that zlib reading is chunked.
+ We used to call "get_data()" in "unpack_non_delta_entry()" to read the
+ entire contents of a blob object, no matter how big it is. This
+ implementation may consume all the memory and cause OOM.
- Using "in_stream" instead of "void *buf", we needn't to allocate enough
- memory in advance, and only part of the contents will be read when
- called "in_stream.read()".
+ This can be improved by feeding data to "write_loose_object()" in a
+ stream. The input stream is implemented as an interface. In the first
+ step, we make a simple implementation, feeding the entire buffer in the
+ "stream" to "write_loose_object()" as a refactor.
Helped-by: Jiang Xin <[email protected]>
Signed-off-by: Han Xin <[email protected]>
@@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filenam
return fd;
}
-+struct input_data_from_buffer {
-+ const char *buf;
++struct simple_input_stream_data {
++ const void *buf;
+ unsigned long len;
+};
+
-+static const char *read_input_stream_from_buffer(void *data, unsigned long *len)
++static const void *feed_simple_input_stream(struct input_stream *in_stream, unsigned long *len)
+{
-+ struct input_data_from_buffer *input = (struct input_data_from_buffer *)data;
++ struct simple_input_stream_data *data = in_stream->data;
+
-+ if (input->len == 0) {
++ if (data->len == 0) {
+ *len = 0;
+ return NULL;
+ }
-+ *len = input->len;
-+ input->len = 0;
-+ return input->buf;
++ *len = data->len;
++ data->len = 0;
++ return data->buf;
+}
+
static int write_loose_object(const struct object_id *oid, char *hdr,
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
struct object_id parano_oid;
static struct strbuf tmp_file = STRBUF_INIT;
static struct strbuf filename = STRBUF_INIT;
-+ const char *buf;
++ const void *buf;
+ unsigned long len;
loose_object_path(the_repository, &filename, oid);
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
the_hash_algo->update_fn(&c, hdr, hdrlen);
/* Then the data itself.. */
-+ buf = in_stream->read(in_stream->data, &len);
++ buf = in_stream->read(in_stream, &len);
stream.next_in = (void *)buf;
stream.avail_in = len;
do {
@@ object-file.c: int write_object_file_flags(const void *buf, unsigned long len,
char hdr[MAX_HEADER_LEN];
int hdrlen = sizeof(hdr);
+ struct input_stream in_stream = {
-+ .read = read_input_stream_from_buffer,
-+ .data = (void *)&(struct input_data_from_buffer) {
++ .read = feed_simple_input_stream,
++ .data = (void *)&(struct simple_input_stream_data) {
+ .buf = buf,
+ .len = len,
+ },
@@ object-file.c: int hash_object_file_literally(const void *buf, unsigned long len
char *header;
int hdrlen, status = 0;
+ struct input_stream in_stream = {
-+ .read = read_input_stream_from_buffer,
-+ .data = (void *)&(struct input_data_from_buffer) {
++ .read = feed_simple_input_stream,
++ .data = (void *)&(struct simple_input_stream_data) {
+ .buf = buf,
+ .len = len,
+ },
@@ object-file.c: int force_object_loose(const struct object_id *oid, time_t mtime)
char hdr[MAX_HEADER_LEN];
int hdrlen;
int ret;
-+ struct input_data_from_buffer data;
++ struct simple_input_stream_data data;
+ struct input_stream in_stream = {
-+ .read = read_input_stream_from_buffer,
++ .read = feed_simple_input_stream,
+ .data = &data,
+ };
@@ object-store.h: struct object_directory {
};
+struct input_stream {
-+ const char *(*read)(void* data, unsigned long *len);
++ const void *(*read)(struct input_stream *, unsigned long *len);
+ void *data;
+};
+
2: a309b7e391 < -: ---------- object-file.c: add dry_run mode for write_loose_object()
3: b0a5b53710 ! 2: d4a2caf2bd object-file.c: handle nil oid in write_loose_object()
@@ Metadata
Author: Han Xin <[email protected]>
## Commit message ##
- object-file.c: handle nil oid in write_loose_object()
+ object-file.c: handle undetermined oid in write_loose_object()
- When read input stream, oid can't get before reading all, and it will be
- filled after reading.
+ When streaming a large blob object to "write_loose_object()", we have no
+ chance to run "write_object_file_prepare()" to calculate the oid in
+ advance. So we need to handle undetermined oid in function
+ "write_loose_object()".
+
+ In the original implementation, we know the oid and we can write the
+ temporary file in the same directory as the final object, but for an
+ object with an undetermined oid, we don't know the exact directory for
+ the object, so we have to save the temporary file in ".git/objects/"
+ directory instead.
Helped-by: Jiang Xin <[email protected]>
Signed-off-by: Han Xin <[email protected]>
## object-file.c ##
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *hdr,
- const char *buf;
+ const void *buf;
unsigned long len;
- loose_object_path(the_repository, &filename, oid);
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
+ strbuf_reset(&filename);
+ strbuf_addstr(&filename, the_repository->objects->odb->path);
+ strbuf_addch(&filename, '/');
-+ } else
++ } else {
+ loose_object_path(the_repository, &filename, oid);
++ }
- if (!dry_run) {
- fd = create_tmpfile(&tmp_file, filename.buf);
+ fd = create_tmpfile(&tmp_file, filename.buf);
+ if (fd < 0) {
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *hdr,
die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
ret);
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
die(_("confused by unstable object source data for %s"),
oid_to_hex(oid));
-@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *hdr,
-
close_loose_object(fd);
+ if (is_null_oid(oid)) {
+ int dirlen;
+
-+ /* copy oid */
+ oidcpy((struct object_id *)oid, ¶no_oid);
-+ /* We get the oid now */
+ loose_object_path(the_repository, &filename, oid);
+
++ /* We finally know the object path, and create the missing dir. */
+ dirlen = directory_size(filename.buf);
+ if (dirlen) {
+ struct strbuf dir = STRBUF_INIT;
-+ /*
-+ * Make sure the directory exists; note that the
-+ * contents of the buffer are undefined after mkstemp
-+ * returns an error, so we have to rewrite the whole
-+ * buffer from scratch.
-+ */
-+ strbuf_reset(&dir);
+ strbuf_add(&dir, filename.buf, dirlen - 1);
+ if (mkdir(dir.buf, 0777) && errno != EEXIST)
+ return -1;
++ if (adjust_shared_perm(dir.buf))
++ return -1;
++ strbuf_release(&dir);
+ }
+ }
+
4: 09d438b692 ! 3: 2575900449 object-file.c: read input stream repeatedly in write_loose_object()
@@ Metadata
Author: Han Xin <[email protected]>
## Commit message ##
- object-file.c: read input stream repeatedly in write_loose_object()
+ object-file.c: read stream in a loop in write_loose_object()
- Read input stream repeatedly in write_loose_object() unless reach the
- end, so that we can divide the large blob write into many small blocks.
+ In order to prepare the stream version of "write_loose_object()", read
+ the input stream in a loop in "write_loose_object()", so that we can
+ feed the contents of large blob object to "write_loose_object()" using
+ a small fixed buffer.
+ Helped-by: Jiang Xin <[email protected]>
Signed-off-by: Han Xin <[email protected]>
## object-file.c ##
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *hdr,
static struct strbuf tmp_file = STRBUF_INIT;
static struct strbuf filename = STRBUF_INIT;
- const char *buf;
+ const void *buf;
- unsigned long len;
+ int flush = 0;
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
the_hash_algo->update_fn(&c, hdr, hdrlen);
/* Then the data itself.. */
-- buf = in_stream->read(in_stream->data, &len);
+- buf = in_stream->read(in_stream, &len);
- stream.next_in = (void *)buf;
- stream.avail_in = len;
do {
unsigned char *in0 = stream.next_in;
- ret = git_deflate(&stream, Z_FINISH);
+ if (!stream.avail_in) {
-+ if ((buf = in_stream->read(in_stream->data, &stream.avail_in))) {
++ buf = in_stream->read(in_stream, &stream.avail_in);
++ if (buf) {
+ stream.next_in = (void *)buf;
+ in0 = (unsigned char *)buf;
-+ } else
++ } else {
+ flush = Z_FINISH;
++ }
+ }
+ ret = git_deflate(&stream, flush);
the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
- if (!dry_run && write_buffer(fd, compressed, stream.next_out - compressed) < 0)
+ if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
die(_("unable to write loose object file"));
5: 9fb188d437 < -: ---------- object-store.h: add write_loose_object()
-: ---------- > 4: ca93ecc780 unpack-objects.c: add dry_run mode for get_data()
6: 80468a6fbc ! 5: 39a072ee2a unpack-objects: unpack large object in stream
@@ Metadata
Author: Han Xin <[email protected]>
## Commit message ##
- unpack-objects: unpack large object in stream
+ unpack-objects: unpack_non_delta_entry() read data in a stream
- When calling "unpack_non_delta_entry()", will allocate full memory for
- the whole size of the unpacked object and write the buffer to loose file
- on disk. This may lead to OOM for the git-unpack-objects process when
- unpacking a very large object.
+ We used to call "get_data()" in "unpack_non_delta_entry()" to read the
+ entire contents of a blob object, no matter how big it is. This
+ implementation may consume all the memory and cause OOM.
- In function "unpack_delta_entry()", will also allocate full memory to
- buffer the whole delta, but since there will be no delta for an object
- larger than "core.bigFileThreshold", this issue is moderate.
+ By implementing a zstream version of input_stream interface, we can use
+ a small fixed buffer for "unpack_non_delta_entry()".
- To resolve the OOM issue in "git-unpack-objects", we can unpack large
- object to file in stream, and use "core.bigFileThreshold" to avoid OOM
- limits when called "get_data()".
+ However, unpack non-delta objects from a stream instead of from an entrie
+ buffer will have 10% performance penalty. Therefore, only unpack object
+ larger than the "big_file_threshold" in zstream. See the following
+ benchmarks:
+ $ hyperfine \
+ --prepare 'rm -rf dest.git && git init --bare dest.git' \
+ 'git -C dest.git unpack-objects <binary_320M.pack'
+ Benchmark 1: git -C dest.git unpack-objects <binary_320M.pack
+ Time (mean ± σ): 10.029 s ± 0.270 s [User: 8.265 s, System: 1.522 s]
+ Range (min … max): 9.786 s … 10.603 s 10 runs
+
+ $ hyperfine \
+ --prepare 'rm -rf dest.git && git init --bare dest.git' \
+ 'git -c core.bigFileThreshold=2m -C dest.git unpack-objects <binary_320M.pack'
+ Benchmark 1: git -c core.bigFileThreshold=2m -C dest.git unpack-objects <binary_320M.pack
+ Time (mean ± σ): 10.859 s ± 0.774 s [User: 8.813 s, System: 1.898 s]
+ Range (min … max): 9.884 s … 12.192 s 10 runs
+
+ $ hyperfine \
+ --prepare 'rm -rf dest.git && git init --bare dest.git' \
+ 'git -C dest.git unpack-objects <binary_96M.pack'
+ Benchmark 1: git -C dest.git unpack-objects <binary_96M.pack
+ Time (mean ± σ): 2.678 s ± 0.037 s [User: 2.205 s, System: 0.450 s]
+ Range (min … max): 2.639 s … 2.743 s 10 runs
+
+ $ hyperfine \
+ --prepare 'rm -rf dest.git && git init --bare dest.git' \
+ 'git -c core.bigFileThreshold=2m -C dest.git unpack-objects <binary_96M.pack'
+ Benchmark 1: git -c core.bigFileThreshold=2m -C dest.git unpack-objects <binary_96M.pack
+ Time (mean ± σ): 2.819 s ± 0.124 s [User: 2.216 s, System: 0.564 s]
+ Range (min … max): 2.679 s … 3.125 s 10 runs
+
+ Helped-by: Jiang Xin <[email protected]>
Signed-off-by: Han Xin <[email protected]>
## builtin/unpack-objects.c ##
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
}
}
-+struct input_data_from_zstream {
++struct input_zstream_data {
+ git_zstream *zstream;
+ unsigned char buf[4096];
+ int status;
+};
+
-+static const char *read_inflate_in_stream(void *data, unsigned long *readlen)
++static const void *feed_input_zstream(struct input_stream *in_stream, unsigned long *readlen)
+{
-+ struct input_data_from_zstream *input = data;
-+ git_zstream *zstream = input->zstream;
++ struct input_zstream_data *data = in_stream->data;
++ git_zstream *zstream = data->zstream;
+ void *in = fill(1);
+
-+ if (!len || input->status == Z_STREAM_END) {
++ if (!len || data->status == Z_STREAM_END) {
+ *readlen = 0;
+ return NULL;
+ }
+
-+ zstream->next_out = input->buf;
-+ zstream->avail_out = sizeof(input->buf);
++ zstream->next_out = data->buf;
++ zstream->avail_out = sizeof(data->buf);
+ zstream->next_in = in;
+ zstream->avail_in = len;
+
-+ input->status = git_inflate(zstream, 0);
++ data->status = git_inflate(zstream, 0);
+ use(len - zstream->avail_in);
-+ *readlen = sizeof(input->buf) - zstream->avail_out;
++ *readlen = sizeof(data->buf) - zstream->avail_out;
+
-+ return (const char *)input->buf;
++ return data->buf;
+}
+
+static void write_stream_blob(unsigned nr, unsigned long size)
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ char hdr[32];
+ int hdrlen;
+ git_zstream zstream;
-+ struct input_data_from_zstream data;
++ struct input_zstream_data data;
+ struct input_stream in_stream = {
-+ .read = read_inflate_in_stream,
++ .read = feed_input_zstream,
+ .data = &data,
+ };
+ struct object_id *oid = &obj_list[nr].oid;
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ /* Generate the header */
+ hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX, type_name(OBJ_BLOB), (uintmax_t)size) + 1;
+
-+ if ((ret = write_loose_object(oid, hdr, hdrlen, &in_stream, dry_run, 0, 0)))
++ if ((ret = write_loose_object(oid, hdr, hdrlen, &in_stream, 0, 0)))
+ die(_("failed to write object in stream %d"), ret);
+
+ if (zstream.total_out != size || data.status != Z_STREAM_END)
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
static void unpack_non_delta_entry(enum object_type type, unsigned long size,
unsigned nr)
{
-- void *buf = get_data(size);
+- void *buf = get_data(size, dry_run);
+ void *buf;
+
+ /* Write large blob in stream without allocating full buffer. */
-+ if (type == OBJ_BLOB && size > big_file_threshold) {
++ if (!dry_run && type == OBJ_BLOB && size > big_file_threshold) {
+ write_stream_blob(nr, size);
+ return;
+ }
-+ buf = get_data(size);
++ buf = get_data(size, dry_run);
if (!dry_run && buf)
write_object(nr, type, buf, size);
else
- ## t/t5590-receive-unpack-objects.sh (new) ##
+ ## object-file.c ##
+@@ object-file.c: static const void *feed_simple_input_stream(struct input_stream *in_stream, unsi
+ return data->buf;
+ }
+
+-static int write_loose_object(const struct object_id *oid, char *hdr,
+- int hdrlen, struct input_stream *in_stream,
+- time_t mtime, unsigned flags)
++int write_loose_object(const struct object_id *oid, char *hdr,
++ int hdrlen, struct input_stream *in_stream,
++ time_t mtime, unsigned flags)
+ {
+ int fd, ret;
+ unsigned char compressed[4096];
+
+ ## object-store.h ##
+@@ object-store.h: int hash_object_file(const struct git_hash_algo *algo, const void *buf,
+ unsigned long len, const char *type,
+ struct object_id *oid);
+
++int write_loose_object(const struct object_id *oid, char *hdr,
++ int hdrlen, struct input_stream *in_stream,
++ time_t mtime, unsigned flags);
++
+ int write_object_file_flags(const void *buf, unsigned long len,
+ const char *type, struct object_id *oid,
+ unsigned flags);
+
+ ## t/t5590-unpack-non-delta-objects.sh (new) ##
@@
+#!/bin/sh
+#
@@ t/t5590-receive-unpack-objects.sh (new)
+ cd .git &&
+ find objects/?? -type f | sort
+ ) >expect &&
-+ git repack -ad
++ PACK=$(echo main | git pack-objects --progress --revs test)
+'
+
+test_expect_success 'setup GIT_ALLOC_LIMIT to 1MB' '
@@ t/t5590-receive-unpack-objects.sh (new)
+ git -C dest.git config receive.unpacklimit 100
+'
+
-+test_expect_success 'fail to push: cannot allocate' '
-+ test_must_fail git push dest.git HEAD 2>err &&
-+ test_i18ngrep "remote: fatal: attempting to allocate" err &&
++test_expect_success 'fail to unpack-objects: cannot allocate' '
++ test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err &&
++ test_i18ngrep "fatal: attempting to allocate" err &&
+ (
+ cd dest.git &&
+ find objects/?? -type f | sort
@@ t/t5590-receive-unpack-objects.sh (new)
+'
+
+test_expect_success 'unpack big object in stream' '
-+ git push dest.git HEAD &&
++ git -C dest.git unpack-objects <test-$PACK.pack &&
+ git -C dest.git fsck &&
+ (
+ cd dest.git &&
@@ t/t5590-receive-unpack-objects.sh (new)
+'
+
+test_expect_success 'setup for unpack-objects dry-run test' '
-+ PACK=$(echo main | git pack-objects --progress --revs test) &&
-+ unset GIT_ALLOC_LIMIT &&
+ git init --bare unpack-test.git
+'
+
-+test_expect_success 'unpack-objects dry-run with large threshold' '
-+ (
-+ cd unpack-test.git &&
-+ git config core.bigFileThreshold 2m &&
-+ git unpack-objects -n <../test-$PACK.pack
-+ ) &&
-+ (
-+ cd unpack-test.git &&
-+ find objects/ -type f
-+ ) >actual &&
-+ test_must_be_empty actual
-+'
-+
-+test_expect_success 'unpack-objects dry-run with small threshold' '
++test_expect_success 'unpack-objects dry-run' '
+ (
+ cd unpack-test.git &&
-+ git config core.bigFileThreshold 1m &&
+ git unpack-objects -n <../test-$PACK.pack
+ ) &&
+ (Signature · 1 line
2.34.0.6.g676eedc724
Re: [PATCH v3 0/5] unpack large objects in stream
Quoted 9 lines · Han Xin
From: Han Xin <[email protected]>
Although we do not recommend users push large binary files to the git repositories, it's difficult to prevent them from doing so. Once, we found a problem with a surge in memory usage on the server. The source of the problem is that a user submitted a single object with a size of 15GB. Once someone initiates a git push, the git process will immediately allocate 15G of memory, resulting in an OOM risk.
Quoted 8 lines
Through further analysis, we found that when we execute git unpack-objects, in
unpack_non_delta_entry(), "void *buf =get_data(size);" will directly allocate memory equal to the size of the object. This is quite a scary thing, because the pre-receive hook has not been executed at this time, and we cannot avoid this by hooks.I got inspiration from the deflate process of zlib, maybe it would be a good idea to change unpack-objects to stream deflate.
Hi, Jeff.
I hope you can share with me how Github solves this problem.
As you said in your reply at:
https://lore.kernel.org/git/[email protected]/
"we don't have a match in unpack-objects, but we always run index-pack on incoming packs".
In the original implementation of "index-pack", for objects larger than big_file_threshold, "fixed_buf" with a size of 8192 will be used to complete the calculation of "oid".
I tried the implementation in jk/no-more-unpack-objects, as you noted:
/* XXX This will expand too-large objects! */
if (!data)
data = new_data = get_data_from_pack(obj_entry);
If the conditions of --unpack are given, there will be risks here. When I create an object larger than 1GB and execute index-pack, the result is as follows:
$GIT_ALLOC_LIMIT=1024m git index-pack --unpack --stdin <large.pack
fatal: attempting to allocate 1228800001 over limit 1073741824
Looking forward to your reply.
Re: [PATCH v3 0/5] unpack large objects in stream
Quoted 23 lines · Han Xin
Quoted 9 lines · Han Xin
From: Han Xin <[email protected]>
Although we do not recommend users push large binary files to the git repositories, it's difficult to prevent them from doing so. Once, we found a problem with a surge in memory usage on the server. The source of the problem is that a user submitted a single object with a size of 15GB. Once someone initiates a git push, the git process will immediately allocate 15G of memory, resulting in an OOM risk.
Quoted 8 lines
Through further analysis, we found that when we execute git unpack-objects, in
unpack_non_delta_entry(), "void *buf =get_data(size);" will directly allocate memory equal to the size of the object. This is quite a scary thing, because the pre-receive hook has not been executed at this time, and we cannot avoid this by hooks.I got inspiration from the deflate process of zlib, maybe it would be a good idea to change unpack-objects to stream deflate.
Hi, Jeff.
I hope you can share with me how Github solves this problem.
Quoted 8 lines
As you said in your reply at:
https://lore.kernel.org/git/[email protected]/
"we don't have a match in unpack-objects, but we always run index-pack on incoming packs".In the original implementation of "index-pack", for objects larger than
big_file_threshold, "fixed_buf" with a size of 8192 will be used to complete the calculation of "oid".
We set transfer.unpackLimit to "1", so we never run unpack-objects at all. We always run index-pack, and every push, no matter how small, results in a pack.
We also set GIT_ALLOC_LIMIT to limit any single allocation. We also have custom code in index-pack to detect large objects (where our definition of "large" is 100MB by default):
- for large blobs, we do index it as normal, writing the oid out to a file which is then processed by a pre-receive hook (since people often push up large files accidentally, the hook generates a nice error message, including finding the path at which the blob is referenced)
- for other large objects, we die immediately (with an error message). 100MB commit messages aren't a common user error, and it closes off a whole set of possible integer-overflow parsing attacks (e.g., index-pack in strict-mode will run every tree through fsck_tree(), so there's otherwise nothing stopping you from having a 4GB filename in a tree).
Quoted 9 lines
I tried the implementation in jk/no-more-unpack-objects, as you noted:
/* XXX This will expand too-large objects! */
if (!data)
data =new_data=get_data_from_pack(obj_entry);
If the conditions of --unpack are given, there will be risks here. When I create an object larger than 1GB and execute index-pack, the result is as follows:
$GIT_ALLOC_LIMIT=1024m git index-pack --unpack --stdin <large.pack
fatal: attempting to allocate 1228800001 over limit 1073741824
Yeah, that issue was one of the reasons I never sent the "index-pack --unpack" code to the list. We don't actually use those patches at GitHub. It was something I was working on for upstream but never finished.
-Peff
Re: [PATCH v3 0/5] unpack large objects in stream
Quoted 14 lines · Jeff King
We set transfer.unpackLimit to "1", so we never run unpack-objects at all. We always run index-pack, and every push, no matter how small, results in a pack.
We also set
GIT_ALLOC_LIMITto limit any single allocation. We also have custom code in index-pack to detect large objects (where our definition of "large" is 100MB by default):- for large blobs, we do index it as normal, writing the oid out to a file which is then processed by a pre-receive hook (since people often push up large files accidentally, the hook generates a nice error message, including finding the path at which the blob is referenced)
Quoted 6 lines
- for other large objects, we die immediately (with an error message). 100MB commit messages aren't a common user error, and it closes off a whole set of possible integer-overflow parsing attacks (e.g., index-pack in strict-mode will run every tree through
fsck_tree(), so there's otherwise nothing stopping you from having a 4GB filename in a tree).
Thank you very much for sharing.
The way Github handles it reminds me of what Shawn Pearce introduced in "Scaling up JGit". I guess "mulit-pack-index" and "bitmap" must play an important role in this.
I will seriously consider this solution, thanks a lot.
v4 cover unpack large objects in stream
In reply to Han XinFrom: Han Xin <[email protected]>
Changes since v3:
* Add "size" to "struct input_stream" which used by following commits.
* Increase the buffer size of "struct input_zstream_data" from 4096 to 8192, which is consistent with the "fixed_buf" in the "index-pack.c".
* Refactor "read stream in a loop in write_loose_object()" which introduced a performance problem reported by Derrick Stolee[1].
* Rewrite benchmarks in "unpack-objects: unpack_non_delta_entry() read data in a stream" with sugguestions by Derrick Stolee[1] and Ævar Arnfjörð Bjarmason[2].
Now use "scalar.git" to benchmark, which contains more than 28000 objects and 96 objects larger than 16kB.
1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/
Han Xin (5):
object-file: refactor write_loose_object() to read buffer from stream object-file.c: handle undetermined oid in write_loose_object() object-file.c: read stream in a loop in write_loose_object() unpack-objects.c: add dry_run mode for get_data()
unpack-objects: unpack_non_delta_entry() read data in a stream
builtin/unpack-objects.c | 93 +++++++++++++++++++++++-- object-file.c | 102 ++++++++++++++++++++++++---- object-store.h | 10 +++ t/t5590-unpack-non-delta-objects.sh | 76 +++++++++++++++++++++ 4 files changed, 262 insertions(+), 19 deletions(-) create mode 100755 t/t5590-unpack-non-delta-objects.sh
Range-diff against v3:
1: 8640b04f6d ! 1: af707ef304 object-file: refactor write_loose_object() to read buffer from stream
@@ object-file.c: int write_object_file_flags(const void *buf, unsigned long len,
+ .buf = buf,
+ .len = len,
+ },
++ .size = len,
+ };
/* Normally if we have it in the pack then we do not bother writing
@@ object-file.c: int hash_object_file_literally(const void *buf, unsigned long len
+ .buf = buf,
+ .len = len,
+ },
++ .size = len,
+ };
/* type string, SP, %lu of the length plus NUL must fit this */
@@ object-file.c: int force_object_loose(const struct object_id *oid, time_t mtime)
if (has_loose_object(oid))
return 0;
buf = read_object(the_repository, oid, &type, &len);
++ in_stream.size = len;
if (!buf)
return error(_("cannot read object for %s"), oid_to_hex(oid));
+ data.buf = buf;
@@ object-store.h: struct object_directory {
+struct input_stream {
+ const void *(*read)(struct input_stream *, unsigned long *len);
+ void *data;
++ size_t size;
+};
+
KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
2: d4a2caf2bd = 2: 321ad90d8e object-file.c: handle undetermined oid in write_loose_object()
3: 2575900449 ! 3: 1992ac39af object-file.c: read stream in a loop in write_loose_object()
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
- ret = git_deflate(&stream, Z_FINISH);
+ if (!stream.avail_in) {
+ buf = in_stream->read(in_stream, &stream.avail_in);
-+ if (buf) {
-+ stream.next_in = (void *)buf;
-+ in0 = (unsigned char *)buf;
-+ } else {
++ stream.next_in = (void *)buf;
++ in0 = (unsigned char *)buf;
++ /* All data has been read. */
++ if (in_stream->size + hdrlen == stream.total_in + stream.avail_in)
+ flush = Z_FINISH;
-+ }
+ }
+ ret = git_deflate(&stream, flush);
the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
die(_("unable to write loose object file"));
+ stream.next_out = compressed;
+ stream.avail_out = sizeof(compressed);
+- } while (ret == Z_OK);
++ } while (ret == Z_OK || ret == Z_BUF_ERROR);
+
+ if (ret != Z_STREAM_END)
+ die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
4: ca93ecc780 = 4: c41eb06533 unpack-objects.c: add dry_run mode for get_data()
5: 39a072ee2a ! 5: 9427775bdc unpack-objects: unpack_non_delta_entry() read data in a stream @@ Commit message
larger than the "big_file_threshold" in zstream. See the following benchmarks:
- $ hyperfine \
- --prepare 'rm -rf dest.git && git init --bare dest.git' \
- 'git -C dest.git unpack-objects <binary_320M.pack'
- Benchmark 1: git -C dest.git unpack-objects <binary_320M.pack
- Time (mean ± σ): 10.029 s ± 0.270 s [User: 8.265 s, System: 1.522 s]
- Range (min … max): 9.786 s … 10.603 s 10 runs
+ hyperfine \
+ --setup \
+ 'if ! test -d scalar.git; then git clone --bare https://github.com/microsoft/scalar.git; cp scalar.git/objects/pack/*.pack small.pack; fi' \
+ --prepare 'rm -rf dest.git && git init --bare dest.git' \
+ -n 'old' 'git -C dest.git unpack-objects <small.pack' \
+ -n 'new' 'new/git -C dest.git unpack-objects <small.pack' \
+ -n 'new (small threshold)' \
+ 'new/git -c core.bigfilethreshold=16k -C dest.git unpack-objects <small.pack'
+ Benchmark 1: old
+ Time (mean ± σ): 6.075 s ± 0.069 s [User: 5.047 s, System: 0.991 s]
+ Range (min … max): 6.018 s … 6.189 s 10 runs
- $ hyperfine \
- --prepare 'rm -rf dest.git && git init --bare dest.git' \
- 'git -c core.bigFileThreshold=2m -C dest.git unpack-objects <binary_320M.pack'
- Benchmark 1: git -c core.bigFileThreshold=2m -C dest.git unpack-objects <binary_320M.pack
- Time (mean ± σ): 10.859 s ± 0.774 s [User: 8.813 s, System: 1.898 s]
- Range (min … max): 9.884 s … 12.192 s 10 runs
+ Benchmark 2: new
+ Time (mean ± σ): 6.090 s ± 0.033 s [User: 5.075 s, System: 0.976 s]
+ Range (min … max): 6.030 s … 6.142 s 10 runs
- $ hyperfine \
- --prepare 'rm -rf dest.git && git init --bare dest.git' \
- 'git -C dest.git unpack-objects <binary_96M.pack'
- Benchmark 1: git -C dest.git unpack-objects <binary_96M.pack
- Time (mean ± σ): 2.678 s ± 0.037 s [User: 2.205 s, System: 0.450 s]
- Range (min … max): 2.639 s … 2.743 s 10 runs
+ Benchmark 3: new (small threshold)
+ Time (mean ± σ): 6.755 s ± 0.029 s [User: 5.150 s, System: 1.560 s]
+ Range (min … max): 6.711 s … 6.809 s 10 runs
- $ hyperfine \
- --prepare 'rm -rf dest.git && git init --bare dest.git' \
- 'git -c core.bigFileThreshold=2m -C dest.git unpack-objects <binary_96M.pack'
- Benchmark 1: git -c core.bigFileThreshold=2m -C dest.git unpack-objects <binary_96M.pack
- Time (mean ± σ): 2.819 s ± 0.124 s [User: 2.216 s, System: 0.564 s]
- Range (min … max): 2.679 s … 3.125 s 10 runs
+ Summary
+ 'old' ran
+ 1.00 ± 0.01 times faster than 'new'
+ 1.11 ± 0.01 times faster than 'new (small threshold)'
+ Helped-by: Derrick Stolee <[email protected]>
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+struct input_zstream_data {
+ git_zstream *zstream;
-+ unsigned char buf[4096];
++ unsigned char buf[8192];
+ int status;
+};
+
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ struct input_stream in_stream = {
+ .read = feed_input_zstream,
+ .data = &data,
++ .size = size,
+ };
+ struct object_id *oid = &obj_list[nr].oid;
+ int ret;
Signature · 1 line
2.34.0
Re: [PATCH v4 0/5] unpack large objects in stream
Quoted 10 lines · Han Xin
From: Han Xin <[email protected]>
Changes since v3:
* Add "size" to "structinput_stream" which used by following commits.* Increase the buffer size of "struct
input_zstream_data" from 4096 to 8192, which is consistent with the "fixed_buf" in the "index-pack.c".* Refactor "read stream in a loop in
write_loose_object()" which introduced a performance problem reported by Derrick Stolee[1].
Thank you for finding the issue. It seems simple enough to add that size information and regain the performance back to nearly no overhead. Your hyperfine statistics are within noise, which is great. Thanks!
-Stolee
v5 cover unpack large blobs in stream
In reply to Han XinFrom: Han Xin <[email protected]>
Changes since v4:
* Refactor to "struct input_stream" implementations so that we can reduce the changes to "write_loose_object()" sugguest by Ævar Arnfjörð Bjarmason.
* Add a new flag called "HASH_STREAM" to support this feature.
* Add a new config "core.bigFileStreamingThreshold" instread of "core.bigFileThreshold" sugguest by Ævar Arnfjörð Bjarmason[1].
* Roll destination repository preparement into a function in "t5590-unpack-non-delta-objects.sh", so that we can run testcases with --run=setup,3,4.
1. https://lore.kernel.org/git/[email protected]/
Han Xin (6):
object-file: refactor write_loose_object() to support read from stream object-file.c: handle undetermined oid in write_loose_object() object-file.c: read stream in a loop in write_loose_object() unpack-objects.c: add dry_run mode for get_data() object-file.c: make "write_object_file_flags()" to support "HASH_STREAM"
unpack-objects: unpack_non_delta_entry() read data in a stream
Documentation/config/core.txt | 11 ++++ builtin/unpack-objects.c | 86 +++++++++++++++++++++++++++-- cache.h | 2 + config.c | 5 ++ environment.c | 1 + object-file.c | 73 +++++++++++++++++++----- object-store.h | 5 ++ t/t5590-unpack-non-delta-objects.sh | 70 +++++++++++++++++++++++ 8 files changed, 234 insertions(+), 19 deletions(-) create mode 100755 t/t5590-unpack-non-delta-objects.sh
Range-diff against v4:
1: af707ef304 < -: ---------- object-file: refactor write_loose_object() to read buffer from stream
2: 321ad90d8e < -: ---------- object-file.c: handle undetermined oid in write_loose_object()
3: 1992ac39af < -: ---------- object-file.c: read stream in a loop in write_loose_object()
-: ---------- > 1: f3595e68cc object-file: refactor write_loose_object() to support read from stream
-: ---------- > 2: c25fdd1fe5 object-file.c: handle undetermined oid in write_loose_object()
-: ---------- > 3: ed226f2f9f object-file.c: read stream in a loop in write_loose_object()
4: c41eb06533 ! 4: 2f91e540f6 unpack-objects.c: add dry_run mode for get_data()
@@ builtin/unpack-objects.c: static void use(int bytes)
{
git_zstream stream;
- void *buf = xmallocz(size);
-+ unsigned long bufsize = dry_run ? 4096 : size;
++ unsigned long bufsize = dry_run ? 8192 : size;
+ void *buf = xmallocz(bufsize);
memset(&stream, 0, sizeof(stream));
-: ---------- > 5: 7698938eac object-file.c: make "write_object_file_flags()" to support "HASH_STREAM"
5: 9427775bdc ! 6: 103bb1db06 unpack-objects: unpack_non_delta_entry() read data in a stream
@@ Commit message
However, unpack non-delta objects from a stream instead of from an entrie
buffer will have 10% performance penalty. Therefore, only unpack object
- larger than the "big_file_threshold" in zstream. See the following
+ larger than the "core.BigFileStreamingThreshold" in zstream. See the following
benchmarks:
hyperfine \
--setup \
'if ! test -d scalar.git; then git clone --bare https://github.com/microsoft/scalar.git; cp scalar.git/objects/pack/*.pack small.pack; fi' \
- --prepare 'rm -rf dest.git && git init --bare dest.git' \
- -n 'old' 'git -C dest.git unpack-objects <small.pack' \
- -n 'new' 'new/git -C dest.git unpack-objects <small.pack' \
- -n 'new (small threshold)' \
- 'new/git -c core.bigfilethreshold=16k -C dest.git unpack-objects <small.pack'
- Benchmark 1: old
- Time (mean ± σ): 6.075 s ± 0.069 s [User: 5.047 s, System: 0.991 s]
- Range (min … max): 6.018 s … 6.189 s 10 runs
-
- Benchmark 2: new
- Time (mean ± σ): 6.090 s ± 0.033 s [User: 5.075 s, System: 0.976 s]
- Range (min … max): 6.030 s … 6.142 s 10 runs
-
- Benchmark 3: new (small threshold)
- Time (mean ± σ): 6.755 s ± 0.029 s [User: 5.150 s, System: 1.560 s]
- Range (min … max): 6.711 s … 6.809 s 10 runs
+ --prepare 'rm -rf dest.git && git init --bare dest.git'
Summary
- 'old' ran
- 1.00 ± 0.01 times faster than 'new'
- 1.11 ± 0.01 times faster than 'new (small threshold)'
+ './git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'origin/master'
+ 1.01 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'HEAD~1'
+ 1.01 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'HEAD~0'
+ 1.03 ± 0.10 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'origin/master'
+ 1.02 ± 0.07 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'HEAD~0'
+ 1.10 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'HEAD~1'
+ Helped-by: Ævar Arnfjörð Bjarmason <[email protected]>
Helped-by: Derrick Stolee <[email protected]>
Helped-by: Jiang Xin <[email protected]>
Signed-off-by: Han Xin <[email protected]>
+ ## Documentation/config/core.txt ##
+@@ Documentation/config/core.txt: be delta compressed, but larger binary media files won't be.
+ +
+ Common unit suffixes of 'k', 'm', or 'g' are supported.
+
++core.bigFileStreamingThreshold::
++ Files larger than this will be streamed out to a temporary
++ object file while being hashed, which will when be renamed
++ in-place to a loose object, particularly if the
++ `core.bigFileThreshold' setting dictates that they're always
++ written out as loose objects.
+++
++Default is 128 MiB on all platforms.
+++
++Common unit suffixes of 'k', 'm', or 'g' are supported.
++
+ core.excludesFile::
+ Specifies the pathname to the file that contains patterns to
+ describe paths that are not meant to be tracked, in addition
+
## builtin/unpack-objects.c ##
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type type,
}
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+
+static void write_stream_blob(unsigned nr, unsigned long size)
+{
-+ char hdr[32];
-+ int hdrlen;
+ git_zstream zstream;
+ struct input_zstream_data data;
+ struct input_stream in_stream = {
+ .read = feed_input_zstream,
+ .data = &data,
-+ .size = size,
+ };
-+ struct object_id *oid = &obj_list[nr].oid;
+ int ret;
+
+ memset(&zstream, 0, sizeof(zstream));
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ data.zstream = &zstream;
+ git_inflate_init(&zstream);
+
-+ /* Generate the header */
-+ hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX, type_name(OBJ_BLOB), (uintmax_t)size) + 1;
-+
-+ if ((ret = write_loose_object(oid, hdr, hdrlen, &in_stream, 0, 0)))
++ if ((ret = write_object_file_flags(&in_stream, size, type_name(OBJ_BLOB) ,&obj_list[nr].oid, HASH_STREAM)))
+ die(_("failed to write object in stream %d"), ret);
+
+ if (zstream.total_out != size || data.status != Z_STREAM_END)
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ git_inflate_end(&zstream);
+
+ if (strict && !dry_run) {
-+ struct blob *blob = lookup_blob(the_repository, oid);
++ struct blob *blob = lookup_blob(the_repository, &obj_list[nr].oid);
+ if (blob)
+ blob->object.flags |= FLAG_WRITTEN;
+ else
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ void *buf;
+
+ /* Write large blob in stream without allocating full buffer. */
-+ if (!dry_run && type == OBJ_BLOB && size > big_file_threshold) {
++ if (!dry_run && type == OBJ_BLOB && size > big_file_streaming_threshold) {
+ write_stream_blob(nr, size);
+ return;
+ }
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
write_object(nr, type, buf, size);
else
- ## object-file.c ##
-@@ object-file.c: static const void *feed_simple_input_stream(struct input_stream *in_stream, unsi
- return data->buf;
- }
+ ## cache.h ##
+@@ cache.h: extern size_t packed_git_window_size;
+ extern size_t packed_git_limit;
+ extern size_t delta_base_cache_limit;
+ extern unsigned long big_file_threshold;
++extern unsigned long big_file_streaming_threshold;
+ extern unsigned long pack_size_limit_cfg;
--static int write_loose_object(const struct object_id *oid, char *hdr,
-- int hdrlen, struct input_stream *in_stream,
-- time_t mtime, unsigned flags)
-+int write_loose_object(const struct object_id *oid, char *hdr,
-+ int hdrlen, struct input_stream *in_stream,
-+ time_t mtime, unsigned flags)
- {
- int fd, ret;
- unsigned char compressed[4096];
+ /*
- ## object-store.h ##
-@@ object-store.h: int hash_object_file(const struct git_hash_algo *algo, const void *buf,
- unsigned long len, const char *type,
- struct object_id *oid);
+ ## config.c ##
+@@ config.c: static int git_default_core_config(const char *var, const char *value, void *cb)
+ return 0;
+ }
-+int write_loose_object(const struct object_id *oid, char *hdr,
-+ int hdrlen, struct input_stream *in_stream,
-+ time_t mtime, unsigned flags);
++ if (!strcmp(var, "core.bigfilestreamingthreshold")) {
++ big_file_streaming_threshold = git_config_ulong(var, value);
++ return 0;
++ }
+
- int write_object_file_flags(const void *buf, unsigned long len,
- const char *type, struct object_id *oid,
- unsigned flags);
+ if (!strcmp(var, "core.packedgitlimit")) {
+ packed_git_limit = git_config_ulong(var, value);
+ return 0;
+
+ ## environment.c ##
+@@ environment.c: size_t packed_git_window_size = DEFAULT_PACKED_GIT_WINDOW_SIZE;
+ size_t packed_git_limit = DEFAULT_PACKED_GIT_LIMIT;
+ size_t delta_base_cache_limit = 96 * 1024 * 1024;
+ unsigned long big_file_threshold = 512 * 1024 * 1024;
++unsigned long big_file_streaming_threshold = 128 * 1024 * 1024;
+ int pager_use_color = 1;
+ const char *editor_program;
+ const char *askpass_program;
## t/t5590-unpack-non-delta-objects.sh (new) ##
@@
@@ t/t5590-unpack-non-delta-objects.sh (new)
+
+. ./test-lib.sh
+
-+test_expect_success "create commit with big blobs (1.5 MB)" '
++prepare_dest () {
++ test_when_finished "rm -rf dest.git" &&
++ git init --bare dest.git &&
++ git -C dest.git config core.bigFileStreamingThreshold $1
++ git -C dest.git config core.bigFileThreshold $1
++}
++
++test_expect_success "setup repo with big blobs (1.5 MB)" '
+ test-tool genrandom foo 1500000 >big-blob &&
+ test_commit --append foo big-blob &&
+ test-tool genrandom bar 1500000 >big-blob &&
@@ t/t5590-unpack-non-delta-objects.sh (new)
+ cd .git &&
+ find objects/?? -type f | sort
+ ) >expect &&
-+ PACK=$(echo main | git pack-objects --progress --revs test)
++ PACK=$(echo main | git pack-objects --revs test)
+'
+
-+test_expect_success 'setup GIT_ALLOC_LIMIT to 1MB' '
++test_expect_success 'setup env: GIT_ALLOC_LIMIT to 1MB' '
+ GIT_ALLOC_LIMIT=1m &&
+ export GIT_ALLOC_LIMIT
+'
+
-+test_expect_success 'prepare dest repository' '
-+ git init --bare dest.git &&
-+ git -C dest.git config core.bigFileThreshold 2m &&
-+ git -C dest.git config receive.unpacklimit 100
-+'
-+
+test_expect_success 'fail to unpack-objects: cannot allocate' '
++ prepare_dest 2m &&
+ test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err &&
-+ test_i18ngrep "fatal: attempting to allocate" err &&
++ grep "fatal: attempting to allocate" err &&
+ (
+ cd dest.git &&
+ find objects/?? -type f | sort
+ ) >actual &&
++ test_file_not_empty actual &&
+ ! test_cmp expect actual
+'
+
-+test_expect_success 'set a lower bigfile threshold' '
-+ git -C dest.git config core.bigFileThreshold 1m
-+'
-+
+test_expect_success 'unpack big object in stream' '
++ prepare_dest 1m &&
+ git -C dest.git unpack-objects <test-$PACK.pack &&
+ git -C dest.git fsck &&
+ (
@@ t/t5590-unpack-non-delta-objects.sh (new)
+ test_cmp expect actual
+'
+
-+test_expect_success 'setup for unpack-objects dry-run test' '
-+ git init --bare unpack-test.git
-+'
-+
+test_expect_success 'unpack-objects dry-run' '
++ prepare_dest 1m &&
++ git -C dest.git unpack-objects -n <test-$PACK.pack &&
+ (
-+ cd unpack-test.git &&
-+ git unpack-objects -n <../test-$PACK.pack
-+ ) &&
-+ (
-+ cd unpack-test.git &&
++ cd dest.git &&
+ find objects/ -type f
+ ) >actual &&
+ test_must_be_empty actualSignature · 1 line
2.34.0
v5 cover unpack large blobs in stream
From: Han Xin <[email protected]>
Changes since v5:
* Refactor write_loose_object() to reuse in stream version sugguest by Ævar Arnfjörð Bjarmason [1].
* Add a new testcase into t5590-unpack-non-delta-objects to cover the case of unpacking existing objects.
* Fix code formatting in unpack-objects.c sugguest by Ævar Arnfjörð Bjarmason [2].
1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/
Han Xin (6):
object-file.c: release strbuf in write_loose_object() object-file.c: refactor object header generation into a function object-file.c: refactor write_loose_object() to reuse in stream
version
object-file.c: make "write_object_file_flags()" to support read in
stream
unpack-objects.c: add dry_run mode for get_data()
unpack-objects: unpack_non_delta_entry() read data in a stream
Documentation/config/core.txt | 11 ++ builtin/unpack-objects.c | 94 ++++++++++++- cache.h | 2 + config.c | 5 + environment.c | 1 + object-file.c | 207 +++++++++++++++++++++++----- object-store.h | 5 + t/t5590-unpack-non-delta-objects.sh | 87 ++++++++++++ 8 files changed, 370 insertions(+), 42 deletions(-) create mode 100755 t/t5590-unpack-non-delta-objects.sh
Range-diff against v5:
1: f3595e68cc < -: ---------- object-file: refactor write_loose_object() to support read from stream
2: c25fdd1fe5 < -: ---------- object-file.c: handle undetermined oid in write_loose_object()
3: ed226f2f9f < -: ---------- object-file.c: read stream in a loop in write_loose_object()
-: ---------- > 1: 59d35dac5f object-file.c: release strbuf in write_loose_object()
-: ---------- > 2: 2174a6cbad object-file.c: refactor object header generation into a function
-: ---------- > 3: 8a704ecc59 object-file.c: refactor write_loose_object() to reuse in stream version
-: ---------- > 4: 96f05632a2 object-file.c: make "write_object_file_flags()" to support read in stream
4: 2f91e540f6 ! 5: 1acbb6e849 unpack-objects.c: add dry_run mode for get_data()
@@ builtin/unpack-objects.c: static void use(int bytes)
{
git_zstream stream;
- void *buf = xmallocz(size);
-+ unsigned long bufsize = dry_run ? 8192 : size;
-+ void *buf = xmallocz(bufsize);
++ unsigned long bufsize;
++ void *buf;
memset(&stream, 0, sizeof(stream));
++ if (dry_run && size > 8192)
++ bufsize = 8192;
++ else
++ bufsize = size;
++ buf = xmallocz(bufsize);
stream.next_out = buf;
- stream.avail_out = size;
5: 7698938eac < -: ---------- object-file.c: make "write_object_file_flags()" to support "HASH_STREAM"
6: 92d69cb84a ! 6: 476aaba527 unpack-objects: unpack_non_delta_entry() read data in a stream
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ int status;
+};
+
-+static const void *feed_input_zstream(struct input_stream *in_stream, unsigned long *readlen)
++static const void *feed_input_zstream(const struct input_stream *in_stream,
++ unsigned long *readlen)
+{
+ struct input_zstream_data *data = in_stream->data;
+ git_zstream *zstream = data->zstream;
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ .read = feed_input_zstream,
+ .data = &data,
+ };
-+ int ret;
+
+ memset(&zstream, 0, sizeof(zstream));
+ memset(&data, 0, sizeof(data));
+ data.zstream = &zstream;
+ git_inflate_init(&zstream);
+
-+ if ((ret = write_object_file_flags(&in_stream, size, type_name(OBJ_BLOB) ,&obj_list[nr].oid, HASH_STREAM)))
-+ die(_("failed to write object in stream %d"), ret);
++ if (write_object_file_flags(&in_stream, size,
++ type_name(OBJ_BLOB),
++ &obj_list[nr].oid,
++ HASH_STREAM))
++ die(_("failed to write object in stream"));
+
+ if (zstream.total_out != size || data.status != Z_STREAM_END)
+ die(_("inflate returned %d"), data.status);
+ git_inflate_end(&zstream);
+
-+ if (strict && !dry_run) {
++ if (strict) {
+ struct blob *blob = lookup_blob(the_repository, &obj_list[nr].oid);
+ if (blob)
+ blob->object.flags |= FLAG_WRITTEN;
+ else
-+ die("invalid blob object from stream");
++ die(_("invalid blob object from stream"));
+ }
+ obj_list[nr].obj = NULL;
+}
@@ t/t5590-unpack-non-delta-objects.sh (new)
+prepare_dest () {
+ test_when_finished "rm -rf dest.git" &&
+ git init --bare dest.git &&
-+ git -C dest.git config core.bigFileStreamingThreshold $1
++ git -C dest.git config core.bigFileStreamingThreshold $1 &&
+ git -C dest.git config core.bigFileThreshold $1
+}
+
@@ t/t5590-unpack-non-delta-objects.sh (new)
+ test_cmp expect actual
+'
+
++test_expect_success 'unpack big object in stream with existing oids' '
++ prepare_dest 1m &&
++ git -C dest.git index-pack --stdin <test-$PACK.pack &&
++ (
++ cd dest.git &&
++ find objects/?? -type f | sort
++ ) >actual &&
++ test_must_be_empty actual &&
++ git -C dest.git unpack-objects <test-$PACK.pack &&
++ git -C dest.git fsck &&
++ (
++ cd dest.git &&
++ find objects/?? -type f | sort
++ ) >actual &&
++ test_must_be_empty actual
++'
++
+test_expect_success 'unpack-objects dry-run' '
+ prepare_dest 1m &&
+ git -C dest.git unpack-objects -n <test-$PACK.pack &&Signature · 1 line
2.34.1.52.gfcc2252aea.agit.6.5.6
v7 cover unpack large blobs in stream
From: Han Xin <[email protected]>
Changes since v6:
* Remove "object-file.c: release strbuf in write_loose_object()" which is not needed anymore. Thanks to René Scharfe[1] for reporting this.
* Reorder the patch series and put "unpack-objects.c: add dry_run mode for get_data()" and its testcases to the front.
* Replace "refactor object header generation into a function" with "object-file API: add a format_object_header() function" sugguested by Ævar Arnfjörð Bjarmason[2].
* Export "write_stream_object_file()" instead of "reusing write_object_file_flags()" sugguested by René Scharfe[3]. The new flag "HASH_STREAM" has been removed.
* Fix the directory creation error and the "strbuf dir" leak in "write_stream_object_file()".
* Change "unsigned long size" to "size_t size" in "write_stream_blob()" and "get_data()" in "unpack-objects.c".
1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/
3. https://lore.kernel.org/git/[email protected]/
Han Xin (4):
unpack-objects.c: add dry_run mode for get_data() object-file.c: refactor write_loose_object() to reuse in stream
version
object-file.c: add "write_stream_object_file()" to support read in
stream
unpack-objects: unpack_non_delta_entry() read data in a stream
Ævar Arnfjörð Bjarmason (1):
object-file API: add a format_object_header() function
Documentation/config/core.txt | 11 ++ builtin/index-pack.c | 3 +- builtin/unpack-objects.c | 94 ++++++++++++- bulk-checkin.c | 4 +- cache.h | 22 +++ config.c | 5 + environment.c | 1 + http-push.c | 2 +- object-file.c | 199 ++++++++++++++++++++++------ object-store.h | 9 ++ t/t5590-unpack-non-delta-objects.sh | 91 +++++++++++++ 11 files changed, 392 insertions(+), 49 deletions(-) create mode 100755 t/t5590-unpack-non-delta-objects.sh
Range-diff against v6:
1: 59d35dac5f < -: ---------- object-file.c: release strbuf in write_loose_object()
2: 2174a6cbad < -: ---------- object-file.c: refactor object header generation into a function
5: 1acbb6e849 ! 1: a8f232f553 unpack-objects.c: add dry_run mode for get_data()
@@ builtin/unpack-objects.c: static void use(int bytes)
}
-static void *get_data(unsigned long size)
-+static void *get_data(unsigned long size, int dry_run)
++static void *get_data(size_t size, int dry_run)
{
git_zstream stream;
- void *buf = xmallocz(size);
-+ unsigned long bufsize;
++ size_t bufsize;
+ void *buf;
memset(&stream, 0, sizeof(stream));
@@ builtin/unpack-objects.c: static void unpack_delta_entry(enum object_type type,
if (dry_run || !delta_data) {
free(delta_data);
return;
+
+ ## t/t5590-unpack-non-delta-objects.sh (new) ##
+@@
++#!/bin/sh
++#
++# Copyright (c) 2021 Han Xin
++#
++
++test_description='Test unpack-objects with non-delta objects'
++
++GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
++export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
++
++. ./test-lib.sh
++
++prepare_dest () {
++ test_when_finished "rm -rf dest.git" &&
++ git init --bare dest.git
++}
++
++test_expect_success "setup repo with big blobs (1.5 MB)" '
++ test-tool genrandom foo 1500000 >big-blob &&
++ test_commit --append foo big-blob &&
++ test-tool genrandom bar 1500000 >big-blob &&
++ test_commit --append bar big-blob &&
++ (
++ cd .git &&
++ find objects/?? -type f | sort
++ ) >expect &&
++ PACK=$(echo main | git pack-objects --revs test)
++'
++
++test_expect_success 'setup env: GIT_ALLOC_LIMIT to 1MB' '
++ GIT_ALLOC_LIMIT=1m &&
++ export GIT_ALLOC_LIMIT
++'
++
++test_expect_success 'fail to unpack-objects: cannot allocate' '
++ prepare_dest &&
++ test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err &&
++ grep "fatal: attempting to allocate" err &&
++ (
++ cd dest.git &&
++ find objects/?? -type f | sort
++ ) >actual &&
++ test_file_not_empty actual &&
++ ! test_cmp expect actual
++'
++
++test_expect_success 'unpack-objects dry-run' '
++ prepare_dest &&
++ git -C dest.git unpack-objects -n <test-$PACK.pack &&
++ (
++ cd dest.git &&
++ find objects/ -type f
++ ) >actual &&
++ test_must_be_empty actual
++'
++
++test_done
-: ---------- > 2: 0d2e0f3a00 object-file API: add a format_object_header() function
3: 8a704ecc59 ! 3: a571b8f16c object-file.c: refactor write_loose_object() to reuse in stream version
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
loose_object_path(the_repository, &filename, oid);
- fd = create_tmpfile(&tmp_file, filename.buf);
-+ fd = create_tmpfile(&tmp_file, filename.buf, flags);
- if (fd < 0) {
+- if (fd < 0) {
- if (flags & HASH_SILENT)
-- ret = -1;
+- return -1;
- else if (errno == EACCES)
-- ret = error(_("insufficient permission for adding an "
-- "object to repository database %s"),
-- get_object_directory());
+- return error(_("insufficient permission for adding an object to repository database %s"), get_object_directory());
- else
-- ret = error_errno(_("unable to create temporary file"));
-+ ret = -1;
- goto cleanup;
- }
-
+- return error_errno(_("unable to create temporary file"));
+- }
+-
- /* Set it up */
- git_deflate_init(&stream, zlib_compression_level);
- stream.next_out = compressed;
- stream.avail_out = sizeof(compressed);
- the_hash_algo->init_fn(&c);
--
++ fd = create_tmpfile(&tmp_file, filename.buf, flags);
++ if (fd < 0)
++ return -1;
+
- /* First header.. */
- stream.next_in = (unsigned char *)hdr;
- stream.avail_in = hdrlen;
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
- warning_errno(_("failed utime() on %s"), tmp_file.buf);
- }
-
-- ret = finalize_object_file(tmp_file.buf, filename.buf);
-+ ret = finalize_object_file_with_mtime(tmp_file.buf, filename.buf, mtime, flags);
- cleanup:
- strbuf_release(&filename);
- strbuf_release(&tmp_file);
+- return finalize_object_file(tmp_file.buf, filename.buf);
++ return finalize_object_file_with_mtime(tmp_file.buf, filename.buf,
++ mtime, flags);
+ }
+
+ static int freshen_loose_object(const struct object_id *oid)
4: 96f05632a2 ! 4: 1de06a8f5c object-file.c: make "write_object_file_flags()" to support read in stream
@@ Metadata
Author: Han Xin <[email protected]>
## Commit message ##
- object-file.c: make "write_object_file_flags()" to support read in stream
+ object-file.c: add "write_stream_object_file()" to support read in stream
We used to call "get_data()" in "unpack_non_delta_entry()" to read the
entire contents of a blob object, no matter how big it is. This
implementation may consume all the memory and cause OOM.
- This can be improved by feeding data to "stream_loose_object()" in a
- stream. The input stream is implemented as an interface.
-
- When streaming a large blob object to "write_loose_object()", we have no
- chance to run "write_object_file_prepare()" to calculate the oid in
- advance. So we need to handle undetermined oid in a new function called
- "stream_loose_object()".
+ This can be improved by feeding data to "write_stream_object_file()"
+ in a stream. The input stream is implemented as an interface.
+ The difference with "write_loose_object()" is that we have no chance
+ to run "write_object_file_prepare()" to calculate the oid in advance.
In "write_loose_object()", we know the oid and we can write the
temporary file in the same directory as the final object, but for an
object with an undetermined oid, we don't know the exact directory for
the object, so we have to save the temporary file in ".git/objects/"
directory instead.
- We will reuse "write_object_file_flags()" in "unpack_non_delta_entry()" to
- read the entire data contents in stream, so a new flag "HASH_STREAM" is
- added. When read in stream, we needn't prepare the "oid" before
- "write_loose_object()", only generate the header.
"freshen_packed_object()" or "freshen_loose_object()" will be called
- inside "stream_loose_object()" after obtaining the "oid".
+ inside "write_stream_object_file()" after obtaining the "oid".
+ Helped-by: René Scharfe <[email protected]>
Helped-by: Ævar Arnfjörð Bjarmason <[email protected]>
Helped-by: Jiang Xin <[email protected]>
Signed-off-by: Han Xin <[email protected]>
- ## cache.h ##
-@@ cache.h: int ie_modified(struct index_state *, const struct cache_entry *, struct stat *,
- #define HASH_FORMAT_CHECK 2
- #define HASH_RENORMALIZE 4
- #define HASH_SILENT 8
-+#define HASH_STREAM 16
- int index_fd(struct index_state *istate, struct object_id *oid, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags);
- int index_path(struct index_state *istate, struct object_id *oid, const char *path, struct stat *st, unsigned flags);
-
-
## object-file.c ##
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
return 1;
}
-+static int stream_loose_object(struct object_id *oid, char *hdr, int hdrlen,
-+ const struct input_stream *in_stream,
-+ unsigned long len, time_t mtime, unsigned flags)
++int write_stream_object_file(struct input_stream *in_stream, size_t len,
++ enum object_type type, time_t mtime,
++ unsigned flags, struct object_id *oid)
+{
-+ int fd, ret, err = 0, flush = 0;
++ int fd, ret, flush = 0;
+ unsigned char compressed[4096];
+ git_zstream stream;
+ git_hash_ctx c;
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
+ static struct strbuf tmp_file = STRBUF_INIT;
+ static struct strbuf filename = STRBUF_INIT;
+ int dirlen;
++ char hdr[MAX_HEADER_LEN];
++ int hdrlen = sizeof(hdr);
+
++ /* Since "filename" is defined as static, it will be reused. So reset it
++ * first before using it. */
++ strbuf_reset(&filename);
+ /* When oid is not determined, save tmp file to odb path. */
+ strbuf_addf(&filename, "%s/", get_object_directory());
+
+ fd = create_tmpfile(&tmp_file, filename.buf, flags);
-+ if (fd < 0) {
-+ err = -1;
-+ goto cleanup;
-+ }
++ if (fd < 0)
++ return -1;
++
++ hdrlen = format_object_header(hdr, hdrlen, type, len);
+
+ /* Set it up and write header */
+ setup_stream_and_header(&stream, compressed, sizeof(compressed),
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
+
+ if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
+ unlink_or_warn(tmp_file.buf);
-+ goto cleanup;
++ return 0;
+ }
+
+ loose_object_path(the_repository, &filename, oid);
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
+ struct strbuf dir = STRBUF_INIT;
+ strbuf_add(&dir, filename.buf, dirlen - 1);
+
-+ if (mkdir_in_gitdir(dir.buf) < 0) {
-+ err = -1;
-+ goto cleanup;
++ if (mkdir_in_gitdir(dir.buf) && errno != EEXIST) {
++ ret = error_errno(_("unable to create directory %s"), dir.buf);
++ strbuf_release(&dir);
++ return ret;
+ }
++ strbuf_release(&dir);
+ }
+
-+ err = finalize_object_file_with_mtime(tmp_file.buf, filename.buf, mtime, flags);
-+cleanup:
-+ strbuf_release(&tmp_file);
-+ strbuf_release(&filename);
-+ return err;
++ return finalize_object_file_with_mtime(tmp_file.buf, filename.buf, mtime, flags);
+}
+
int write_object_file_flags(const void *buf, unsigned long len,
const char *type, struct object_id *oid,
unsigned flags)
-@@ object-file.c: int write_object_file_flags(const void *buf, unsigned long len,
- char hdr[MAX_HEADER_LEN];
- int hdrlen = sizeof(hdr);
-
-+ /* When streaming a large blob object (marked as HASH_STREAM),
-+ * we have no chance to run "write_object_file_prepare()" to
-+ * calculate the "oid" in advance. Call "stream_loose_object()"
-+ * to write loose object in stream.
-+ */
-+ if (flags & HASH_STREAM) {
-+ hdrlen = generate_object_header(hdr, hdrlen, type, len);
-+ return stream_loose_object(oid, hdr, hdrlen, buf, len, 0, flags);
-+ }
-+
- /* Normally if we have it in the pack then we do not bother writing
- * it out into .git/objects/??/?{38} file.
- */
## object-store.h ##
@@ object-store.h: struct object_directory {
@@ object-store.h: struct object_directory {
};
+struct input_stream {
-+ const void *(*read)(const struct input_stream *, unsigned long *len);
++ const void *(*read)(struct input_stream *, unsigned long *len);
+ void *data;
+};
+
KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
struct object_directory *, 1, fspathhash, fspatheq)
+@@ object-store.h: static inline int write_object_file(const void *buf, unsigned long len,
+ return write_object_file_flags(buf, len, type, oid, 0);
+ }
+
++int write_stream_object_file(struct input_stream *in_stream, size_t len,
++ enum object_type type, time_t mtime,
++ unsigned flags, struct object_id *oid);
++
+ int hash_object_file_literally(const void *buf, unsigned long len,
+ const char *type, struct object_id *oid,
+ unsigned flags);
6: 476aaba527 ! 5: e7b4e426ef unpack-objects: unpack_non_delta_entry() read data in a stream
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ int status;
+};
+
-+static const void *feed_input_zstream(const struct input_stream *in_stream,
++static const void *feed_input_zstream(struct input_stream *in_stream,
+ unsigned long *readlen)
+{
+ struct input_zstream_data *data = in_stream->data;
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ return data->buf;
+}
+
-+static void write_stream_blob(unsigned nr, unsigned long size)
++static void write_stream_blob(unsigned nr, size_t size)
+{
+ git_zstream zstream;
+ struct input_zstream_data data;
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ data.zstream = &zstream;
+ git_inflate_init(&zstream);
+
-+ if (write_object_file_flags(&in_stream, size,
-+ type_name(OBJ_BLOB),
-+ &obj_list[nr].oid,
-+ HASH_STREAM))
++ if (write_stream_object_file(&in_stream, size, OBJ_BLOB, 0, 0,
++ &obj_list[nr].oid))
+ die(_("failed to write object in stream"));
+
+ if (zstream.total_out != size || data.status != Z_STREAM_END)
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ git_inflate_end(&zstream);
+
+ if (strict) {
-+ struct blob *blob = lookup_blob(the_repository, &obj_list[nr].oid);
++ struct blob *blob =
++ lookup_blob(the_repository, &obj_list[nr].oid);
+ if (blob)
+ blob->object.flags |= FLAG_WRITTEN;
+ else
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ void *buf;
+
+ /* Write large blob in stream without allocating full buffer. */
-+ if (!dry_run && type == OBJ_BLOB && size > big_file_streaming_threshold) {
++ if (!dry_run && type == OBJ_BLOB &&
++ size > big_file_streaming_threshold) {
+ write_stream_blob(nr, size);
+ return;
+ }
@@ environment.c: size_t packed_git_window_size = DEFAULT_PACKED_GIT_WINDOW_SIZE;
const char *editor_program;
const char *askpass_program;
- ## t/t5590-unpack-non-delta-objects.sh (new) ##
-@@
-+#!/bin/sh
-+#
-+# Copyright (c) 2021 Han Xin
-+#
-+
-+test_description='Test unpack-objects when receive pack'
-+
-+GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
-+export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
-+
-+. ./test-lib.sh
-+
-+prepare_dest () {
-+ test_when_finished "rm -rf dest.git" &&
-+ git init --bare dest.git &&
-+ git -C dest.git config core.bigFileStreamingThreshold $1 &&
-+ git -C dest.git config core.bigFileThreshold $1
-+}
-+
-+test_expect_success "setup repo with big blobs (1.5 MB)" '
-+ test-tool genrandom foo 1500000 >big-blob &&
-+ test_commit --append foo big-blob &&
-+ test-tool genrandom bar 1500000 >big-blob &&
-+ test_commit --append bar big-blob &&
-+ (
-+ cd .git &&
-+ find objects/?? -type f | sort
-+ ) >expect &&
-+ PACK=$(echo main | git pack-objects --revs test)
-+'
-+
-+test_expect_success 'setup env: GIT_ALLOC_LIMIT to 1MB' '
-+ GIT_ALLOC_LIMIT=1m &&
-+ export GIT_ALLOC_LIMIT
-+'
-+
-+test_expect_success 'fail to unpack-objects: cannot allocate' '
+ ## t/t5590-unpack-non-delta-objects.sh ##
+@@ t/t5590-unpack-non-delta-objects.sh: export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
+ prepare_dest () {
+ test_when_finished "rm -rf dest.git" &&
+ git init --bare dest.git
++ if test -n "$1"
++ then
++ git -C dest.git config core.bigFileStreamingThreshold $1
++ git -C dest.git config core.bigFileThreshold $1
++ fi
+ }
+
+ test_expect_success "setup repo with big blobs (1.5 MB)" '
+@@ t/t5590-unpack-non-delta-objects.sh: test_expect_success 'setup env: GIT_ALLOC_LIMIT to 1MB' '
+ '
+
+ test_expect_success 'fail to unpack-objects: cannot allocate' '
+- prepare_dest &&
+ prepare_dest 2m &&
-+ test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err &&
-+ grep "fatal: attempting to allocate" err &&
-+ (
-+ cd dest.git &&
-+ find objects/?? -type f | sort
-+ ) >actual &&
-+ test_file_not_empty actual &&
-+ ! test_cmp expect actual
-+'
-+
+ test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err &&
+ grep "fatal: attempting to allocate" err &&
+ (
+@@ t/t5590-unpack-non-delta-objects.sh: test_expect_success 'fail to unpack-objects: cannot allocate' '
+ ! test_cmp expect actual
+ '
+
+test_expect_success 'unpack big object in stream' '
+ prepare_dest 1m &&
++ mkdir -p dest.git/objects/05 &&
+ git -C dest.git unpack-objects <test-$PACK.pack &&
+ git -C dest.git fsck &&
+ (
@@ t/t5590-unpack-non-delta-objects.sh (new)
+ test_must_be_empty actual
+'
+
-+test_expect_success 'unpack-objects dry-run' '
-+ prepare_dest 1m &&
-+ git -C dest.git unpack-objects -n <test-$PACK.pack &&
-+ (
-+ cd dest.git &&
-+ find objects/ -type f
-+ ) >actual &&
-+ test_must_be_empty actual
-+'
-+
-+test_done
+ test_expect_success 'unpack-objects dry-run' '
+ prepare_dest &&
+ git -C dest.git unpack-objects -n <test-$PACK.pack &&Signature · 1 line
2.34.1.52.g80008efde6.agit.6.5.6
PATCH v7 1/5 unpack-objects.c: add dry_run mode for get_data()
In reply to Han XinFrom: Han Xin <[email protected]>
In dry_run mode, "get_data()" is used to verify the inflation of data, and the returned buffer will not be used at all and will be freed immediately. Even in dry_run mode, it is dangerous to allocate a full-size buffer for a large blob object. Therefore, only allocate a low memory footprint when calling "get_data()" in dry_run mode.
--- builtin/unpack-objects.c | 23 +++++++++--- t/t5590-unpack-non-delta-objects.sh | 57 +++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 6 deletions(-) create mode 100755 t/t5590-unpack-non-delta-objects.sh
builtin/unpack-objects.c +17 -6
@@ -96,15 +96,21 @@ static void use(int bytes)display_throughput (progress, consumed_bytes); } -static void *get_data (unsigned long size) +static void *get_data (size_t size,int dry_run) {git_zstream stream; -void *buf =xmallocz (size); +size_t bufsize; +void *buf;memset (&stream,0 ,sizeof (stream)); +if (dry_run && size >8192 ) + bufsize =8192 ; +else + bufsize = size; + buf =xmallocz (bufsize); stream.next_out = buf; - stream.avail_out = size; + stream.avail_out = bufsize; stream.next_in =fill (1 ); stream.avail_in = len;git_inflate_init (&stream);
@@ -124,6 +130,11 @@ static void *get_data(unsigned long size) } stream.next_in =fill (1 ); stream.avail_in = len; +if (dry_run) { +/* reuse the buffer in dry_run mode */ + stream.next_out = buf; + stream.avail_out = bufsize; + } }git_inflate_end (&stream);return buf;
@@ -323,7 +334,7 @@ static void added_object(unsigned nr, enum object_type type,static void unpack_non_delta_entry (enum object_type type,unsigned long size,unsigned nr) { -void *buf =get_data (size); +void *buf =get_data (size, dry_run);if (!dry_run && buf)write_object (nr, type, buf, size);
@@ -357,7 +368,7 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,if (type ==OBJ_REF_DELTA ) {oidread (&base_oid,fill (the_hash_algo->rawsz));use (the_hash_algo->rawsz); - delta_data =get_data (delta_size); + delta_data =get_data (delta_size, dry_run);if (dry_run || !delta_data) {free (delta_data);return ;
@@ -396,7 +407,7 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,if (base_offset <=0 || base_offset >= obj_list[nr].offset)die ("offset value out of bound for delta base object" ); - delta_data =get_data (delta_size); + delta_data =get_data (delta_size, dry_run);if (dry_run || !delta_data) {free (delta_data);return ;
t/t5590-unpack-non-delta-objects.sh new file +57 -0
57 lines not on this page · raw message
Signature · 1 line
2.34.1.52.g80008efde6.agit.6.5.6
Re: [PATCH v7 1/5] unpack-objects.c: add dry_run mode for get_data()
Quoted 28 lines · Han Xin
From: Han Xin <[email protected]>
In
dry_runmode, "get_data()" is used to verify the inflation of data, and the returned buffer will not be used at all and will be freed immediately. Even indry_runmode, it is dangerous to allocate a full-size buffer for a large blob object. Therefore, only allocate a low memory footprint when calling "get_data()" indry_runmode.Suggested-by Jiang Xin <[email protected]>Signed-off-by Han Xin <[email protected]>--- builtin/unpack-objects.c | 23 +++++++++--- t/t5590-unpack-non-delta-objects.sh | 57 +++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 6 deletions(-) create mode 100755 t/t5590-unpack-non-delta-objects.shbuiltin/unpack-objects.c +1
-1@@ -96,15 +96,21 @@ static void use(int bytes)display_throughput (progress, consumed_bytes); } -static void *get_data (unsigned long size) +static void *get_data (size_t size,int dry_run) {git_zstream stream;
builtin/unpack-objects.c static void use(int bytes) · line 110
Quoted 10 lines
builtin/unpack-objects.c +7
-1-void *buf =xmallocz (size); +size_t bufsize; +void *buf;memset (&stream,0 ,sizeof (stream)); +if (dry_run && size >8192 ) + bufsize =8192 ; +else + bufsize = size; + buf =xmallocz (bufsize);
Maybe I'm misunderstanding this, but the commit message says it would be dangerous to allocate a very larger buffer, but here we only limit the size under "dry_run".
Removing that "&& size > 8192" makes all the tests pass still, so there seems to be some missing coverage here in any case.
Quoted 53 lines
t/t5590-unpack-non-delta-objects.sh new file +47
-0@@ -0,0 +1,57 @@ +#!/bin/sh +# +# Copyright (c) 2021 Han Xin +# + +test_description='Test unpack-objects with non-delta objects' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +. ./test-lib.sh + +prepare_dest () { + test_when_finished "rm -rf dest.git" && + git init --bare dest.git +} + +test_expect_success "setup repo with big blobs (1.5 MB)" ' + test-tool genrandom foo 1500000 >big-blob && + test_commit --append foo big-blob && + test-tool genrandom bar 1500000 >big-blob && + test_commit --append bar big-blob && + ( + cd .git && + find objects/?? -type f | sort + ) >expect && + PACK=$(echo main | git pack-objects --revs test) +' + +test_expect_success 'setup env: GIT_ALLOC_LIMIT to 1MB' ' + GIT_ALLOC_LIMIT=1m && + export GIT_ALLOC_LIMIT +' + +test_expect_success 'fail to unpack-objects: cannot allocate' ' + prepare_dest && + test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err && + grep "fatal: attempting to allocate" err && + ( + cd dest.git && + find objects/?? -type f | sort + ) >actual && + test_file_not_empty actual && + ! test_cmp expect actual +' + +test_expect_success 'unpack-objects dry-run' '
t/t5590-unpack-non-delta-objects.sh line 57
Quoted 10 lines
t/t5590-unpack-non-delta-objects.sh new file +10
-0+ prepare_dest && + git -C dest.git unpack-objects -n <test-$PACK.pack && + ( + cd dest.git && + find objects/ -type f + ) >actual && + test_must_be_empty actual +' + +test_done
I commented on this "find" usage in an earlier round, I think there's a much easier way to do this. You're really just going back and forth between checking whether or not all the objects are loose.
I think that the below fix-up on top of this series is a better way to do that, and more accurate. I.e. in your test here you check "! test_cmp", which means that we could have some packed and some loose, but really what you're meaning to check is a flip-flop between "all loose?" and "no loose?.
In addition to that there was no reason to hardcode "main", we can just use HEAD. All in all I think the below fix-up makes sense:
t/t5590-unpack-non-delta-objects.sh +15 -36
@@ -5,9 +5,6 @@ test_description='Test unpack-objects with non-delta objects' -GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main -export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME - . ./test-lib.sh prepare_dest () {
@@ -20,16 +17,22 @@ prepare_dest () { fi } +assert_no_loose () { + glob=dest.git/objects/?? && + echo "$glob" >expect && + echo $glob >actual && + test_cmp expect actual +} + test_expect_success "setup repo with big blobs (1.5 MB)" ' test-tool genrandom foo 1500000 >big-blob && test_commit --append foo big-blob && test-tool genrandom bar 1500000 >big-blob && test_commit --append bar big-blob && - ( - cd .git && - find objects/?? -type f | sort - ) >expect && - PACK=$(echo main | git pack-objects --revs test) + + # Everything is loose + rmdir .git/objects/pack && + PACK=$(echo HEAD | git pack-objects --revs test) ' test_expect_success 'setup env: GIT_ALLOC_LIMIT to 1MB' '
@@ -41,51 +44,27 @@ test_expect_success 'fail to unpack-objects: cannot allocate' ' prepare_dest 2m && test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err && grep "fatal: attempting to allocate" err && - ( - cd dest.git && - find objects/?? -type f | sort - ) >actual && - test_file_not_empty actual && - ! test_cmp expect actual + rmdir dest.git/objects/pack ' test_expect_success 'unpack big object in stream' ' prepare_dest 1m && mkdir -p dest.git/objects/05 && git -C dest.git unpack-objects <test-$PACK.pack && - git -C dest.git fsck && - ( - cd dest.git && - find objects/?? -type f | sort - ) >actual && - test_cmp expect actual + rmdir dest.git/objects/pack ' test_expect_success 'unpack big object in stream with existing oids' ' prepare_dest 1m && git -C dest.git index-pack --stdin <test-$PACK.pack && - ( - cd dest.git && - find objects/?? -type f | sort - ) >actual && - test_must_be_empty actual && git -C dest.git unpack-objects <test-$PACK.pack && - git -C dest.git fsck && - ( - cd dest.git && - find objects/?? -type f | sort - ) >actual && - test_must_be_empty actual + assert_no_loose ' test_expect_success 'unpack-objects dry-run' ' prepare_dest && git -C dest.git unpack-objects -n <test-$PACK.pack && - ( - cd dest.git && - find objects/ -type f - ) >actual && - test_must_be_empty actual + assert_no_loose ' test_done
Re: [PATCH v7 1/5] unpack-objects.c: add dry_run mode for get_data()
Am 21.12.21 um 15:09 schrieb Ævar Arnfjörð Bjarmason:
Quoted 35 lines · Han Xin
Quoted 33 lines
From: Han Xin <[email protected]>
In
dry_runmode, "get_data()" is used to verify the inflation of data, and the returned buffer will not be used at all and will be freed immediately. Even indry_runmode, it is dangerous to allocate a full-size buffer for a large blob object. Therefore, only allocate a low memory footprint when calling "get_data()" indry_runmode.Suggested-by Jiang Xin <[email protected]>Signed-off-by Han Xin <[email protected]>--- builtin/unpack-objects.c | 23 +++++++++--- t/t5590-unpack-non-delta-objects.sh | 57 +++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 6 deletions(-) create mode 100755 t/t5590-unpack-non-delta-objects.shbuiltin/unpack-objects.c +3
-2@@ -96,15 +96,21 @@ static void use(int bytes)display_throughput (progress, consumed_bytes); } -static void *get_data (unsigned long size) +static void *get_data (size_t size,int dry_run) {git_zstream stream; -void *buf =xmallocz (size); +size_t bufsize; +void *buf;
Quoted 10 lines
builtin/unpack-objects.c static void use(int bytes) · line 110
Quoted 6 lines
builtin/unpack-objects.c +5
-0memset (&stream,0 ,sizeof (stream)); +if (dry_run && size >8192 ) + bufsize =8192 ; +else + bufsize = size; + buf =xmallocz (bufsize);Maybe I'm misunderstanding this, but the commit message says it would be dangerous to allocate a very larger buffer, but here we only limit the size under "
dry_run".
This patch reduces the memory usage of dry runs, as its commit message says. The memory usage of one type of actual (non-dry) unpack is reduced by patch 5.
Quoted 2 lines
Removing that "&& size > 8192" makes all the tests pass still, so there seems to be some missing coverage here in any case.
How would you test that an 8KB buffer is allocated even though a smaller one would suffice? And why? Wasting a few KB shouldn't be noticeable.
René
Re: [PATCH v7 1/5] unpack-objects.c: add dry_run mode for get_data()
Quoted 51 lines · René Scharfe
Am 21.12.21 um 15:09 schrieb Ævar Arnfjörð Bjarmason:
Quoted 35 lines · Han Xin
Quoted 33 lines
From: Han Xin <[email protected]>
In
dry_runmode, "get_data()" is used to verify the inflation of data, and the returned buffer will not be used at all and will be freed immediately. Even indry_runmode, it is dangerous to allocate a full-size buffer for a large blob object. Therefore, only allocate a low memory footprint when calling "get_data()" indry_runmode.Suggested-by Jiang Xin <[email protected]>Signed-off-by Han Xin <[email protected]>--- builtin/unpack-objects.c | 23 +++++++++--- t/t5590-unpack-non-delta-objects.sh | 57 +++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 6 deletions(-) create mode 100755 t/t5590-unpack-non-delta-objects.shbuiltin/unpack-objects.c +3
-2@@ -96,15 +96,21 @@ static void use(int bytes)display_throughput (progress, consumed_bytes); } -static void *get_data (unsigned long size) +static void *get_data (size_t size,int dry_run) {git_zstream stream; -void *buf =xmallocz (size); +size_t bufsize; +void *buf;Quoted 10 lines
builtin/unpack-objects.c static void use(int bytes) · line 110
Quoted 6 lines
builtin/unpack-objects.c +5
-0memset (&stream,0 ,sizeof (stream)); +if (dry_run && size >8192 ) + bufsize =8192 ; +else + bufsize = size; + buf =xmallocz (bufsize);Maybe I'm misunderstanding this, but the commit message says it would be dangerous to allocate a very larger buffer, but here we only limit the size under "
dry_run".This patch reduces the memory usage of dry runs, as its commit message says. The memory usage of one type of actual (non-dry) unpack is reduced by patch 5.
Quoted 5 lines
Quoted 2 lines
Removing that "&& size > 8192" makes all the tests pass still, so there seems to be some missing coverage here in any case.
How would you test that an 8KB buffer is allocated even though a smaller one would suffice? And why? Wasting a few KB shouldn't be noticeable.
That doesn't sound like it needs to be tested. I was just trying to grok what this was all doing. Thanks!
Re: [PATCH v7 1/5] unpack-objects.c: add dry_run mode for get_data()
In reply to René ScharfeQuoted 10 lines · René Scharfe
Am 21.12.21 um 15:09 schrieb Ævar Arnfjörð Bjarmason:
Quoted 3 lines
Maybe I'm misunderstanding this, but the commit message says it would be dangerous to allocate a very larger buffer, but here we only limit the size under "
dry_run".This patch reduces the memory usage of dry runs, as its commit message says. The memory usage of one type of actual (non-dry) unpack is reduced by patch 5.
For Han Xin and me, it is very challenging to write better commit log in English. Since the commit is moved to the beginning, the commit log should be rewritten as follows:
unpack-objects.c: low memory footprint for get_data() in dry_run mode
As the name implies, "get_data(size)" will allocate and return a given size of memory. Allocating memory for a large blob object may cause the system to run out of memory. Before preparing to replace calling of "get_data()" to resolve unpack issue of large blob objects, refactor "get_data()" to reduce memory footprint for dry_run mode. Because in dry_run mode, "get_data()" is only used to check the integrity of data, and the returned buffer is not used at all.
Therefore, add the flag "dry_run" as an additional parameter of "get_data()" and reuse a small buffer in dry_run mode. Because in dry_run mode, the return buffer is not the entire data that the user wants, for this reason, we will release the buffer and return NULL.
Han Xin, I think you can try to free the allocated buffer for dry_run mode inside "get_data()".
--
Jiang Xin
Re: [PATCH v7 1/5] unpack-objects.c: add dry_run mode for get_data()
In reply to Ævar Arnfjörð BjarmasonQuoted 31 lines · Ævar Arnfjörð Bjarmason
On Tue, Dec 21 2021, Han Xin wrote:
I commented on this "find" usage in an earlier round, I think there's a much easier way to do this. You're really just going back and forth between checking whether or not all the objects are loose.
I think that the below fix-up on top of this series is a better way to do that, and more accurate. I.e. in your test here you check "!
test_cmp", which means that we could have some packed and some loose, but really what you're meaning to check is a flip-flop between "all loose?" and "no loose?.In addition to that there was no reason to hardcode "main", we can just use HEAD. All in all I think the below fix-up makes sense:
t/t5590-unpack-non-delta-objects.sh +0
-3@@ -5,9 +5,6 @@ test_description='Test unpack-objects with non-delta objects' -GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main -export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME - . ./test-lib.sh prepare_dest () {
t/t5590-unpack-non-delta-objects.sh prepare_dest () { · line 23
Quoted 7 lines
t/t5590-unpack-non-delta-objects.sh +4
-0@@ -20,16 +17,22 @@ prepare_dest () { fi } +assert_no_loose () { + glob=dest.git/objects/?? && + echo "$glob" >expect && + echo $glob >actual &&
Incompatible for zsh. This may work:
eval "echo $glob" >actual &&
--
Jiang Xin
Re: [PATCH v7 1/5] unpack-objects.c: add dry_run mode for get_data()
In reply to Han XinQuoted 22 lines · Han Xin
From: Han Xin <[email protected]>
In
dry_runmode, "get_data()" is used to verify the inflation of data, and the returned buffer will not be used at all and will be freed immediately. Even indry_runmode, it is dangerous to allocate a full-size buffer for a large blob object. Therefore, only allocate a low memory footprint when calling "get_data()" indry_runmode.Suggested-by Jiang Xin <[email protected]>Signed-off-by Han Xin <[email protected]>--- builtin/unpack-objects.c | 23 +++++++++--- t/t5590-unpack-non-delta-objects.sh | 57 +++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 6 deletions(-) create mode 100755 t/t5590-unpack-non-delta-objects.shbuiltin/unpack-objects.c +0
-0
builtin/unpack-objects.c static void use(int bytes) · line 99
Quoted 5 lines
builtin/unpack-objects.c +1
-1@@ -96,15 +96,21 @@ static void use(int bytes)display_throughput (progress, consumed_bytes); } -static void *get_data (unsigned long size) +static void *get_data (size_t size,int dry_run)
After a offline talk with Han Xin, we feel it is not necessary to pass "dry_run" as a argument, use the file-scope static variable directly in "get_data()".
builtin/unpack-objects.c static void use(int bytes) · line 106
Quoted 8 lines
builtin/unpack-objects.c +3
-1{git_zstream stream; -void *buf =xmallocz (size); +size_t bufsize; +void *buf;memset (&stream,0 ,sizeof (stream)); +if (dry_run && size >8192 )
Use the file-scope static variable "dry_run".
PATCH v7 2/5 object-file API: add a format_object_header() function
In reply to Han XinFrom: Ævar Arnfjörð Bjarmason <[email protected]>
Add a convenience function to wrap the xsnprintf() command that generates loose object headers. This code was copy/pasted in various parts of the codebase, let's define it in one place and re-use it from there.
All except one caller of it had a valid "enum object_type" for us, it's only write_object_file_prepare() which might need to deal with "git hash-object --literally" and a potential garbage type. Let's have the primary API use an "enum object_type", and define an *_extended() function that can take an arbitrary "const char *" for the type.
See [1] for the discussion that prompted this patch, i.e. new code in object-file.c that wanted to copy/paste the xsnprintf() invocation.
1. https://lore.kernel.org/git/[email protected]/
--- builtin/index-pack.c | 3 +-- bulk-checkin.c | 4 ++-- cache.h | 21 +++++++++++++++++++++ http-push.c | 2 +- object-file.c | 14 +++++++++++--- 5 files changed, 36 insertions(+), 8 deletions(-)
builtin/index-pack.c +1 -2
@@ -449,8 +449,7 @@ static void *unpack_entry_data(off_t offset, unsigned long size,int hdrlen;if (!is_delta_type (type)) { - hdrlen =xsnprintf (hdr,sizeof (hdr),"%s %" PRIuMAX, -type_name (type),(uintmax_t )size) +1 ; + hdrlen =format_object_header (hdr,sizeof (hdr), type, (uintmax_t )size); the_hash_algo->init_fn (&c); the_hash_algo->update_fn (&c, hdr, hdrlen); }else
bulk-checkin.c +2 -2
@@ -220,8 +220,8 @@ static int deflate_to_pack(struct bulk_checkin_state *state,if (seekback == (off_t) -1 )return error ("cannot find the current offset" ); - header_len =xsnprintf ((char *)obuf,sizeof (obuf),"%s %" PRIuMAX, -type_name (type), (uintmax_t )size) +1 ; + header_len =format_object_header ((char *)obuf,sizeof (obuf), + type, (uintmax_t )size); the_hash_algo->init_fn (&ctx); the_hash_algo->update_fn (&ctx, obuf, header_len);
cache.h +21 -0
@@ -1310,6 +1310,27 @@ enum unpack_loose_header_result unpack_loose_header(git_zstream *stream,unsigned long bufsiz, struct strbuf *hdrbuf); +/** +* format_object_header() is a thin wrapper around s xsnprintf() that +* writes the initial "<type> <obj-len>" part of the loose object +* header. It returns the size that snprintf() returns + 1. +* +* The format_object_header_extended() function allows for writing a +* type_name that's not one of the "enum object_type" types. This is +* used for "git hash-object --literally". Pass in a OBJ_NONE as the +* type, and a non-NULL "type_str" to do that. +* +* format_object_header() is a convenience wrapper for +* format_object_header_extended(). +*/ +int format_object_header_extended (char *str,size_t size,enum object_type type, +const char *type_str,size_t objsize); +static inline int format_object_header (char *str,size_t size, +enum object_type type,size_t objsize) +{ +return format_object_header_extended (str, size, type,NULL , objsize); +} +/** * parse_loose_header() parses the starting "<type> <len>\0" of an * object. If it doesn't follow that format -1 is returned. To check
http-push.c +1 -1
@@ -363,7 +363,7 @@ static void start_put(struct transfer_request *request)git_zstream stream; unpacked =read_object_file (&request->obj->oid, &type, &len); - hdrlen =xsnprintf (hdr,sizeof (hdr),"%s %" PRIuMAX ,type_name (type), (uintmax_t )len) +1 ; + hdrlen =format_object_header (hdr,sizeof (hdr), type, (uintmax_t )len);/* Set it up */ git_deflate_init (&stream, zlib_compression_level);
object-file.c +11 -3
@@ -1006,6 +1006,14 @@ void *xmmap(void *start, size_t length,return ret; } +int format_object_header_extended (char *str,size_t size,enum object_type type, +const char *typestr,size_t objsize) +{ +const char *s = type ==OBJ_NONE ? typestr :type_name (type); + +return xsnprintf (str, size,"%s %" PRIuMAX, s, (uintmax_t )objsize) +1 ; +} +/* * With an in-core object data in "map", rehash it to make sure the * object name actually matches "oid" to detect object corruption.
@@ -1034,7 +1042,7 @@ int check_object_signature(struct repository *r, const struct object_id *oid,return -1 ;/* Generate the header */ - hdrlen =xsnprintf (hdr,sizeof (hdr),"%s %" PRIuMAX ,type_name (obj_type), (uintmax_t )size) +1 ; + hdrlen =format_object_header (hdr,sizeof (hdr), obj_type, size);/* Sha1.. */ r->hash_algo->init_fn (&c);
@@ -1734,7 +1742,7 @@ static void write_object_file_prepare(const struct git_hash_algo *algo,git_hash_ctx c;/* Generate the header */ - *hdrlen =xsnprintf (hdr, *hdrlen,"%s %" PRIuMAX , type, (uintmax_t )len)+1 ; + *hdrlen =format_object_header_extended (hdr, *hdrlen,OBJ_NONE , type, len);/* Sha1.. */ algo->init_fn (&c);
@@ -2006,7 +2014,7 @@ int force_object_loose(const struct object_id *oid, time_t mtime) buf =read_object (the_repository, oid, &type, &len);if (!buf)return error (_ ("cannot read object for %s" ),oid_to_hex (oid)); - hdrlen =xsnprintf (hdr,sizeof (hdr),"%s %" PRIuMAX ,type_name (type), (uintmax_t )len) +1 ; + hdrlen =format_object_header (hdr,sizeof (hdr), type, len); ret =write_loose_object (oid, hdr, hdrlen, buf, len, mtime,0 );free (buf);
Signature · 1 line
2.34.1.52.g80008efde6.agit.6.5.6
Re: [PATCH v7 2/5] object-file API: add a format_object_header() function
Am 21.12.21 um 12:51 schrieb Han Xin:
Quoted 33 lines
From: Ævar Arnfjörð Bjarmason <[email protected]>
Add a convenience function to wrap the
xsnprintf()command that generates loose object headers. This code was copy/pasted in various parts of the codebase, let's define it in one place and re-use it from there.All except one caller of it had a valid "enum
object_type" for us, it's onlywrite_object_file_prepare()which might need to deal with "git hash-object --literally" and a potential garbage type. Let's have the primary API use an "enumobject_type", and define an*_extended()function that can take an arbitrary "const char *" for the type.See [1] for the discussion that prompted this patch, i.e. new code in
object-file.cthat wanted to copy/paste thexsnprintf()invocation.1. https://lore.kernel.org/git/[email protected]/
Signed-off-by Ævar Arnfjörð Bjarmason <[email protected]>Signed-off-by Han Xin <[email protected]>--- builtin/index-pack.c | 3 +-- bulk-checkin.c | 4 ++-- cache.h | 21 +++++++++++++++++++++ http-push.c | 2 +- object-file.c | 14 +++++++++++--- 5 files changed, 36 insertions(+), 8 deletions(-)builtin/index-pack.c +0
-0
builtin/index-pack.c static void *unpack_entry_data(off_t offset, unsigned long size, · line 452
Quoted 6 lines
builtin/index-pack.c +1
-2@@ -449,8 +449,7 @@ static void *unpack_entry_data(off_t offset, unsigned long size,int hdrlen;if (!is_delta_type (type)) { - hdrlen =xsnprintf (hdr,sizeof (hdr),"%s %" PRIuMAX, -type_name (type),(uintmax_t )size) +1 ; + hdrlen =format_object_header (hdr,sizeof (hdr), type, (uintmax_t )size);
^^^^^^^^^^^
This explicit cast is unnecessary. It was needed with xsnprintf(), but that implementation detail is handled inside the new helper function.
(format_object_header() takes a size_t; even if unsigned long would be wider than that on some weird architecture, casting the size to uintmax_t will not avoid the implicit truncation happening during the function call.)
Quoted 8 lines
builtin/index-pack.c +0
-0the_hash_algo->init_fn (&c); the_hash_algo->update_fn (&c, hdr, hdrlen); }else bulk-checkin.c +0
-0
bulk-checkin.c static int deflate_to_pack(struct bulk_checkin_state *state, · line 224
Quoted 7 lines
bulk-checkin.c +2
-2@@ -220,8 +220,8 @@ static int deflate_to_pack(struct bulk_checkin_state *state,if (seekback == (off_t) -1 )return error ("cannot find the current offset" ); - header_len =xsnprintf ((char *)obuf,sizeof (obuf),"%s %" PRIuMAX, -type_name (type), (uintmax_t )size) +1 ; + header_len =format_object_header ((char *)obuf,sizeof (obuf), + type, (uintmax_t )size);
^^^^^^^^^^^
Same here, just that size is already of type size_t, so a cast makes even less sense.
Quoted 40 lines
bulk-checkin.c +0
-0the_hash_algo->init_fn (&ctx); the_hash_algo->update_fn (&ctx, obuf, header_len);cache.h +21
-0@@ -1310,6 +1310,27 @@ enum unpack_loose_header_result unpack_loose_header(git_zstream *stream,unsigned long bufsiz, struct strbuf *hdrbuf); +/** +* format_object_header() is a thin wrapper around s xsnprintf() that +* writes the initial "<type> <obj-len>" part of the loose object +* header. It returns the size that snprintf() returns + 1. +* +* The format_object_header_extended() function allows for writing a +* type_name that's not one of the "enum object_type" types. This is +* used for "git hash-object --literally". Pass in a OBJ_NONE as the +* type, and a non-NULL "type_str" to do that. +* +* format_object_header() is a convenience wrapper for +* format_object_header_extended(). +*/ +int format_object_header_extended (char *str,size_t size,enum object_type type, +const char *type_str,size_t objsize); +static inline int format_object_header (char *str,size_t size, +enum object_type type,size_t objsize) +{ +return format_object_header_extended (str, size, type,NULL , objsize); +} +/** * parse_loose_header() parses the starting "<type> <len>\0" of an * object. If it doesn't follow that format -1 is returned. To check http-push.c +0
-0
http-push.c static void start_put(struct transfer_request *request) · line 366
Quoted 5 lines
http-push.c +1
-1@@ -363,7 +363,7 @@ static void start_put(struct transfer_request *request)git_zstream stream; unpacked =read_object_file (&request->obj->oid, &type, &len); - hdrlen =xsnprintf (hdr,sizeof (hdr),"%s %" PRIuMAX ,type_name (type), (uintmax_t )len) +1 ; + hdrlen =format_object_header (hdr,sizeof (hdr), type, (uintmax_t )len);
^^^^^^^^^^^
Same here; len is of type unsigned long.
Quoted 8 lines
http-push.c +0
-0/* Set it up */ git_deflate_init (&stream, zlib_compression_level);object-file.c +0
-0
object-file.c void *xmmap(void *start, size_t length, · line 1014
Quoted 9 lines
object-file.c +6
-0@@ -1006,6 +1006,14 @@ void *xmmap(void *start, size_t length,return ret; } +int format_object_header_extended (char *str,size_t size,enum object_type type, +const char *typestr,size_t objsize) +{ +const char *s = type ==OBJ_NONE ? typestr :type_name (type); + +return xsnprintf (str, size,"%s %" PRIuMAX, s, (uintmax_t )objsize) +1 ;
^^^^^^^^^^^
This cast is necessary to match PRIuMAX. And that is used because the z modifier (as in e.g. printf("%zu", sizeof(size_t));) was only added in C99 and not all platforms may have it. (Perhaps this cautious approach is worth revisiting separately, now that some time has passed, but this patch series should still use PRIuMAX, as it does.)
Quoted 24 lines
object-file.c +4
-2+} +/* * With an in-core object data in "map", rehash it to make sure the * object name actually matches "oid" to detect object corruption. @@ -1034,7 +1042,7 @@ int check_object_signature(struct repository *r, const struct object_id *oid,return -1 ;/* Generate the header */ - hdrlen =xsnprintf (hdr,sizeof (hdr),"%s %" PRIuMAX ,type_name (obj_type), (uintmax_t )size) +1 ; + hdrlen =format_object_header (hdr,sizeof (hdr), obj_type, size);/* Sha1.. */ r->hash_algo->init_fn (&c);@@ -1734,7 +1742,7 @@ static void write_object_file_prepare(const struct git_hash_algo *algo,git_hash_ctx c;/* Generate the header */ - *hdrlen =xsnprintf (hdr, *hdrlen,"%s %" PRIuMAX , type, (uintmax_t )len)+1 ; + *hdrlen =format_object_header_extended (hdr, *hdrlen,OBJ_NONE , type, len);/* Sha1.. */ algo->init_fn (&c);
object-file.c int force_object_loose(const struct object_id *oid, time_t mtime) · line 2020
Quoted 8 lines
object-file.c +1
-1@@ -2006,7 +2014,7 @@ int force_object_loose(const struct object_id *oid, time_t mtime) buf =read_object (the_repository, oid, &type, &len);if (!buf)return error (_ ("cannot read object for %s" ),oid_to_hex (oid)); - hdrlen =xsnprintf (hdr,sizeof (hdr),"%s %" PRIuMAX ,type_name (type), (uintmax_t )len) +1 ; + hdrlen =format_object_header (hdr,sizeof (hdr), type, len); ret =write_loose_object (oid, hdr, hdrlen, buf, len, mtime,0 );free (buf);
No explicit cast in these three cases -- good. They all pass an unsigned long as last parameter btw.
René
C99 %z (was: [PATCH v7 2/5] object-file API: add a format_object_header() function)
Quoted 14 lines · René Scharfe
Am 21.12.21 um 12:51 schrieb Han Xin:
Quoted 13 lines
From: Ævar Arnfjörð Bjarmason <[email protected]>
[...]builtin/index-pack.c +0
-0the_hash_algo->init_fn (&c); the_hash_algo->update_fn (&c, hdr, hdrlen); }else bulk-checkin.c +0
-0@@ -220,8 +220,8 @@ static int deflate_to_pack(struct bulk_checkin_state *state,if (seekback == (off_t) -1 )return error ("cannot find the current offset" );
Quoted 7 lines
bulk-checkin.c static int deflate_to_pack(struct bulk_checkin_state *state, · line 224
Quoted 4 lines
bulk-checkin.c +2
-2- header_len =xsnprintf ((char *)obuf,sizeof (obuf),"%s %" PRIuMAX, -type_name (type), (uintmax_t )size) +1 ; + header_len =format_object_header ((char *)obuf,sizeof (obuf), + type, (uintmax_t )size);^^^^^^^^^^^Same here, just that size is already of type
size_t, so a cast makes even less sense.
Thanks, this and the below is something I made sure to include in a re-roll I'm about to send (to do these cleanups in object-file.c separately from Han Xin's series).
Quoted 7 lines
bulk-checkin.c static int deflate_to_pack(struct bulk_checkin_state *state,
Quoted 6 lines
bulk-checkin.c +6
-0+int format_object_header_extended (char *str,size_t size,enum object_type type, +const char *typestr,size_t objsize) +{ +const char *s = type ==OBJ_NONE ? typestr :type_name (type); + +return xsnprintf (str, size,"%s %" PRIuMAX, s, (uintmax_t )objsize) +1 ;^^^^^^^^^^^
Quoted 5 lines
This cast is necessary to match PRIuMAX. And that is used because the z modifier (as in e.g.
printf("%zu", sizeof(size_t));) was only added in C99 and not all platforms may have it. (Perhaps this cautious approach is worth revisiting separately, now that some time has passed, but this patch series should still use PRIuMAX, as it does.)
I tried to use %z recently and found that the CI breaks on Windows, but this was a few months ago. But I think the status of that particular C99 feature is that we can't use it freely, unfortunately. I may be wrong about that, I haven't looked it any detail beyond running those CI errors.
Re: [PATCH v7 2/5] object-file API: add a format_object_header() function
In reply to Han XinQuoted 21 lines · Han Xin
From: Ævar Arnfjörð Bjarmason <[email protected]>
Add a convenience function to wrap the
xsnprintf()command that generates loose object headers. This code was copy/pasted in various parts of the codebase, let's define it in one place and re-use it from there.All except one caller of it had a valid "enum
object_type" for us, it's onlywrite_object_file_prepare()which might need to deal with "git hash-object --literally" and a potential garbage type. Let's have the primary API use an "enumobject_type", and define an*_extended()function that can take an arbitrary "const char *" for the type.See [1] for the discussion that prompted this patch, i.e. new code in
object-file.cthat wanted to copy/paste thexsnprintf()invocation.1. https://lore.kernel.org/git/[email protected]/
Signed-off-by Ævar Arnfjörð Bjarmason <[email protected]>Signed-off-by Han Xin <[email protected]>
Quoted 7 lines
--- builtin/index-pack.c | 3 +-- bulk-checkin.c | 4 ++-- cache.h | 21 +++++++++++++++++++++ http-push.c | 2 +- object-file.c | 14 +++++++++++--- 5 files changed, 36 insertions(+), 8 deletions(-)
After a offline review with Han Xin, we feel it's better to move this fixup commit to the end of this series, and this commit will also fix an additional "xsnprintf()" we introduced in this series.
--
Jiang Xin
PATCH v7 3/5 object-file.c: refactor write_loose_object() to reuse in stream version
In reply to Han XinFrom: Han Xin <[email protected]>
We used to call "get_data()" in "unpack_non_delta_entry()" to read the entire contents of a blob object, no matter how big it is. This implementation may consume all the memory and cause OOM.
This can be improved by feeding data to "stream_loose_object()" in stream instead of read into the whole buf.
As this new method "stream_loose_object()" has many similarities with "write_loose_object()", we split up "write_loose_object()" into some steps:
1. Figuring out a path for the (temp) object file.
2. Creating the tempfile.
3. Setting up zlib and write header.
4. Write object data and handle errors.
5. Optionally, do someting after write, maybe force a loose object if
"mtime".
--- object-file.c | 100 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 65 insertions(+), 35 deletions(-)
object-file.c +65 -35
@@ -1751,6 +1751,25 @@ static void write_object_file_prepare(const struct git_hash_algo *algo, algo->final_oid_fn (oid, &c); } +/* +* Move the just written object with proper mtime into its final resting place. +*/ +static int finalize_object_file_with_mtime (const char *tmpfile, +const char *filename, +time_t mtime, +unsigned flags) +{ +struct utimbuf utb; + +if (mtime) { + utb.actime = mtime; + utb.modtime = mtime; +if (utime (tmpfile, &utb) <0 && !(flags &HASH_SILENT )) +warning_errno (_ ("failed utime() on %s" ), tmpfile); + } +return finalize_object_file (tmpfile, filename); +} +/* * Move the just written object into its final resting place. */
@@ -1836,7 +1855,8 @@ static inline int directory_size(const char *filename)* We want to avoid cross-directory filename renames, because those * can have problems on various filesystems (FAT, NFS, Coda). */ -static int create_tmpfile (struct strbuf *tmp,const char *filename) +static int create_tmpfile (struct strbuf *tmp,const char *filename, +unsigned flags) {int fd, dirlen =directory_size (filename);
@@ -1844,7 +1864,9 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)strbuf_add (tmp, filename, dirlen);strbuf_addstr (tmp,"tmp_obj_XXXXXX" ); fd =git_mkstemp_mode (tmp->buf,0444 ); -if (fd <0 && dirlen && errno ==ENOENT ) { + do { + if (fd >= 0 || !dirlen || errno != ENOENT) + break; /* * Make sure the directory exists; note that the contents * of the buffer are undefined after mkstemp returns an
@@ -1854,17 +1876,48 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)strbuf_reset (tmp);strbuf_add (tmp, filename, dirlen -1 );if (mkdir (tmp->buf,0777 ) && errno !=EEXIST ) -return -1 ; +break ;if (adjust_shared_perm (tmp->buf)) -return -1 ; +break ;/* Try again */ strbuf_addstr (tmp,"/tmp_obj_XXXXXX" ); fd =git_mkstemp_mode (tmp->buf,0444 ); + }while (0 ); + +if (fd <0 && !(flags &HASH_SILENT )) { +if (errno ==EACCES ) +return error (_ ("insufficient permission for adding an " +"object to repository database %s" ), +get_object_directory ()); +else +return error_errno (_ ("unable to create temporary file" )); } +return fd; } +static void setup_stream_and_header (git_zstream *stream, +unsigned char *compressed, +unsigned long compressed_size, +git_hash_ctx *c, +char *hdr, +int hdrlen) +{ +/* Set it up */ +git_deflate_init (stream, zlib_compression_level); + stream->next_out = compressed; + stream->avail_out = compressed_size; + the_hash_algo->init_fn (c); + +/* First header.. */ + stream->next_in = (unsigned char *)hdr; + stream->avail_in = hdrlen; +while (git_deflate (stream,0 ) ==Z_OK ) + ;/* nothing */ + the_hash_algo->update_fn (c, hdr, hdrlen); +} +static int write_loose_object (const struct object_id *oid,char *hdr,int hdrlen,const void *buf,unsigned long len,time_t mtime,unsigned flags)
@@ -1879,28 +1932,13 @@ static int write_loose_object(const struct object_id *oid, char *hdr,loose_object_path (the_repository, &filename, oid); - fd =create_tmpfile (&tmp_file, filename.buf); -if (fd <0 ) { -if (flags &HASH_SILENT ) -return -1 ; -else if (errno ==EACCES ) -return error (_ ("insufficient permission for adding an object to repository database %s" ),get_object_directory ()); -else -return error_errno (_ ("unable to create temporary file" )); - } - -/* Set it up */ -git_deflate_init (&stream, zlib_compression_level); - stream.next_out = compressed; - stream.avail_out =sizeof (compressed); - the_hash_algo->init_fn (&c); + fd =create_tmpfile (&tmp_file, filename.buf, flags); +if (fd <0 ) +return -1 ; -/* First header.. */ - stream.next_in = (unsigned char *)hdr; - stream.avail_in = hdrlen; -while (git_deflate (&stream,0 ) ==Z_OK ) - ;/* nothing */ - the_hash_algo->update_fn (&c, hdr, hdrlen); +/* Set it up and write header */ +setup_stream_and_header (&stream, compressed,sizeof (compressed), + &c, hdr, hdrlen);/* Then the data itself.. */ stream.next_in = (void *)buf;
@@ -1929,16 +1967,8 @@ static int write_loose_object(const struct object_id *oid, char *hdr,close_loose_object (fd); -if (mtime) { -struct utimbuf utb; - utb.actime = mtime; - utb.modtime = mtime; -if (utime (tmp_file.buf, &utb) <0 && - !(flags &HASH_SILENT )) -warning_errno (_ ("failed utime() on %s" ), tmp_file.buf); - } - -return finalize_object_file (tmp_file.buf, filename.buf); +return finalize_object_file_with_mtime (tmp_file.buf, filename.buf, + mtime, flags); }static int freshen_loose_object (const struct object_id *oid)
Signature · 1 line
2.34.1.52.g80008efde6.agit.6.5.6
Re: [PATCH v7 3/5] object-file.c: refactor write_loose_object() to reuse in stream version
Quoted 12 lines · Han Xin
From: Han Xin <[email protected]>
[...]object-file.c +2
-2@@ -1854,17 +1876,48 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)strbuf_reset (tmp);strbuf_add (tmp, filename, dirlen -1 );if (mkdir (tmp->buf,0777 ) && errno !=EEXIST ) -return -1 ; +break ;if (adjust_shared_perm (tmp->buf)) -return -1 ; +break ;
object-file.c static int create_tmpfile(struct strbuf *tmp, const char *filename) · line 1892
Quoted 10 lines
object-file.c +7
-0/* Try again */ strbuf_addstr (tmp,"/tmp_obj_XXXXXX" ); fd =git_mkstemp_mode (tmp->buf,0444 ); + }while (0 ); + +if (fd <0 && !(flags &HASH_SILENT )) { +if (errno ==EACCES ) +return error (_ ("insufficient permission for adding an " +"object to repository database %s" ), +get_object_directory ());
This should be an error_errno() instead, ...
object-file.c static int create_tmpfile(struct strbuf *tmp, const char *filename) · line 1894
Quoted 2 lines
object-file.c +2
-0+else +return error_errno (_ ("unable to create temporary file" ));
...and we can just fold this whole if/else into one condition with a briefer message, e.g.:
error_errno(_("unable to add object to '%s'"), get_object_directory());Or whatever, unless there's another bug here where you inverted these conditions, and the "else" really should not use "error_errno" but "error".... (I don't know...)
Re: [PATCH v7 3/5] object-file.c: refactor write_loose_object() to reuse in stream version
Quoted 16 lines · Ævar Arnfjörð Bjarmason
Quoted 12 lines · Han Xin
From: Han Xin <[email protected]>
[...]object-file.c +2
-2@@ -1854,17 +1876,48 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)strbuf_reset (tmp);strbuf_add (tmp, filename, dirlen -1 );if (mkdir (tmp->buf,0777 ) && errno !=EEXIST ) -return -1 ; +break ;if (adjust_shared_perm (tmp->buf)) -return -1 ; +break ;
Quoted 12 lines
object-file.c static int create_tmpfile(struct strbuf *tmp, const char *filename) · line 1892
Quoted 10 lines
object-file.c +7
-0/* Try again */ strbuf_addstr (tmp,"/tmp_obj_XXXXXX" ); fd =git_mkstemp_mode (tmp->buf,0444 ); + }while (0 ); + +if (fd <0 && !(flags &HASH_SILENT )) { +if (errno ==EACCES ) +return error (_ ("insufficient permission for adding an " +"object to repository database %s" ), +get_object_directory ());This should be an
error_errno()instead, ...
We already know the errno (EACCESS) and output a decent error message, so using error() is OK. BTW, it's just a refactor by copy & paste.
Quoted 12 lines
object-file.c static int create_tmpfile(struct strbuf *tmp, const char *filename) · line 1894
Quoted 3 lines
object-file.c +2
-0+else +return error_errno (_ ("unable to create temporary file" ));...and we can just fold this whole if/else into one condition with a briefer message, e.g.:
error_errno(_("unable to add object to '%s'"), get_object_directory());Or whatever, unless there's another bug here where you inverted these conditions, and the "else" really should not use "
error_errno" but "error".... (I don't know...)
PATCH v7 4/5 object-file.c: add "write_stream_object_file()" to support read in stream
In reply to Han XinFrom: Han Xin <[email protected]>
We used to call "get_data()" in "unpack_non_delta_entry()" to read the entire contents of a blob object, no matter how big it is. This implementation may consume all the memory and cause OOM.
This can be improved by feeding data to "write_stream_object_file()" in a stream. The input stream is implemented as an interface.
The difference with "write_loose_object()" is that we have no chance to run "write_object_file_prepare()" to calculate the oid in advance. In "write_loose_object()", we know the oid and we can write the temporary file in the same directory as the final object, but for an object with an undetermined oid, we don't know the exact directory for the object, so we have to save the temporary file in ".git/objects/" directory instead.
"freshen_packed_object()" or "freshen_loose_object()" will be called inside "write_stream_object_file()" after obtaining the "oid".
--- object-file.c | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++ object-store.h | 9 ++++++ 2 files changed, 94 insertions(+)
object-file.c +85 -0
@@ -1989,6 +1989,91 @@ static int freshen_packed_object(const struct object_id *oid)return 1 ; } +int write_stream_object_file (struct input_stream *in_stream,size_t len, +enum object_type type,time_t mtime, +unsigned flags,struct object_id *oid) +{ +int fd, ret, flush =0 ; +unsigned char compressed[4096 ]; +git_zstream stream; +git_hash_ctx c; +struct object_id parano_oid; +static struct strbuf tmp_file =STRBUF_INIT ; +static struct strbuf filename =STRBUF_INIT ; +int dirlen; +char hdr[MAX_HEADER_LEN ]; +int hdrlen =sizeof (hdr); + +/* Since "filename" is defined as static, it will be reused. So reset it +* first before using it. */ +strbuf_reset (&filename); +/* When oid is not determined, save tmp file to odb path. */ +strbuf_addf (&filename,"%s/" ,get_object_directory ()); + + fd =create_tmpfile (&tmp_file, filename.buf, flags); +if (fd <0 ) +return -1 ; + + hdrlen =format_object_header (hdr, hdrlen, type, len); + +/* Set it up and write header */ +setup_stream_and_header (&stream, compressed,sizeof (compressed), + &c, hdr, hdrlen); + +/* Then the data itself.. */ +do { +unsigned char *in0 = stream.next_in; +if (!stream.avail_in) { +const void *in = in_stream->read (in_stream, &stream.avail_in); + stream.next_in = (void *)in; + in0 = (unsigned char *)in; +/* All data has been read. */ +if (len + hdrlen == stream.total_in + stream.avail_in) + flush =Z_FINISH ; + } + ret =git_deflate (&stream, flush); + the_hash_algo->update_fn (&c, in0, stream.next_in - in0); +if (write_buffer (fd, compressed, stream.next_out - compressed) <0 ) +die (_ ("unable to write loose object file" )); + stream.next_out = compressed; + stream.avail_out =sizeof (compressed); + }while (ret ==Z_OK || ret ==Z_BUF_ERROR ); + +if (ret !=Z_STREAM_END ) +die (_ ("unable to deflate new object streamingly (%d)" ), ret); + ret =git_deflate_end_gently (&stream); +if (ret !=Z_OK ) +die (_ ("deflateEnd on object streamingly failed (%d)" ), ret); + the_hash_algo->final_oid_fn (¶no_oid, &c); + +close_loose_object (fd); + +oidcpy (oid, ¶no_oid); + +if (freshen_packed_object (oid) ||freshen_loose_object (oid)) { +unlink_or_warn (tmp_file.buf); +return 0 ; + } + +loose_object_path (the_repository, &filename, oid); + +/* We finally know the object path, and create the missing dir. */ + dirlen =directory_size (filename.buf); +if (dirlen) { +struct strbuf dir =STRBUF_INIT ; +strbuf_add (&dir, filename.buf, dirlen -1 ); + +if (mkdir_in_gitdir (dir.buf) && errno !=EEXIST ) { + ret =error_errno (_ ("unable to create directory %s" ), dir.buf); +strbuf_release (&dir); +return ret; + } +strbuf_release (&dir); + } + +return finalize_object_file_with_mtime (tmp_file.buf, filename.buf, mtime, flags); +} +int write_object_file_flags (const void *buf,unsigned long len,const char *type,struct object_id *oid,unsigned flags)
object-store.h +9 -0
@@ -34,6 +34,11 @@ struct object_directory {char *path; }; +struct input_stream { +const void *(*read)(struct input_stream *,unsigned long *len); +void *data; +}; + KHASH_INIT(odb_path_map, const char * /* key: odb_path */, struct object_directory *, 1, fspathhash, fspatheq)
@@ -232,6 +237,10 @@ static inline int write_object_file(const void *buf, unsigned long len,return write_object_file_flags (buf, len, type, oid,0 ); } +int write_stream_object_file (struct input_stream *in_stream,size_t len, +enum object_type type,time_t mtime, +unsigned flags,struct object_id *oid); +int hash_object_file_literally (const void *buf,unsigned long len,const char *type,struct object_id *oid,unsigned flags);
Signature · 1 line
2.34.1.52.g80008efde6.agit.6.5.6
Re: [PATCH v7 4/5] object-file.c: add "write_stream_object_file()" to support read in stream
Quoted 12 lines · Han Xin
From: Han Xin <[email protected]>
[...]object-file.c +10
-0+int write_stream_object_file (struct input_stream *in_stream,size_t len, +enum object_type type,time_t mtime, +unsigned flags,struct object_id *oid) +{ +int fd, ret, flush =0 ; +unsigned char compressed[4096 ]; +git_zstream stream; +git_hash_ctx c; +struct object_id parano_oid; +static struct strbuf tmp_file =STRBUF_INIT ;
object-file.c static int freshen_packed_object(const struct object_id *oid) · line 2011
Quoted 10 lines
object-file.c +10
-0+static struct strbuf filename =STRBUF_INIT ; +int dirlen; +char hdr[MAX_HEADER_LEN ]; +int hdrlen =sizeof (hdr); + +/* Since "filename" is defined as static, it will be reused. So reset it +* first before using it. */ +strbuf_reset (&filename); +/* When oid is not determined, save tmp file to odb path. */ +strbuf_addf (&filename,"%s/" ,get_object_directory ());
I realize this is somewhat following the pattern of code you moved around earlier, but FWIW I think these sorts of comments are really over-doing it. I.e. we try not to comment on things that are obvious from the code itself.
Also René's comment on v6 still applies here:
Given that this function is only used for huge objects I think making the strbufs non-static and releasing them is the best choice here.
I thin just making them non-static and doing a strbuf_release() as he suggested is best here.
Quoted 25 lines
object-file.c +25
-0+ + fd =create_tmpfile (&tmp_file, filename.buf, flags); +if (fd <0 ) +return -1 ; + + hdrlen =format_object_header (hdr, hdrlen, type, len); + +/* Set it up and write header */ +setup_stream_and_header (&stream, compressed,sizeof (compressed), + &c, hdr, hdrlen); + +/* Then the data itself.. */ + do { + unsigned char *in0 = stream.next_in; + if (!stream.avail_in) { + const void *in = in_stream->read(in_stream, &stream.avail_in); + stream.next_in = (void *)in; + in0 = (unsigned char *)in; + /* All data has been read. */ + if (len + hdrlen == stream.total_in + stream.avail_in) + flush = Z_FINISH; + } + ret = git_deflate(&stream, flush); + the_hash_algo->update_fn(&c, in0, stream.next_in - in0); + if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
object-file.c static int freshen_packed_object(const struct object_id *oid) · line 2046
Quoted 10 lines
object-file.c +10
-0+die (_ ("unable to write loose object file" )); + stream.next_out = compressed; + stream.avail_out =sizeof (compressed); + }while (ret ==Z_OK || ret ==Z_BUF_ERROR ); + +if (ret !=Z_STREAM_END ) +die (_ ("unable to deflate new object streamingly (%d)" ), ret); + ret =git_deflate_end_gently (&stream); +if (ret !=Z_OK ) +die (_ ("deflateEnd on object streamingly failed (%d)" ), ret);
nit: let's say "unable to stream deflate new object" or something, and not use the confusing (invented?) word "streamingly".
object-file.c static int freshen_packed_object(const struct object_id *oid) · line 2051
Quoted 5 lines
object-file.c +5
-0+ the_hash_algo->final_oid_fn (¶no_oid, &c); + +close_loose_object (fd); + +oidcpy (oid, ¶no_oid);
I see there's still quite a bit of duplication between this and write_loose_object(), but maybe it's not easy to factor out.
object-file.c static int freshen_packed_object(const struct object_id *oid) · line 2064
Quoted 12 lines
object-file.c +12
-0+if (freshen_packed_object (oid) ||freshen_loose_object (oid)) { +unlink_or_warn (tmp_file.buf); +return 0 ; + } + +loose_object_path (the_repository, &filename, oid); + +/* We finally know the object path, and create the missing dir. */ + dirlen =directory_size (filename.buf); +if (dirlen) { +struct strbuf dir =STRBUF_INIT ; +strbuf_add (&dir, filename.buf, dirlen -1 );
Just a minor nit, but I noticed we could have this on top, i.e. this "remove the slash" is now what 1/3 users of it wan:
object-file.c | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/object-file.c b/object-file.c
index 77a3217fd0e..b0dea96906e 100644
--- a/object-file.c
+++ b/object-file.c
@@ -1878,13 +1878,13 @@ static void close_loose_object(int fd)
die_errno(_("error when closing loose object file"));
}
-/* Size of directory component, including the ending '/' */
+/* Size of directory component, excluding the ending '/' */
static inline int directory_size(const char *filename)
{
const char *s = strrchr(filename, '/');
if (!s)
return 0;
- return s - filename + 1;
+ return s - filename;
}
/*
@@ -1901,7 +1901,7 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename,
strbuf_reset(tmp);
strbuf_add(tmp, filename, dirlen);
- strbuf_addstr(tmp, "tmp_obj_XXXXXX");
+ strbuf_addstr(tmp, "/tmp_obj_XXXXXX");
fd = git_mkstemp_mode(tmp->buf, 0444);
do {
if (fd >= 0 || !dirlen || errno != ENOENT)
@@ -1913,7 +1913,7 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename,
* scratch.
*/
strbuf_reset(tmp);
- strbuf_add(tmp, filename, dirlen - 1);
+ strbuf_add(tmp, filename, dirlen);
if (mkdir(tmp->buf, 0777) && errno != EEXIST)
break;
if (adjust_shared_perm(tmp->buf))
@@ -2100,7 +2100,7 @@ int write_stream_object_file(struct input_stream *in_stream, size_t len,
dirlen = directory_size(filename.buf);
if (dirlen) {
struct strbuf dir = STRBUF_INIT;
- strbuf_add(&dir, filename.buf, dirlen - 1);
+ strbuf_add(&dir, filename.buf, dirlen);
if (mkdir_in_gitdir(dir.buf) && errno != EEXIST) {
ret = error_errno(_("unable to create directory %s"), dir.buf);On my platform (linux) it's not needed either way, a "mkdir foo" works as well as "mkdir foo/", but maybe some oS's have trouble with it.
Re: [PATCH v7 4/5] object-file.c: add "write_stream_object_file()" to support read in stream
Quoted 30 lines · Ævar Arnfjörð Bjarmason
Quoted 14 lines · Han Xin
(quoted diff) +14
-0+ /* Then the data itself.. */ + do { + unsigned char *in0 = stream.next_in; + if (!stream.avail_in) { + const void *in = in_stream->read(in_stream, &stream.avail_in); + stream.next_in = (void *)in; + in0 = (unsigned char *)in; + /* All data has been read. */ + if (len + hdrlen == stream.total_in + stream.avail_in) + flush = Z_FINISH; + } + ret = git_deflate(&stream, flush); + the_hash_algo->update_fn(&c, in0, stream.next_in - in0); + if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)Review
Quoted 10 lines
(quoted diff) +10
-0+ die(_("unable to write loose object file")); + stream.next_out = compressed; + stream.avail_out = sizeof(compressed); + } while (ret == Z_OK || ret == Z_BUF_ERROR); + + if (ret != Z_STREAM_END) + die(_("unable to deflate new object streamingly (%d)"), ret); + ret = git_deflate_end_gently(&stream); + if (ret != Z_OK) + die(_("deflateEnd on object streamingly failed (%d)"), ret);nit: let's say "unable to stream deflate new object" or something, and not use the confusing (invented?) word "streamingly".
Quoted 8 lines
Review
Quoted 5 lines
(quoted diff) +5
-0+ the_hash_algo->final_oid_fn(¶no_oid, &c); + + close_loose_object(fd); + + oidcpy(oid, ¶no_oid);I see there's still quite a bit of duplication between this and
write_loose_object(), but maybe it's not easy to factor out.
For what it's worth I tried to do that and the result doesn't really seem worth it. I.e. something like the below. The inner loop of the do/while looks like it could get a similar treatment, but likewise doesn't seem worth the effort.
object-file.c +52 -27
@@ -1957,6 +1957,46 @@ static void setup_stream_and_header(git_zstream *stream, the_hash_algo->update_fn (c, hdr, hdrlen); } +static int start_loose_object_common (struct strbuf *tmp_file, +const char *filename,unsigned flags, +git_zstream *stream, +unsigned char *buf,size_t buflen, +git_hash_ctx *c, +enum object_type type,size_t len, +char *hdr,int *hdrlen) +{ +int fd; + + fd =create_tmpfile (tmp_file, filename, flags); +if (fd <0 ) +return -1 ; + +if (type !=OBJ_NONE ) + *hdrlen =format_object_header (hdr, *hdrlen, type, len); + +/* Set it up and write header */ +setup_stream_and_header (stream, buf, buflen, c, hdr, *hdrlen); + +return fd; + +} + +static void end_loose_object_common (int ret,git_hash_ctx *c, +git_zstream *stream, +struct object_id *parano_oid, +const struct object_id *expected_oid, +const char *zstream_end_fmt, +const char *z_ok_fmt) +{ +if (ret !=Z_STREAM_END ) +die (_ (zstream_end_fmt), ret, expected_oid); + ret =git_deflate_end_gently (stream); +if (ret !=Z_OK ) +die (_ (z_ok_fmt), ret, expected_oid); + the_hash_algo->final_oid_fn (parano_oid, c); +} + +static int write_loose_object (const struct object_id *oid,char *hdr,int hdrlen,const void *buf,unsigned long len,time_t mtime,unsigned flags)
@@ -1970,15 +2010,12 @@ static int write_loose_object(const struct object_id *oid, char *hdr,static struct strbuf filename =STRBUF_INIT ;loose_object_path (the_repository, &filename, oid); - - fd =create_tmpfile (&tmp_file, filename.buf, flags); + fd =start_loose_object_common (&tmp_file, filename.buf, flags, + &stream, compressed,sizeof (compressed), + &c,OBJ_NONE ,0 , hdr, &hdrlen);if (fd <0 )return -1 ; -/* Set it up and write header */ -setup_stream_and_header (&stream, compressed,sizeof (compressed), - &c, hdr, hdrlen); -/* Then the data itself.. */ stream.next_in = (void *)buf; stream.avail_in = len;
@@ -1992,14 +2029,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr, stream.avail_out =sizeof (compressed); }while (ret ==Z_OK ); -if (ret !=Z_STREAM_END ) -die (_ ("unable to deflate new object %s (%d)" ),oid_to_hex (oid), - ret); - ret =git_deflate_end_gently (&stream); -if (ret !=Z_OK ) -die (_ ("deflateEnd on object %s failed (%d)" ),oid_to_hex (oid), - ret); - the_hash_algo->final_oid_fn (¶no_oid, &c); +end_loose_object_common (ret, &c, &stream, ¶no_oid, oid, +N_ ("unable to deflate new object %s (%d)" ), +N_ ("deflateEnd on object %s failed (%d)" ));if (!oideq (oid, ¶no_oid))die (_ ("confused by unstable object source data for %s" ),oid_to_hex (oid));
@@ -2049,16 +2081,12 @@ int write_stream_object_file(struct input_stream *in_stream, size_t len,/* When oid is not determined, save tmp file to odb path. */ strbuf_addf (&filename,"%s/" ,get_object_directory ()); - fd =create_tmpfile (&tmp_file, filename.buf, flags); + fd =start_loose_object_common (&tmp_file, filename.buf, flags, + &stream, compressed,sizeof (compressed), + &c, type, len, hdr, &hdrlen);if (fd <0 )return -1 ; - hdrlen =format_object_header (hdr, hdrlen, type, len); - -/* Set it up and write header */ -setup_stream_and_header (&stream, compressed,sizeof (compressed), - &c, hdr, hdrlen); -/* Then the data itself.. */ do { unsigned char *in0 = stream.next_in;
@@ -2078,12 +2106,9 @@ int write_stream_object_file(struct input_stream *in_stream, size_t len, stream.avail_out =sizeof (compressed); }while (ret ==Z_OK || ret ==Z_BUF_ERROR ); -if (ret !=Z_STREAM_END ) -die (_ ("unable to deflate new object streamingly (%d)" ), ret); - ret =git_deflate_end_gently (&stream); -if (ret !=Z_OK ) -die (_ ("deflateEnd on object streamingly failed (%d)" ), ret); - the_hash_algo->final_oid_fn (¶no_oid, &c); +end_loose_object_common (ret, &c, &stream, ¶no_oid,NULL , +N_ ("unable to deflate new object streamingly (%d)" ), +N_ ("deflateEnd on object streamingly failed (%d)" ));close_loose_object (fd);
PATCH v7 5/5 unpack-objects: unpack_non_delta_entry() read data in a stream
In reply to Han XinFrom: Han Xin <[email protected]>
We used to call "get_data()" in "unpack_non_delta_entry()" to read the entire contents of a blob object, no matter how big it is. This implementation may consume all the memory and cause OOM.
By implementing a zstream version of input_stream interface, we can use a small fixed buffer for "unpack_non_delta_entry()".
However, unpack non-delta objects from a stream instead of from an entrie buffer will have 10% performance penalty. Therefore, only unpack object larger than the "core.BigFileStreamingThreshold" in zstream. See the following benchmarks:
hyperfine \
--setup \
'if ! test -d scalar.git; then git clone --bare https://github.com/microsoft/scalar.git; cp scalar.git/objects/pack/*.pack small.pack; fi' \
--prepare 'rm -rf dest.git && git init --bare dest.git'
Summary
'./git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'origin/master'
1.01 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'HEAD~1' 1.01 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'HEAD~0' 1.03 ± 0.10 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'origin/master' 1.02 ± 0.07 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'HEAD~0' 1.10 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'HEAD~1'
--- Documentation/config/core.txt | 11 +++++ builtin/unpack-objects.c | 73 ++++++++++++++++++++++++++++- cache.h | 1 + config.c | 5 ++ environment.c | 1 + t/t5590-unpack-non-delta-objects.sh | 36 +++++++++++++- 6 files changed, 125 insertions(+), 2 deletions(-)
Documentation/config/core.txt +11 -0
@@ -424,6 +424,17 @@ be delta compressed, but larger binary media files won't be. + Common unit suffixes of 'k', 'm', or 'g' are supported. +core.bigFileStreamingThreshold:: + Files larger than this will be streamed out to a temporary + object file while being hashed, which will when be renamed + in-place to a loose object, particularly if the + `core.bigFileThreshold' setting dictates that they're always + written out as loose objects. ++ +Default is 128 MiB on all platforms. ++ +Common unit suffixes of 'k', 'm', or 'g' are supported. + core.excludesFile:: Specifies the pathname to the file that contains patterns to describe paths that are not meant to be tracked, in addition
builtin/unpack-objects.c +72 -1
83 lines not on this page · raw message
cache.h +1 -0
@@ -974,6 +974,7 @@ extern size_t packed_git_window_size;extern size_t packed_git_limit;extern size_t delta_base_cache_limit;extern unsigned long big_file_threshold; +extern unsigned long big_file_streaming_threshold;extern unsigned long pack_size_limit_cfg;/*
config.c +5 -0
@@ -1408,6 +1408,11 @@ static int git_default_core_config(const char *var, const char *value, void *cb)return 0 ; } +if (!strcmp (var,"core.bigfilestreamingthreshold" )) { + big_file_streaming_threshold =git_config_ulong (var, value); +return 0 ; + } +if (!strcmp (var,"core.packedgitlimit" )) { packed_git_limit =git_config_ulong (var, value);return 0 ;
environment.c +1 -0
@@ -47,6 +47,7 @@ size_t packed_git_window_size = DEFAULT_PACKED_GIT_WINDOW_SIZE;size_t packed_git_limit =DEFAULT_PACKED_GIT_LIMIT ;size_t delta_base_cache_limit =96 *1024 *1024 ;unsigned long big_file_threshold =512 *1024 *1024 ; +unsigned long big_file_streaming_threshold =128 *1024 *1024 ;int pager_use_color =1 ;const char *editor_program;const char *askpass_program;
t/t5590-unpack-non-delta-objects.sh +35 -1
@@ -13,6 +13,11 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME prepare_dest () { test_when_finished "rm -rf dest.git" && git init --bare dest.git + if test -n "$1" + then + git -C dest.git config core.bigFileStreamingThreshold $1 + git -C dest.git config core.bigFileThreshold $1 + fi } test_expect_success "setup repo with big blobs (1.5 MB)" '
@@ -33,7 +38,7 @@ test_expect_success 'setup env: GIT_ALLOC_LIMIT to 1MB' ' ' test_expect_success 'fail to unpack-objects: cannot allocate' ' - prepare_dest && + prepare_dest 2m && test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err && grep "fatal: attempting to allocate" err && (
@@ -44,6 +49,35 @@ test_expect_success 'fail to unpack-objects: cannot allocate' ' ! test_cmp expect actual ' +test_expect_success 'unpack big object in stream' ' + prepare_dest 1m && + mkdir -p dest.git/objects/05 && + git -C dest.git unpack-objects <test-$PACK.pack && + git -C dest.git fsck && + ( + cd dest.git && + find objects/?? -type f | sort + ) >actual && + test_cmp expect actual +' + +test_expect_success 'unpack big object in stream with existing oids' ' + prepare_dest 1m && + git -C dest.git index-pack --stdin <test-$PACK.pack && + ( + cd dest.git && + find objects/?? -type f | sort + ) >actual && + test_must_be_empty actual && + git -C dest.git unpack-objects <test-$PACK.pack && + git -C dest.git fsck && + ( + cd dest.git && + find objects/?? -type f | sort + ) >actual && + test_must_be_empty actual +' + test_expect_success 'unpack-objects dry-run' ' prepare_dest && git -C dest.git unpack-objects -n <test-$PACK.pack &&
Signature · 1 line
2.34.1.52.g80008efde6.agit.6.5.6
Re: [PATCH v7 5/5] unpack-objects: unpack_non_delta_entry() read data in a stream
Quoted 101 lines · Han Xin
From: Han Xin <[email protected]>
We used to call "
get_data()" in "unpack_non_delta_entry()" to read the entire contents of a blob object, no matter how big it is. This implementation may consume all the memory and cause OOM.By implementing a zstream version of
input_streaminterface, we can use a small fixed buffer for "unpack_non_delta_entry()".However, unpack non-delta objects from a stream instead of from an entrie buffer will have 10% performance penalty. Therefore, only unpack object larger than the "core.BigFileStreamingThreshold" in zstream. See the following benchmarks:
hyperfine \
--setup \
'if ! test -d scalar.git; then git clone --bare https://github.com/microsoft/scalar.git; cp scalar.git/objects/pack/*.pack small.pack; fi' \
--prepare 'rm -rf dest.git && git init --bare dest.git'Summary
'./git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'origin/master'
1.01 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'HEAD~1' 1.01 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'HEAD~0' 1.03 ± 0.10 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'origin/master' 1.02 ± 0.07 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'HEAD~0' 1.10 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'HEAD~1'Helped-by Derrick Stolee <[email protected]>Helped-by Jiang Xin <[email protected]>Signed-off-by Han Xin <[email protected]>--- Documentation/config/core.txt | 11 +++++ builtin/unpack-objects.c | 73 ++++++++++++++++++++++++++++- cache.h | 1 + config.c | 5 ++ environment.c | 1 + t/t5590-unpack-non-delta-objects.sh | 36 +++++++++++++- 6 files changed, 125 insertions(+), 2 deletions(-)Documentation/config/core.txt +11
-0@@ -424,6 +424,17 @@ be delta compressed, but larger binary media files won't be. + Common unit suffixes of 'k', 'm', or 'g' are supported. +core.bigFileStreamingThreshold:: + Files larger than this will be streamed out to a temporary + object file while being hashed, which will when be renamed + in-place to a loose object, particularly if the + `core.bigFileThreshold' setting dictates that they're always + written out as loose objects. ++ +Default is 128 MiB on all platforms. ++ +Common unit suffixes of 'k', 'm', or 'g' are supported. + core.excludesFile:: Specifies the pathname to the file that contains patterns to describe paths that are not meant to be tracked, in additionbuiltin/unpack-objects.c +31
-0@@ -331,11 +331,82 @@ static void added_object(unsigned nr, enum object_type type, } } +struct input_zstream_data { +git_zstream *zstream; +unsigned char buf[8192 ]; +int status; +}; + +static const void *feed_input_zstream (struct input_stream *in_stream, +unsigned long *readlen) +{ +struct input_zstream_data *data = in_stream->data; +git_zstream *zstream = data->zstream; +void *in =fill (1 ); + +if (!len || data->status ==Z_STREAM_END ) { + *readlen =0 ; +return NULL ; + } + + zstream->next_out = data->buf; + zstream->avail_out =sizeof (data->buf); + zstream->next_in = in; + zstream->avail_in = len; + + data->status =git_inflate (zstream,0 ); +use (len - zstream->avail_in); + *readlen =sizeof (data->buf) - zstream->avail_out; + +return data->buf; +} + +static void write_stream_blob (unsigned nr,size_t size)
builtin/unpack-objects.c static void added_object(unsigned nr, enum object_type type, · line 374
Quoted 10 lines
builtin/unpack-objects.c +10
-0+{ +git_zstream zstream; +struct input_zstream_data data; +struct input_stream in_stream = { + .read = feed_input_zstream, + .data = &data, + }; + +memset (&zstream,0 ,sizeof (zstream)); +memset (&data,0 ,sizeof (data));
nit/style: both of these memset can be replaced by "{ 0 }", e.g. "git_zstream zstream = { 0 }".
builtin/unpack-objects.c static void added_object(unsigned nr, enum object_type type, · line 379
Quoted 5 lines
builtin/unpack-objects.c +5
-0+ data.zstream = &zstream; +git_inflate_init (&zstream); + +if (write_stream_object_file (&in_stream, size,OBJ_BLOB ,0 ,0 , + &obj_list[nr].oid))
So at the end of this series we never pass in anything but blob here, mtime is always 0 etc. So there was no reason to create a factored out finalize_object_file_with_mtime() earlier in the series.
Well, I don't mind the finalize_object_file_with_mtime() exiting, but let's not pretend this is more generalized than it is. We're unlikely to ever want to do this for non-blobs.
This on top of this series (and my local WIP fixups as I'm reviewing this, so it won't cleanly apply, but the idea should be clear) makes this simpler:
diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c
index 2f8d34a2e47..a3a1d4b266f 100644
--- a/builtin/unpack-objects.c
+++ b/builtin/unpack-objects.c
@@ -375,8 +375,7 @@ static void write_stream_blob(unsigned nr, size_t size)
data.zstream = &zstream;
git_inflate_init(&zstream);
- if (write_stream_object_file(&in_stream, size, OBJ_BLOB, 0, 0,
- &obj_list[nr].oid))
+ if (write_stream_object_file(&in_stream, size, &obj_list[nr].oid))
die(_("failed to write object in stream"));
if (zstream.total_out != size || data.status != Z_STREAM_END)
diff --git a/object-file.c b/object-file.c
index 7fc2363cfa1..0572b34fc5a 100644
--- a/object-file.c
+++ b/object-file.c
@@ -2061,8 +2061,7 @@ static int freshen_packed_object(const struct object_id *oid)
}
int write_stream_object_file(struct input_stream *in_stream, size_t len,
- enum object_type type, time_t mtime,
- unsigned flags, struct object_id *oid)
+ struct object_id *oid)
{
int fd, ret, flush = 0;
unsigned char compressed[4096];
@@ -2081,9 +2080,9 @@ int write_stream_object_file(struct input_stream *in_stream, size_t len,
/* When oid is not determined, save tmp file to odb path. */
strbuf_addf(&filename, "%s/", get_object_directory());
- fd = start_loose_object_common(&tmp_file, filename.buf, flags,
+ fd = start_loose_object_common(&tmp_file, filename.buf, 0,
&stream, compressed, sizeof(compressed),
- &c, type, len, hdr, &hdrlen);
+ &c, OBJ_BLOB, len, hdr, &hdrlen);
if (fd < 0)
return -1;
@@ -2135,7 +2134,7 @@ int write_stream_object_file(struct input_stream *in_stream, size_t len,
strbuf_release(&dir);
}
- return finalize_object_file_with_mtime(tmp_file.buf, filename.buf, mtime, flags);
+ return finalize_object_file(tmp_file.buf, filename.buf);
}
int write_object_file_flags(const void *buf, unsigned long len,
diff --git a/object-store.h b/object-store.h
index 87d370d39ca..1362b58a4d3 100644
--- a/object-store.h
+++ b/object-store.h
@@ -257,8 +257,7 @@ int hash_write_object_file_literally(const void *buf, unsigned long len,
unsigned flags);
int write_stream_object_file(struct input_stream *in_stream, size_t len,
- enum object_type type, time_t mtime,
- unsigned flags, struct object_id *oid);
+ struct object_id *oid);
/*
* Add an object file to the in-memory object store, without writing itQuoted 18 lines
builtin/unpack-objects.c +1
-0+die (_ ("failed to write object in stream" ));environment.c +1
-0@@ -47,6 +47,7 @@ size_t packed_git_window_size = DEFAULT_PACKED_GIT_WINDOW_SIZE;size_t packed_git_limit =DEFAULT_PACKED_GIT_LIMIT ;size_t delta_base_cache_limit =96 *1024 *1024 ;unsigned long big_file_threshold =512 *1024 *1024 ; +unsigned long big_file_streaming_threshold =128 *1024 *1024 ;int pager_use_color =1 ;const char *editor_program;const char *askpass_program;t/t5590-unpack-non-delta-objects.sh +0
-0
t/t5590-unpack-non-delta-objects.sh export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME · line 20
Quoted 8 lines
t/t5590-unpack-non-delta-objects.sh +5
-0@@ -13,6 +13,11 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME prepare_dest () { test_when_finished "rm -rf dest.git" && git init --bare dest.git + if test -n "$1" + then + git -C dest.git config core.bigFileStreamingThreshold $1 + git -C dest.git config core.bigFileThreshold $1 + fi
All of this new code is missing "&&" to chain & test forfailures.
Re: [PATCH v7 5/5] unpack-objects: unpack_non_delta_entry() read data in a stream
In reply to Han XinQuoted 46 lines · Han Xin
From: Han Xin <[email protected]>
We used to call "
get_data()" in "unpack_non_delta_entry()" to read the entire contents of a blob object, no matter how big it is. This implementation may consume all the memory and cause OOM.By implementing a zstream version of
input_streaminterface, we can use a small fixed buffer for "unpack_non_delta_entry()".However, unpack non-delta objects from a stream instead of from an entrie buffer will have 10% performance penalty. Therefore, only unpack object larger than the "core.BigFileStreamingThreshold" in zstream. See the following benchmarks:
hyperfine \
--setup \
'if ! test -d scalar.git; then git clone --bare https://github.com/microsoft/scalar.git; cp scalar.git/objects/pack/*.pack small.pack; fi' \
--prepare 'rm -rf dest.git && git init --bare dest.git'Summary
'./git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'origin/master'
1.01 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'HEAD~1' 1.01 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=512m unpack-objects <small.pack' in 'HEAD~0' 1.03 ± 0.10 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'origin/master' 1.02 ± 0.07 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'HEAD~0' 1.10 ± 0.04 times faster than './git -C dest.git -c core.bigfilethreshold=16k unpack-objects <small.pack' in 'HEAD~1'Helped-by Derrick Stolee <[email protected]>Helped-by Jiang Xin <[email protected]>Signed-off-by Han Xin <[email protected]>--- Documentation/config/core.txt | 11 +++++ builtin/unpack-objects.c | 73 ++++++++++++++++++++++++++++- cache.h | 1 + config.c | 5 ++ environment.c | 1 + t/t5590-unpack-non-delta-objects.sh | 36 +++++++++++++- 6 files changed, 125 insertions(+), 2 deletions(-)Documentation/config/core.txt +0
-0
Documentation/config/core.txt be delta compressed, but larger binary media files won't be. · line 432
Quoted 9 lines
Documentation/config/core.txt +6
-0@@ -424,6 +424,17 @@ be delta compressed, but larger binary media files won't be. + Common unit suffixes of 'k', 'm', or 'g' are supported. +core.bigFileStreamingThreshold:: + Files larger than this will be streamed out to a temporary + object file while being hashed, which will when be renamed + in-place to a loose object, particularly if the + `core.bigFileThreshold' setting dictates that they're always + written out as loose objects.
Han Xin told me the reason to introduce another git config variable, but I feel it not good to introduce an application specific config variable as "core.XXX" and parsing it in "config.c".
So in patch v8, will still reuse the config variable "core.bigFileThreshold", and will introduce an application specific config variable, such as unpack.bigFileThreshold and parse the new config in "builtin/unpack-objects.c".
--
Jiang Xin
v8 cover unpack large blobs in stream
In reply to Han XinFrom: Han Xin <[email protected]>
Changes since v7:
* Use functions "assert_no_loose()" and "assert_no_pack()" to do tests instead of "find" sugguseted by Ævar Arnfjörð Bjarmason[1].
* "get_data()" now use the global "dry_run" and it will release the buf before returning.
* Add a new commit "object-file.c: remove the slash for directory_size()" sugguseted by Ævar Arnfjörð Bjarmason[2].
* Add "int is_finished" to "struct input_stream" who will tell us if there is next buffer in the stream.
* Remove the config "core.bigFileStreamingThreshold" introduced in v5, and keep using "core.bigFileThreshold". Until now, the config variable has been used in the cases listed in "unpack-objects: unpack_non_delta_entry() read data in a stream", this new case belongs to the packfile category.
* Remove unnecessary explicit cast in "object-file API: add a format_object_header() function" sugguseted by René Scharfe[3].
1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/
3. https://lore.kernel.org/git/[email protected]/
Han Xin (5):
unpack-objects: low memory footprint for get_data() in dry_run mode object-file.c: refactor write_loose_object() to several steps object-file.c: remove the slash for directory_size() object-file.c: add "stream_loose_object()" to handle large object
unpack-objects: unpack_non_delta_entry() read data in a stream
Ævar Arnfjörð Bjarmason (1):
object-file API: add a format_object_header() function
builtin/index-pack.c | 3 +- builtin/unpack-objects.c | 110 +++++++++++-- bulk-checkin.c | 4 +- cache.h | 21 +++ http-push.c | 2 +- object-file.c | 272 ++++++++++++++++++++++++++------ object-store.h | 9 ++ t/t5329-unpack-large-objects.sh | 69 ++++++++ 8 files changed, 422 insertions(+), 68 deletions(-) create mode 100755 t/t5329-unpack-large-objects.sh
Range-diff against v7:
1: a8f232f553 < -: ---------- unpack-objects.c: add dry_run mode for get_data()
-: ---------- > 1: bd34da5816 unpack-objects: low memory footprint for get_data() in dry_run mode
3: a571b8f16c ! 2: f9a4365a7d object-file.c: refactor write_loose_object() to reuse in stream version
@@ Metadata
Author: Han Xin <[email protected]>
## Commit message ##
- object-file.c: refactor write_loose_object() to reuse in stream version
+ object-file.c: refactor write_loose_object() to several steps
- We used to call "get_data()" in "unpack_non_delta_entry()" to read the
- entire contents of a blob object, no matter how big it is. This
- implementation may consume all the memory and cause OOM.
+ When writing a large blob using "write_loose_object()", we have to pass
+ a buffer with the whole content of the blob, and this behavior will
+ consume lots of memory and may cause OOM. We will introduce a stream
+ version function ("stream_loose_object()") in latter commit to resolve
+ this issue.
- This can be improved by feeding data to "stream_loose_object()" in
- stream instead of read into the whole buf.
+ Before introducing a stream vesion function for writing loose object,
+ do some refactoring on "write_loose_object()" to reuse code for both
+ versions.
- As this new method "stream_loose_object()" has many similarities with
- "write_loose_object()", we split up "write_loose_object()" into some
- steps:
- 1. Figuring out a path for the (temp) object file.
- 2. Creating the tempfile.
- 3. Setting up zlib and write header.
- 4. Write object data and handle errors.
- 5. Optionally, do someting after write, maybe force a loose object if
- "mtime".
+ Rewrite "write_loose_object()" as follows:
+
+ 1. Figure out a path for the (temp) object file. This step is only
+ used in "write_loose_object()".
+
+ 2. Move common steps for starting to write loose objects into a new
+ function "start_loose_object_common()".
+
+ 3. Compress data.
+
+ 4. Move common steps for ending zlib stream into a new funciton
+ "end_loose_object_common()".
+
+ 5. Close fd and finalize the object file.
Helped-by: Ævar Arnfjörð Bjarmason <[email protected]>
+ Helped-by: Jiang Xin <[email protected]>
Signed-off-by: Han Xin <[email protected]>
## object-file.c ##
@@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filenam
return fd;
}
-+static void setup_stream_and_header(git_zstream *stream,
-+ unsigned char *compressed,
-+ unsigned long compressed_size,
-+ git_hash_ctx *c,
-+ char *hdr,
-+ int hdrlen)
++static int start_loose_object_common(struct strbuf *tmp_file,
++ const char *filename, unsigned flags,
++ git_zstream *stream,
++ unsigned char *buf, size_t buflen,
++ git_hash_ctx *c,
++ enum object_type type, size_t len,
++ char *hdr, int hdrlen)
+{
-+ /* Set it up */
++ int fd;
++
++ fd = create_tmpfile(tmp_file, filename, flags);
++ if (fd < 0)
++ return -1;
++
++ /* Setup zlib stream for compression */
+ git_deflate_init(stream, zlib_compression_level);
-+ stream->next_out = compressed;
-+ stream->avail_out = compressed_size;
++ stream->next_out = buf;
++ stream->avail_out = buflen;
+ the_hash_algo->init_fn(c);
+
-+ /* First header.. */
++ /* Start to feed header to zlib stream */
+ stream->next_in = (unsigned char *)hdr;
+ stream->avail_in = hdrlen;
+ while (git_deflate(stream, 0) == Z_OK)
+ ; /* nothing */
+ the_hash_algo->update_fn(c, hdr, hdrlen);
++
++ return fd;
++}
++
++static void end_loose_object_common(int ret, git_hash_ctx *c,
++ git_zstream *stream,
++ struct object_id *parano_oid,
++ const struct object_id *expected_oid,
++ const char *die_msg1_fmt,
++ const char *die_msg2_fmt)
++{
++ if (ret != Z_STREAM_END)
++ die(_(die_msg1_fmt), ret, expected_oid);
++ ret = git_deflate_end_gently(stream);
++ if (ret != Z_OK)
++ die(_(die_msg2_fmt), ret, expected_oid);
++ the_hash_algo->final_oid_fn(parano_oid, c);
+}
+
static int write_loose_object(const struct object_id *oid, char *hdr,
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
- stream.next_out = compressed;
- stream.avail_out = sizeof(compressed);
- the_hash_algo->init_fn(&c);
-+ fd = create_tmpfile(&tmp_file, filename.buf, flags);
-+ if (fd < 0)
-+ return -1;
-
+-
- /* First header.. */
- stream.next_in = (unsigned char *)hdr;
- stream.avail_in = hdrlen;
- while (git_deflate(&stream, 0) == Z_OK)
- ; /* nothing */
- the_hash_algo->update_fn(&c, hdr, hdrlen);
-+ /* Set it up and write header */
-+ setup_stream_and_header(&stream, compressed, sizeof(compressed),
-+ &c, hdr, hdrlen);
++ /* Common steps for write_loose_object and stream_loose_object to
++ * start writing loose oject:
++ *
++ * - Create tmpfile for the loose object.
++ * - Setup zlib stream for compression.
++ * - Start to feed header to zlib stream.
++ */
++ fd = start_loose_object_common(&tmp_file, filename.buf, flags,
++ &stream, compressed, sizeof(compressed),
++ &c, OBJ_NONE, 0, hdr, hdrlen);
++ if (fd < 0)
++ return -1;
/* Then the data itself.. */
stream.next_in = (void *)buf;
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *hdr,
+ stream.avail_out = sizeof(compressed);
+ } while (ret == Z_OK);
+
+- if (ret != Z_STREAM_END)
+- die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
+- ret);
+- ret = git_deflate_end_gently(&stream);
+- if (ret != Z_OK)
+- die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
+- ret);
+- the_hash_algo->final_oid_fn(¶no_oid, &c);
++ /* Common steps for write_loose_object and stream_loose_object to
++ * end writing loose oject:
++ *
++ * - End the compression of zlib stream.
++ * - Get the calculated oid to "parano_oid".
++ */
++ end_loose_object_common(ret, &c, &stream, ¶no_oid, oid,
++ N_("unable to deflate new object %s (%d)"),
++ N_("deflateEnd on object %s failed (%d)"));
++
+ if (!oideq(oid, ¶no_oid))
+ die(_("confused by unstable object source data for %s"),
+ oid_to_hex(oid));
close_loose_object(fd);-: ---------- > 3: 18dd21122d object-file.c: remove the slash for directory_size()
-: ---------- > 4: 964715451b object-file.c: add "stream_loose_object()" to handle large object
-: ---------- > 5: 3f620466fe unpack-objects: unpack_non_delta_entry() read data in a stream
2: 0d2e0f3a00 ! 6: 8073a3888d object-file API: add a format_object_header() function
@@ builtin/index-pack.c: static void *unpack_entry_data(off_t offset, unsigned long
if (!is_delta_type(type)) {
- hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX,
- type_name(type),(uintmax_t)size) + 1;
-+ hdrlen = format_object_header(hdr, sizeof(hdr), type, (uintmax_t)size);
++ hdrlen = format_object_header(hdr, sizeof(hdr), type, size);
the_hash_algo->init_fn(&c);
the_hash_algo->update_fn(&c, hdr, hdrlen);
} else
@@ bulk-checkin.c: static int deflate_to_pack(struct bulk_checkin_state *state,
- header_len = xsnprintf((char *)obuf, sizeof(obuf), "%s %" PRIuMAX,
- type_name(type), (uintmax_t)size) + 1;
+ header_len = format_object_header((char *)obuf, sizeof(obuf),
-+ type, (uintmax_t)size);
++ type, size);
the_hash_algo->init_fn(&ctx);
the_hash_algo->update_fn(&ctx, obuf, header_len);
@@ http-push.c: static void start_put(struct transfer_request *request)
unpacked = read_object_file(&request->obj->oid, &type, &len);
- hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX , type_name(type), (uintmax_t)len) + 1;
-+ hdrlen = format_object_header(hdr, sizeof(hdr), type, (uintmax_t)len);
++ hdrlen = format_object_header(hdr, sizeof(hdr), type, len);
/* Set it up */
git_deflate_init(&stream, zlib_compression_level);
@@ object-file.c: static void write_object_file_prepare(const struct git_hash_algo
/* Sha1.. */
algo->init_fn(&c);
+@@ object-file.c: int stream_loose_object(struct input_stream *in_stream, size_t len,
+
+ /* Since oid is not determined, save tmp file to odb path. */
+ strbuf_addf(&filename, "%s/", get_object_directory());
+- hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX, type_name(OBJ_BLOB), len) + 1;
++ hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
+
+ /* Common steps for write_loose_object and stream_loose_object to
+ * start writing loose oject:
@@ object-file.c: int force_object_loose(const struct object_id *oid, time_t mtime)
buf = read_object(the_repository, oid, &type, &len);
if (!buf)
4: 1de06a8f5c < -: ---------- object-file.c: add "write_stream_object_file()" to support read in stream
5: e7b4e426ef < -: ---------- unpack-objects: unpack_non_delta_entry() read data in a streamSignature · 1 line
2.34.1.52.gc288e771b4.agit.6.5.6
PATCH v9 1/5 unpack-objects: low memory footprint for get_data() in dry_run mode
From: Han Xin <[email protected]>
As the name implies, "get_data(size)" will allocate and return a given size of memory. Allocating memory for a large blob object may cause the system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint for dry_run mode.
Because in dry_run mode, "get_data()" is only used to check the integrity of data, and the returned buffer is not used at all, we can allocate a smaller buffer and reuse it as zstream output. Therefore, in dry_run mode, "get_data()" will release the allocated buffer and return NULL instead of returning garbage data.
--- builtin/unpack-objects.c | 39 +++++++++++++++++++-------- t/t5328-unpack-large-objects.sh | 48 +++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 11 deletions(-) create mode 100755 t/t5328-unpack-large-objects.sh
builtin/unpack-objects.c +28 -11
@@ -96,15 +96,31 @@ static void use(int bytes)display_throughput (progress, consumed_bytes); } +/* +* Decompress zstream from stdin and return specific size of data. +* The caller is responsible to free the returned buffer. +* +* But for dry_run mode, "get_data()" is only used to check the +* integrity of data, and the returned buffer is not used at all. +* Therefore, in dry_run mode, "get_data()" will release the small +* allocated buffer which is reused to hold temporary zstream output +* and return NULL instead of returning garbage data. +*/ static void *get_data (unsigned long size) {git_zstream stream; -void *buf =xmallocz (size); +unsigned long bufsize; +void *buf;memset (&stream,0 ,sizeof (stream)); +if (dry_run && size >8192 ) + bufsize =8192 ; +else + bufsize = size; + buf =xmallocz (bufsize); stream.next_out = buf; - stream.avail_out = size; + stream.avail_out = bufsize; stream.next_in =fill (1 ); stream.avail_in = len;git_inflate_init (&stream);
@@ -124,8 +140,15 @@ static void *get_data(unsigned long size) } stream.next_in =fill (1 ); stream.avail_in = len; +if (dry_run) { +/* reuse the buffer in dry_run mode */ + stream.next_out = buf; + stream.avail_out = bufsize; + } }git_inflate_end (&stream); +if (dry_run) +FREE_AND_NULL (buf);return buf; }
@@ -325,10 +348,8 @@ static void unpack_non_delta_entry(enum object_type type, unsigned long size, {void *buf =get_data (size); -if (!dry_run && buf) +if (buf)write_object (nr, type, buf, size); -else -free (buf); } static int resolve_against_held(unsigned nr, const struct object_id *base,
@@ -358,10 +379,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,oidread (&base_oid,fill (the_hash_algo->rawsz));use (the_hash_algo->rawsz); delta_data =get_data (delta_size); -if (dry_run || !delta_data) { -free (delta_data); +if (!delta_data)return ; - }if (has_object_file (&base_oid)) ;/* Ok we have this one */ else if (resolve_against_held(nr, &base_oid,
@@ -397,10 +416,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,die ("offset value out of bound for delta base object" ); delta_data =get_data (delta_size); -if (dry_run || !delta_data) { -free (delta_data); +if (!delta_data)return ; - } lo =0 ; hi = nr;while (lo < hi) {
t/t5328-unpack-large-objects.sh new file +48 -0
48 lines not on this page · raw message
Signature · 1 line
2.34.1.52.gc288e771b4.agit.6.5.6
v9 cover unpack large blobs in stream
In reply to Han XinFrom: Han Xin <[email protected]>
Changes since v8:
* Rename "assert_no_loose ()" into "test_no_loose ()" in "t5329-unpack-large-objects.sh". Remove "assert_no_pack ()" and use "test_dir_is_empty" instead.
* Revert changes to "create_tmpfile()" and error handling is now in "start_loose_object_common()".
* Remove "finalize_object_file_with_mtime()" which seems to be an overkill for "write_loose_object()" now.
* Remove the commit "object-file.c: remove the slash for directory_size()", it can be in a separate patch if necessary.
Han Xin (4):
unpack-objects: low memory footprint for get_data() in dry_run mode object-file.c: refactor write_loose_object() to several steps object-file.c: add "stream_loose_object()" to handle large object
unpack-objects: unpack_non_delta_entry() read data in a stream
Ævar Arnfjörð Bjarmason (1):
object-file API: add a format_object_header() function
builtin/index-pack.c | 3 +- builtin/unpack-objects.c | 110 ++++++++++++++-- bulk-checkin.c | 4 +- cache.h | 21 +++ http-push.c | 2 +- object-file.c | 220 +++++++++++++++++++++++++++----- object-store.h | 9 ++ t/t5328-unpack-large-objects.sh | 65 ++++++++++ 8 files changed, 384 insertions(+), 50 deletions(-) create mode 100755 t/t5328-unpack-large-objects.sh
Range-diff against v8:
1: bd34da5816 ! 1: 6a6c11ba93 unpack-objects: low memory footprint for get_data() in dry_run mode
@@ builtin/unpack-objects.c: static void unpack_delta_entry(enum object_type type,
hi = nr;
while (lo < hi) {
- ## t/t5329-unpack-large-objects.sh (new) ##
+ ## t/t5328-unpack-large-objects.sh (new) ##
@@
+#!/bin/sh
+#
-+# Copyright (c) 2021 Han Xin
++# Copyright (c) 2022 Han Xin
+#
+
+test_description='git unpack-objects with large objects'
@@ t/t5329-unpack-large-objects.sh (new)
+ git init --bare dest.git
+}
+
-+assert_no_loose () {
++test_no_loose () {
+ glob=dest.git/objects/?? &&
+ echo "$glob" >expect &&
+ eval "echo $glob" >actual &&
+ test_cmp expect actual
+}
+
-+assert_no_pack () {
-+ rmdir dest.git/objects/pack
-+}
-+
+test_expect_success "create large objects (1.5 MB) and PACK" '
+ test-tool genrandom foo 1500000 >big-blob &&
+ test_commit --append foo big-blob &&
@@ t/t5329-unpack-large-objects.sh (new)
+test_expect_success 'unpack-objects works with memory limitation in dry-run mode' '
+ prepare_dest &&
+ git -C dest.git unpack-objects -n <test-$PACK.pack &&
-+ assert_no_loose &&
-+ assert_no_pack
++ test_no_loose &&
++ test_dir_is_empty dest.git/objects/pack
+'
+
+test_done
2: f9a4365a7d ! 2: bab9e0402f object-file.c: refactor write_loose_object() to several steps
@@ Commit message
Signed-off-by: Han Xin <[email protected]>
## object-file.c ##
-@@ object-file.c: static void write_object_file_prepare(const struct git_hash_algo *algo,
- algo->final_oid_fn(oid, &c);
- }
-
-+/*
-+ * Move the just written object with proper mtime into its final resting place.
-+ */
-+static int finalize_object_file_with_mtime(const char *tmpfile,
-+ const char *filename,
-+ time_t mtime,
-+ unsigned flags)
-+{
-+ struct utimbuf utb;
-+
-+ if (mtime) {
-+ utb.actime = mtime;
-+ utb.modtime = mtime;
-+ if (utime(tmpfile, &utb) < 0 && !(flags & HASH_SILENT))
-+ warning_errno(_("failed utime() on %s"), tmpfile);
-+ }
-+ return finalize_object_file(tmpfile, filename);
-+}
-+
- /*
- * Move the just written object into its final resting place.
- */
-@@ object-file.c: static inline int directory_size(const char *filename)
- * We want to avoid cross-directory filename renames, because those
- * can have problems on various filesystems (FAT, NFS, Coda).
- */
--static int create_tmpfile(struct strbuf *tmp, const char *filename)
-+static int create_tmpfile(struct strbuf *tmp, const char *filename,
-+ unsigned flags)
- {
- int fd, dirlen = directory_size(filename);
-
-@@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filename)
- strbuf_add(tmp, filename, dirlen);
- strbuf_addstr(tmp, "tmp_obj_XXXXXX");
- fd = git_mkstemp_mode(tmp->buf, 0444);
-- if (fd < 0 && dirlen && errno == ENOENT) {
-+ do {
-+ if (fd >= 0 || !dirlen || errno != ENOENT)
-+ break;
- /*
- * Make sure the directory exists; note that the contents
- * of the buffer are undefined after mkstemp returns an
@@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filename)
- strbuf_reset(tmp);
- strbuf_add(tmp, filename, dirlen - 1);
- if (mkdir(tmp->buf, 0777) && errno != EEXIST)
-- return -1;
-+ break;
- if (adjust_shared_perm(tmp->buf))
-- return -1;
-+ break;
-
- /* Try again */
- strbuf_addstr(tmp, "/tmp_obj_XXXXXX");
- fd = git_mkstemp_mode(tmp->buf, 0444);
-+ } while (0);
-+
-+ if (fd < 0 && !(flags & HASH_SILENT)) {
-+ if (errno == EACCES)
-+ return error(_("insufficient permission for adding an "
-+ "object to repository database %s"),
-+ get_object_directory());
-+ else
-+ return error_errno(_("unable to create temporary file"));
- }
-+
return fd;
}
@@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filenam
+ git_zstream *stream,
+ unsigned char *buf, size_t buflen,
+ git_hash_ctx *c,
-+ enum object_type type, size_t len,
+ char *hdr, int hdrlen)
+{
+ int fd;
+
-+ fd = create_tmpfile(tmp_file, filename, flags);
-+ if (fd < 0)
-+ return -1;
++ fd = create_tmpfile(tmp_file, filename);
++ if (fd < 0) {
++ if (flags & HASH_SILENT)
++ return -1;
++ else if (errno == EACCES)
++ return error(_("insufficient permission for adding "
++ "an object to repository database %s"),
++ get_object_directory());
++ else
++ return error_errno(
++ _("unable to create temporary file"));
++ }
+
+ /* Setup zlib stream for compression */
+ git_deflate_init(stream, zlib_compression_level);
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
+ */
+ fd = start_loose_object_common(&tmp_file, filename.buf, flags,
+ &stream, compressed, sizeof(compressed),
-+ &c, OBJ_NONE, 0, hdr, hdrlen);
++ &c, hdr, hdrlen);
+ if (fd < 0)
+ return -1;
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
if (!oideq(oid, ¶no_oid))
die(_("confused by unstable object source data for %s"),
oid_to_hex(oid));
-
- close_loose_object(fd);
-
-- if (mtime) {
-- struct utimbuf utb;
-- utb.actime = mtime;
-- utb.modtime = mtime;
-- if (utime(tmp_file.buf, &utb) < 0 &&
-- !(flags & HASH_SILENT))
-- warning_errno(_("failed utime() on %s"), tmp_file.buf);
-- }
--
-- return finalize_object_file(tmp_file.buf, filename.buf);
-+ return finalize_object_file_with_mtime(tmp_file.buf, filename.buf,
-+ mtime, flags);
- }
-
- static int freshen_loose_object(const struct object_id *oid)
3: 18dd21122d < -: ---------- object-file.c: remove the slash for directory_size()
4: 964715451b ! 3: dd13614985 object-file.c: add "stream_loose_object()" to handle large object
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
+ */
+ fd = start_loose_object_common(&tmp_file, filename.buf, 0,
+ &stream, compressed, sizeof(compressed),
-+ &c, OBJ_BLOB, len, hdr, hdrlen);
++ &c, hdr, hdrlen);
+ if (fd < 0) {
+ err = -1;
+ goto cleanup;
5: 3f620466fe ! 4: cd84e27b08 unpack-objects: unpack_non_delta_entry() read data in a stream
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
write_object(nr, type, buf, size);
}
- ## t/t5329-unpack-large-objects.sh ##
-@@ t/t5329-unpack-large-objects.sh: test_description='git unpack-objects with large objects'
+ ## t/t5328-unpack-large-objects.sh ##
+@@ t/t5328-unpack-large-objects.sh: test_description='git unpack-objects with large objects'
prepare_dest () {
test_when_finished "rm -rf dest.git" &&
@@ t/t5329-unpack-large-objects.sh: test_description='git unpack-objects with large
+ fi
}
- assert_no_loose () {
-@@ t/t5329-unpack-large-objects.sh: test_expect_success 'set memory limitation to 1MB' '
+ test_no_loose () {
+@@ t/t5328-unpack-large-objects.sh: test_expect_success 'set memory limitation to 1MB' '
'
test_expect_success 'unpack-objects failed under memory limitation' '
@@ t/t5329-unpack-large-objects.sh: test_expect_success 'set memory limitation to 1
- prepare_dest &&
+ prepare_dest 2m &&
git -C dest.git unpack-objects -n <test-$PACK.pack &&
- assert_no_loose &&
- assert_no_pack
+ test_no_loose &&
+ test_dir_is_empty dest.git/objects/pack
'
+test_expect_success 'unpack big object in stream' '
+ prepare_dest 1m &&
+ git -C dest.git unpack-objects <test-$PACK.pack &&
-+ assert_no_pack
++ test_dir_is_empty dest.git/objects/pack
+'
+
+test_expect_success 'do not unpack existing large objects' '
+ prepare_dest 1m &&
+ git -C dest.git index-pack --stdin <test-$PACK.pack &&
+ git -C dest.git unpack-objects <test-$PACK.pack &&
-+ assert_no_loose
++ test_no_loose
+'
+
test_done6: 8073a3888d = 5: 59f0ad95c7 object-file API: add a format_object_header() function
Signature · 1 line
2.34.1.52.gc288e771b4.agit.6.5.6
Re: [PATCH v9 0/5] unpack large blobs in stream
Quoted 16 lines · Han Xin
From: Han Xin <[email protected]>
Changes since v8:
* Rename "assert_no_loose()" into "test_no_loose()" in "t5329-unpack-large-objects.sh". Remove "assert_no_pack()" and use "test_dir_is_empty" instead.* Revert changes to "
create_tmpfile()" and error handling is now in "start_loose_object_common()".* Remove "
finalize_object_file_with_mtime()" which seems to be an overkill for "write_loose_object()" now.* Remove the commit "
object-file.c: remove the slash fordirectory_size()", it can be in a separate patch if necessary.
Quoted 8 lines
Han Xin (4):
unpack-objects: low memory footprint forget_data()indry_runmodeobject-file.c: refactorwrite_loose_object()to several stepsobject-file.c: add "stream_loose_object()" to handle large object
unpack-objects:unpack_non_delta_entry()read data in a streamÆvar Arnfjörð Bjarmason (1):
object-file API: add aformat_object_header()function
I sent
https://lore.kernel.org/git/[email protected]/
today which suggests splitting out the 5/5 cleanup you'd integrated.
I then rebased these patches of yours on top of that, the result is here:
https://github.com/avar/git/tree/han-xin-avar/unpack-loose-object-streaming-9
The range-diff to your version is below. There's a few unrelated fixes/nits in it.
I think with/without basing this on top of my series above your patches here look good with the nits pointed out in the diff below addressed (and some don't need to be). I.e. the dependency on it is rather trivial, and the two could be split up.
What do you think is a good way to proceed? I could just submit the below as a proposed v10 if you'd like & agree...
1: 553a9377eb3 ! 1: 61fcfe7b840 unpack-objects: low memory footprint for get_data() in dry_run mode
@@ Commit message
unpack-objects: low memory footprint for get_data() in dry_run mode
As the name implies, "get_data(size)" will allocate and return a given
- size of memory. Allocating memory for a large blob object may cause the
+ amount of memory. Allocating memory for a large blob object may cause the
system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint for dry_run mode.
@@ Commit message
+ Signed-off-by: Ævar Arnfjörð Bjarmason <[email protected]>
## builtin/unpack-objects.c ##
@@ builtin/unpack-objects.c: static void use(int bytes)
@@ t/t5328-unpack-large-objects.sh (new)
+
+test_no_loose () {
+ glob=dest.git/objects/?? &&
-+ echo "$glob" >expect &&
-+ eval "echo $glob" >actual &&
++ echo $glob >expect &&
++ echo "$glob" >actual &&
+ test_cmp expect actual
+}
+
-: ----------- > 2: c6b0437db03 object-file.c: do fsync() and close() before post-write die() 2: 88c91affd61 ! 3: 77bcfe3da6f object-file.c: refactor write_loose_object() to several steps
@@ Commit message
When writing a large blob using "write_loose_object()", we have to pass a buffer with the whole content of the blob, and this behavior will consume lots of memory and may cause OOM. We will introduce a stream
- version function ("stream_loose_object()") in latter commit to resolve
+ version function ("stream_loose_object()") in later commit to resolve
this issue.
- Before introducing a stream vesion function for writing loose object,
- do some refactoring on "write_loose_object()" to reuse code for both
- versions.
+ Before introducing that streaming function, do some refactoring on
+ "write_loose_object()" to reuse code for both versions.
Rewrite "write_loose_object()" as follows:
@@ Commit message
3. Compress data.
- 4. Move common steps for ending zlib stream into a new funciton
+ 4. Move common steps for ending zlib stream into a new function
"end_loose_object_common()".
5. Close fd and finalize the object file.
@@ Commit message
+ Signed-off-by: Ævar Arnfjörð Bjarmason <[email protected]>
## object-file.c ##
@@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filename)
return fd;
}
++/**
++ * Common steps for loose object writers to start writing loose
++ * objects:
++ *
++ * - Create tmpfile for the loose object.
++ * - Setup zlib stream for compression.
++ * - Start to feed header to zlib stream.
++ *
++ * Returns a "fd", which should later be provided to
++ * end_loose_object_common().
++ */
+static int start_loose_object_common(struct strbuf *tmp_file,
+ const char *filename, unsigned flags,
+ git_zstream *stream,
@@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filenam
+ return fd;
+}
+
-+static void end_loose_object_common(int ret, git_hash_ctx *c,
++/**
++ * Common steps for loose object writers to end writing loose objects:
++ *
++ * - End the compression of zlib stream.
++ * - Get the calculated oid to "parano_oid".
++ * - fsync() and close() the "fd"
++ */
++static void end_loose_object_common(int fd, int ret, git_hash_ctx *c,
+ git_zstream *stream,
+ struct object_id *parano_oid,
+ const struct object_id *expected_oid,
@@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filenam
+ if (ret != Z_OK)
+ die(_(die_msg2_fmt), ret, expected_oid);
+ the_hash_algo->final_oid_fn(parano_oid, c);
++
++ /*
++ * We already did a write_buffer() to the "fd", let's fsync()
++ * and close().
++ *
++ * We might still die() on a subsequent sanity check, but
++ * let's not add to that confusion by not flushing any
++ * outstanding writes to disk first.
++ */
++ close_loose_object(fd);
+}
+
static int write_loose_object(const struct object_id *oid, char *hdr,
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
- while (git_deflate(&stream, 0) == Z_OK)
- ; /* nothing */
- the_hash_algo->update_fn(&c, hdr, hdrlen);
-+ /* Common steps for write_loose_object and stream_loose_object to
-+ * start writing loose oject:
-+ *
-+ * - Create tmpfile for the loose object.
-+ * - Setup zlib stream for compression.
-+ * - Start to feed header to zlib stream.
-+ */
+ fd = start_loose_object_common(&tmp_file, filename.buf, flags,
+ &stream, compressed, sizeof(compressed),
+ &c, hdr, hdrlen);
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
- die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
- ret);
- the_hash_algo->final_oid_fn(¶no_oid, &c);
-+ /* Common steps for write_loose_object and stream_loose_object to
-+ * end writing loose oject:
-+ *
-+ * - End the compression of zlib stream.
-+ * - Get the calculated oid to "parano_oid".
-+ */
-+ end_loose_object_common(ret, &c, &stream, ¶no_oid, oid,
+-
+- /*
+- * We already did a write_buffer() to the "fd", let's fsync()
+- * and close().
+- *
+- * We might still die() on a subsequent sanity check, but
+- * let's not add to that confusion by not flushing any
+- * outstanding writes to disk first.
+- */
+- close_loose_object(fd);
++ end_loose_object_common(fd, ret, &c, &stream, ¶no_oid, oid,
+ N_("unable to deflate new object %s (%d)"),
+ N_("deflateEnd on object %s failed (%d)"));
-+
+
if (!oideq(oid, ¶no_oid))
die(_("confused by unstable object source data for %s"),
- oid_to_hex(oid));
3: 054a00ed21d ! 4: 71c10e734d1 object-file.c: add "stream_loose_object()" to handle large object
@@ Commit message
Add a new function "stream_loose_object()", which is a stream version of "write_loose_object()" but with a low memory footprint. We will use this
- function to unpack large blob object in latter commit.
+ function to unpack large blob object in later commit.
Another difference with "write_loose_object()" is that we have no chance to run "write_object_file_prepare()" to calculate the oid in advance. In "write_loose_object()", we know the oid and we can write the temporary file in the same directory as the final object, but for an object with an undetermined oid, we don't know the exact directory for
- the object, so we have to save the temporary file in ".git/objects/"
- directory instead.
+ the object.
+
+ Still, we need to save the temporary file we're preparing
+ somewhere. We'll do that in the top-level ".git/objects/"
+ directory (or whatever "GIT_OBJECT_DIRECTORY" is set to). Once we've
+ streamed it we'll know the OID, and will move it to its canonical
+ path.
"freshen_packed_object()" or "freshen_loose_object()" will be called inside "stream_loose_object()" after obtaining the "oid".
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
+
+ /* Since oid is not determined, save tmp file to odb path. */
+ strbuf_addf(&filename, "%s/", get_object_directory());
-+ hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX, type_name(OBJ_BLOB), len) + 1;
++ hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
+
+ /* Common steps for write_loose_object and stream_loose_object to
+ * start writing loose oject:
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
+ * - End the compression of zlib stream.
+ * - Get the calculated oid.
+ */
-+ end_loose_object_common(ret, &c, &stream, oid, NULL,
++ end_loose_object_common(fd, ret, &c, &stream, oid, NULL,
+ N_("unable to stream deflate new object (%d)"),
+ N_("deflateEnd on stream object failed (%d)"));
+
-+ close_loose_object(fd);
-+
+ if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
+ unlink_or_warn(tmp_file.buf);
+ goto cleanup;
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
+}
+
int write_object_file_flags(const void *buf, unsigned long len,
- const char *type, struct object_id *oid,
+ enum object_type type, struct object_id *oid,
unsigned flags)
## object-store.h ##
@@ object-store.h: static inline int write_object_file(const void *buf, unsigned lo
+int stream_loose_object(struct input_stream *in_stream, size_t len,
+ struct object_id *oid);
-+
- int hash_object_file_literally(const void *buf, unsigned long len,
- const char *type, struct object_id *oid,
- unsigned flags);
+ int hash_write_object_file_literally(const void *buf, unsigned long len,
+ const char *type, struct object_id *oid,
+ unsigned flags);
-: ----------- > 5: 3c1d788d69d core doc: modernize core.bigFileThreshold documentation 4: 6bcba6bce66 ! 6: 8b83f6d6b83 unpack-objects: unpack_non_delta_entry() read data in a stream
@@ Metadata
Author: Han Xin <[email protected]>
## Commit message ##
- unpack-objects: unpack_non_delta_entry() read data in a stream
+ unpack-objects: use stream_loose_object() to unpack large objects
- We used to call "get_data()" in "unpack_non_delta_entry()" to read the
- entire contents of a blob object, no matter how big it is. This
- implementation may consume all the memory and cause OOM.
+ Make use of the stream_loose_object() function introduced in the
+ preceding commit to unpack large objects. Before this we'd need to
+ malloc() the size of the blob before unpacking it, which could cause
+ OOM with very large blobs.
- By implementing a zstream version of input_stream interface, we can use
- a small fixed buffer for "unpack_non_delta_entry()". However, unpack
- non-delta objects from a stream instead of from an entrie buffer will
- have 10% performance penalty.
+ We could use this new interface to unpack all blobs, but doing so
+ would result in a performance penalty of around 10%, as the below
+ "hyperfine" benchmark will show. We therefore limit this to files
+ larger than "core.bigFileThreshold":
$ hyperfine \
--setup \
@@ Commit message
-c core.bigFileThreshold=16k unpack-objects <small.pack' in 'HEAD~1'
- Therefore, only unpack objects larger than the "core.bigFileThreshold"
- in zstream. Until now, the config variable has been used in the
- following cases, and our new case belongs to the packfile category.
+ An earlier version of this patch introduced a new
+ "core.bigFileStreamingThreshold" instead of re-using the existing
+ "core.bigFileThreshold" variable[1]. As noted in a detailed overview
+ of its users in [2] using it has several different meanings.
- * Archive:
+ Still, we consider it good enough to simply re-use it. While it's
+ possible that someone might want to e.g. consider objects "small" for
+ the purposes of diffing but "big" for the purposes of writing them
+ such use-cases are probably too obscure to worry about. We can always
+ split up "core.bigFileThreshold" in the future if there's a need for
+ that.
- + archive.c: write_entry(): write large blob entries to archive in
- stream.
-
- * Loose objects:
-
- + object-file.c: index_fd(): when hashing large files in worktree,
- read files in a stream, and create one packfile per large blob if
- want to save files to git object store.
-
- + object-file.c: read_loose_object(): when checking loose objects
- using "git-fsck", do not read full content of large loose objects.
-
- * Packfile:
-
- + fast-import.c: parse_and_store_blob(): streaming large blob from
- foreign source to packfile.
-
- + index-pack.c: check_collison(): read and check large blob in stream.
-
- + index-pack.c: unpack_entry_data(): do not return the entire
- contents of the big blob from packfile, but uses a fixed buf to
- perform some integrity checks on the object.
-
- + pack-check.c: verify_packfile(): used by "git-fsck" and will call
- check_object_signature() to check large blob in pack with the
- streaming interface.
-
- + pack-objects.c: get_object_details(): set "no_try_delta" for large
- blobs when counting objects.
-
- + pack-objects.c: write_no_reuse_object(): streaming large blob to
- pack.
-
- + unpack-objects.c: unpack_non_delta_entry(): unpack large blob in
- stream from packfile.
-
- * Others:
-
- + diff.c: diff_populate_filespec(): treat large blob file as binary.
-
- + streaming.c: istream_source(): as a helper of "open_istream()" to
- select proper streaming interface to read large blob from packfile.
+ 1. https://lore.kernel.org/git/[email protected]/
+ 2. https://lore.kernel.org/git/[email protected]/
+ ## Documentation/config/core.txt ##
+@@ Documentation/config/core.txt: usage, at the slight expense of increased disk usage.
+ * Will be generally be streamed when written, which avoids excessive
+ memory usage, at the cost of some fixed overhead. Commands that make
+ use of this include linkgit:git-archive[1],
+-linkgit:git-fast-import[1], linkgit:git-index-pack[1] and +-linkgit:git-fsck[1].
++linkgit:git-fast-import[1], linkgit:git-index-pack[1], ++linkgit:git-unpack-objects[1] and linkgit:git-fsck[1].
+
+ core.excludesFile::
+ Specifies the pathname to the file that contains patterns to
+
## builtin/unpack-objects.c ##
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type type,
}
5: 1bfaf89ee0b < -: ----------- object-file API: add a format_object_header() function
Re: [PATCH v9 0/5] unpack large blobs in stream
Quoted 40 lines · Ævar Arnfjörð Bjarmason
Quoted 16 lines · Han Xin
From: Han Xin <[email protected]>
Changes since v8:
* Rename "assert_no_loose()" into "test_no_loose()" in "t5329-unpack-large-objects.sh". Remove "assert_no_pack()" and use "test_dir_is_empty" instead.* Revert changes to "
create_tmpfile()" and error handling is now in "start_loose_object_common()".* Remove "
finalize_object_file_with_mtime()" which seems to be an overkill for "write_loose_object()" now.* Remove the commit "
object-file.c: remove the slash fordirectory_size()", it can be in a separate patch if necessary.Quoted 8 lines
Han Xin (4):
unpack-objects: low memory footprint forget_data()indry_runmodeobject-file.c: refactorwrite_loose_object()to several stepsobject-file.c: add "stream_loose_object()" to handle large object
unpack-objects:unpack_non_delta_entry()read data in a streamÆvar Arnfjörð Bjarmason (1):
object-file API: add aformat_object_header()functionI sent
https://lore.kernel.org/git/[email protected]/
today which suggests splitting out the 5/5 cleanup you'd integrated.I then rebased these patches of yours on top of that, the result is here:
https://github.com/avar/git/tree/han-xin-avar/unpack-loose-object-streaming-9The range-diff to your version is below. There's a few unrelated fixes/nits in it.
Quoted 8 lines
I think with/without basing this on top of my series above your patches here look good with the nits pointed out in the diff below addressed (and some don't need to be). I.e. the dependency on it is rather trivial, and the two could be split up.
What do you think is a good way to proceed? I could just submit the below as a proposed v10 if you'd like & agree...
Yes, thanks for the suggestions, and I'm glad you're happy to do so.
Thanks.
-Han Xin
Quoted 16 lines
1:
553a9377eb3! 1:61fcfe7b840unpack-objects: low memory footprint forget_data()indry_runmode
@@ Commit message
unpack-objects: low memory footprint forget_data()indry_runmodeAs the name implies, "
get_data(size)" will allocate and return a given
- size of memory. Allocating memory for a large blob object may cause the
+ amount of memory. Allocating memory for a large blob object may cause the
system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint fordry_runmode.
@@ Commit messageSuggested-by Jiang Xin <[email protected]>Signed-off-by Han Xin <[email protected]>+ Signed-off-by: Ævar Arnfjörð Bjarmason <[email protected]>
Quoted 13 lines
##
builtin/unpack-objects.c##
@@builtin/unpack-objects.c: static void use(int bytes)
@@t/t5328-unpack-large-objects.sh(new)
+
+test_no_loose () {
+ glob=dest.git/objects/?? &&
-+ echo "$glob" >expect &&
-+ eval "echo $glob" >actual &&
++ echo $glob >expect &&
++ echo "$glob" >actual &&
+test_cmpexpect actual
+}
+
I have a small doubt with this, it works fine with dash, but not others like zsh. Wouldn't
it be better to do compatibility, or would it introduce other issues that I don't know?
Thanks.
-Han Xin
Re: [PATCH v9 0/5] unpack large blobs in stream
Quoted 46 lines · Han Xin
Quoted 45 lines · Ævar Arnfjörð Bjarmason
Quoted 16 lines · Han Xin
From: Han Xin <[email protected]>
Changes since v8:
* Rename "assert_no_loose()" into "test_no_loose()" in "t5329-unpack-large-objects.sh". Remove "assert_no_pack()" and use "test_dir_is_empty" instead.* Revert changes to "
create_tmpfile()" and error handling is now in "start_loose_object_common()".* Remove "
finalize_object_file_with_mtime()" which seems to be an overkill for "write_loose_object()" now.* Remove the commit "
object-file.c: remove the slash fordirectory_size()", it can be in a separate patch if necessary.Quoted 8 lines
Han Xin (4):
unpack-objects: low memory footprint forget_data()indry_runmodeobject-file.c: refactorwrite_loose_object()to several stepsobject-file.c: add "stream_loose_object()" to handle large object
unpack-objects:unpack_non_delta_entry()read data in a streamÆvar Arnfjörð Bjarmason (1):
object-file API: add aformat_object_header()functionI sent
https://lore.kernel.org/git/[email protected]/
today which suggests splitting out the 5/5 cleanup you'd integrated.I then rebased these patches of yours on top of that, the result is here:
https://github.com/avar/git/tree/han-xin-avar/unpack-loose-object-streaming-9The range-diff to your version is below. There's a few unrelated fixes/nits in it.
I think with/without basing this on top of my series above your patches here look good with the nits pointed out in the diff below addressed (and some don't need to be). I.e. the dependency on it is rather trivial, and the two could be split up.
Quoted 5 lines
Quoted 3 lines
What do you think is a good way to proceed? I could just submit the below as a proposed v10 if you'd like & agree...
Yes, thanks for the suggestions, and I'm glad you're happy to do so.
Willdo.
Quoted 30 lines
Quoted 16 lines
1:
553a9377eb3! 1:61fcfe7b840unpack-objects: low memory footprint forget_data()indry_runmode
@@ Commit message
unpack-objects: low memory footprint forget_data()indry_runmodeAs the name implies, "
get_data(size)" will allocate and return a given
- size of memory. Allocating memory for a large blob object may cause the
+ amount of memory. Allocating memory for a large blob object may cause the
system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint fordry_runmode.
@@ Commit messageSuggested-by Jiang Xin <[email protected]>Signed-off-by Han Xin <[email protected]>+ Signed-off-by: Ævar Arnfjörð Bjarmason <[email protected]>
Quoted 13 lines
##
builtin/unpack-objects.c##
@@builtin/unpack-objects.c: static void use(int bytes)
@@t/t5328-unpack-large-objects.sh(new)
+
+test_no_loose () {
+ glob=dest.git/objects/?? &&
-+ echo "$glob" >expect &&
-+ eval "echo $glob" >actual &&
++ echo $glob >expect &&
++ echo "$glob" >actual &&
+test_cmpexpect actual
+}
+
Quoted 4 lines
I have a small doubt with this, it works fine with dash, but not others like zsh. Wouldn't
it be better to do compatibility, or would it introduce other issues that I don't know?
Ah, I hadn't spotted that zsh issue. I don't think the test suite will run on it in general, but in any case I'll fix this.
There's a few other tests that do this just by piping "find" to "wc -l", it's probably better to just follow that pattern. I think the eval works, but I thought it was a bit unusual/stood out.
v10 cover unpack-objects: support streaming large objects to disk
In reply to Han XinThis is a v10 re-roll of Han Xin's series[1] to stream large objects to disk in "git unpack-objects". This v9 had integrated a proposed cleanup patch of mine, which is now a part of its own series, which this series now depends on: [2]. This v10 is sent with Han Xin's approval[3].
Changes since v9:
* Now based on [2]
* Small grammar/typo fixes in commit messages
* Replaced an echo/eval pattern in a test with a $(find ... | wc -l) comparison, which is a pattern we already use in another test for the same (or similar) assertion.
* I added a new 2/6 to do an fsync() before an oideq() assertion. I don't think it matters in practice, but allows 3/6 to be smaller by having that code-now-utility-function share more logic among its two callers.
* Changed inline comments in 3/6 to API docs where appropriate, the helper function now gets a "fd" per 2/6.
* 4/6 could use the format_object_header() function in the base topic, and now does so (instead of that conversion coming later in v9).
* A new 5/6 updates the core.bigFileThreshold documentation to account for 12 years of behavior changes we hadn't documented.
* The updated 6/6 now links to those docs, and I removed a very detailed accounting of all in-tree uses of core.bigFileThreshold from the commit message. I think linking to the summary docs should suffice, and for anyone digging in the future 5/6 links to the more detailed summary in the old patch.
More generally I've been heavily involved in the review for the past iterations, and I think barring any last minute nits in this v10 this topic should be ready to advance. As the above summary shows we're down to typo fixes, doc and test tweaks etc. at this point.
The core functionality being added here isn't changed in any meaningful way, and has had a lot of careful review already.
1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/
3. https://lore.kernel.org/git/CAO0brD2Pe0aKSiBphZS861gC=nZk+q2GtXDN4pPjAQnPdns3TA@mail.gmail.com/
Han Xin (4):
unpack-objects: low memory footprint for get_data() in dry_run mode object-file.c: refactor write_loose_object() to several steps object-file.c: add "stream_loose_object()" to handle large object
unpack-objects: use stream_loose_object() to unpack large objects
Ævar Arnfjörð Bjarmason (2):
object-file.c: do fsync() and close() before post-write die() core doc: modernize core.bigFileThreshold documentation
Documentation/config/core.txt | 33 +++-- builtin/unpack-objects.c | 110 ++++++++++++++-- object-file.c | 221 +++++++++++++++++++++++++++----- object-store.h | 8 ++ t/t5328-unpack-large-objects.sh | 62 +++++++++ 5 files changed, 381 insertions(+), 53 deletions(-) create mode 100755 t/t5328-unpack-large-objects.sh
Range-diff against v9:
1: 553a9377eb3 ! 1: e46eb75b98f unpack-objects: low memory footprint for get_data() in dry_run mode
@@ Commit message
unpack-objects: low memory footprint for get_data() in dry_run mode
As the name implies, "get_data(size)" will allocate and return a given
- size of memory. Allocating memory for a large blob object may cause the
+ amount of memory. Allocating memory for a large blob object may cause the
system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint for dry_run mode.
@@ Commit message
in dry_run mode, "get_data()" will release the allocated buffer and return NULL instead of returning garbage data.
+ The "find [...]objects/?? -type f | wc -l" test idiom being used here
+ is adapted from the same "find" use added to another test in
+ d9545c7f465 (fast-import: implement unpack limit, 2016-04-25).
+
+ Signed-off-by: Ævar Arnfjörð Bjarmason <[email protected]>
## builtin/unpack-objects.c ##
@@ builtin/unpack-objects.c: static void use(int bytes)
@@ t/t5328-unpack-large-objects.sh (new)
+}
+
+test_no_loose () {
-+ glob=dest.git/objects/?? &&
-+ echo "$glob" >expect &&
-+ eval "echo $glob" >actual &&
-+ test_cmp expect actual
++ test $(find dest.git/objects/?? -type f | wc -l) = 0
+}
+
+test_expect_success "create large objects (1.5 MB) and PACK" '
-: ----------- > 2: 48bf9090058 object-file.c: do fsync() and close() before post-write die() 2: 88c91affd61 ! 3: 0e33d2a6e35 object-file.c: refactor write_loose_object() to several steps
@@ Commit message
When writing a large blob using "write_loose_object()", we have to pass a buffer with the whole content of the blob, and this behavior will consume lots of memory and may cause OOM. We will introduce a stream
- version function ("stream_loose_object()") in latter commit to resolve
+ version function ("stream_loose_object()") in later commit to resolve
this issue.
- Before introducing a stream vesion function for writing loose object,
- do some refactoring on "write_loose_object()" to reuse code for both
- versions.
+ Before introducing that streaming function, do some refactoring on
+ "write_loose_object()" to reuse code for both versions.
Rewrite "write_loose_object()" as follows:
@@ Commit message
3. Compress data.
- 4. Move common steps for ending zlib stream into a new funciton
+ 4. Move common steps for ending zlib stream into a new function
"end_loose_object_common()".
5. Close fd and finalize the object file.
@@ Commit message
+ Signed-off-by: Ævar Arnfjörð Bjarmason <[email protected]>
## object-file.c ##
@@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filename)
return fd;
}
++/**
++ * Common steps for loose object writers to start writing loose
++ * objects:
++ *
++ * - Create tmpfile for the loose object.
++ * - Setup zlib stream for compression.
++ * - Start to feed header to zlib stream.
++ *
++ * Returns a "fd", which should later be provided to
++ * end_loose_object_common().
++ */
+static int start_loose_object_common(struct strbuf *tmp_file,
+ const char *filename, unsigned flags,
+ git_zstream *stream,
@@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filenam
+ return fd;
+}
+
-+static void end_loose_object_common(int ret, git_hash_ctx *c,
++/**
++ * Common steps for loose object writers to end writing loose objects:
++ *
++ * - End the compression of zlib stream.
++ * - Get the calculated oid to "parano_oid".
++ * - fsync() and close() the "fd"
++ */
++static void end_loose_object_common(int fd, int ret, git_hash_ctx *c,
+ git_zstream *stream,
+ struct object_id *parano_oid,
+ const struct object_id *expected_oid,
@@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filenam
+ if (ret != Z_OK)
+ die(_(die_msg2_fmt), ret, expected_oid);
+ the_hash_algo->final_oid_fn(parano_oid, c);
++
++ /*
++ * We already did a write_buffer() to the "fd", let's fsync()
++ * and close().
++ *
++ * We might still die() on a subsequent sanity check, but
++ * let's not add to that confusion by not flushing any
++ * outstanding writes to disk first.
++ */
++ close_loose_object(fd);
+}
+
static int write_loose_object(const struct object_id *oid, char *hdr,
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
- while (git_deflate(&stream, 0) == Z_OK)
- ; /* nothing */
- the_hash_algo->update_fn(&c, hdr, hdrlen);
-+ /* Common steps for write_loose_object and stream_loose_object to
-+ * start writing loose oject:
-+ *
-+ * - Create tmpfile for the loose object.
-+ * - Setup zlib stream for compression.
-+ * - Start to feed header to zlib stream.
-+ */
+ fd = start_loose_object_common(&tmp_file, filename.buf, flags,
+ &stream, compressed, sizeof(compressed),
+ &c, hdr, hdrlen);
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
- die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
- ret);
- the_hash_algo->final_oid_fn(¶no_oid, &c);
-+ /* Common steps for write_loose_object and stream_loose_object to
-+ * end writing loose oject:
-+ *
-+ * - End the compression of zlib stream.
-+ * - Get the calculated oid to "parano_oid".
-+ */
-+ end_loose_object_common(ret, &c, &stream, ¶no_oid, oid,
+-
+- /*
+- * We already did a write_buffer() to the "fd", let's fsync()
+- * and close().
+- *
+- * We might still die() on a subsequent sanity check, but
+- * let's not add to that confusion by not flushing any
+- * outstanding writes to disk first.
+- */
+- close_loose_object(fd);
++ end_loose_object_common(fd, ret, &c, &stream, ¶no_oid, oid,
+ N_("unable to deflate new object %s (%d)"),
+ N_("deflateEnd on object %s failed (%d)"));
-+
+
if (!oideq(oid, ¶no_oid))
die(_("confused by unstable object source data for %s"),
- oid_to_hex(oid));
3: 054a00ed21d ! 4: 9644df5c744 object-file.c: add "stream_loose_object()" to handle large object
@@ Commit message
Add a new function "stream_loose_object()", which is a stream version of "write_loose_object()" but with a low memory footprint. We will use this
- function to unpack large blob object in latter commit.
+ function to unpack large blob object in later commit.
Another difference with "write_loose_object()" is that we have no chance to run "write_object_file_prepare()" to calculate the oid in advance. In "write_loose_object()", we know the oid and we can write the temporary file in the same directory as the final object, but for an object with an undetermined oid, we don't know the exact directory for
- the object, so we have to save the temporary file in ".git/objects/"
- directory instead.
+ the object.
+
+ Still, we need to save the temporary file we're preparing
+ somewhere. We'll do that in the top-level ".git/objects/"
+ directory (or whatever "GIT_OBJECT_DIRECTORY" is set to). Once we've
+ streamed it we'll know the OID, and will move it to its canonical
+ path.
"freshen_packed_object()" or "freshen_loose_object()" will be called inside "stream_loose_object()" after obtaining the "oid".
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
+
+ /* Since oid is not determined, save tmp file to odb path. */
+ strbuf_addf(&filename, "%s/", get_object_directory());
-+ hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX, type_name(OBJ_BLOB), len) + 1;
++ hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
+
+ /* Common steps for write_loose_object and stream_loose_object to
+ * start writing loose oject:
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
+ * - End the compression of zlib stream.
+ * - Get the calculated oid.
+ */
-+ end_loose_object_common(ret, &c, &stream, oid, NULL,
++ end_loose_object_common(fd, ret, &c, &stream, oid, NULL,
+ N_("unable to stream deflate new object (%d)"),
+ N_("deflateEnd on stream object failed (%d)"));
+
-+ close_loose_object(fd);
-+
+ if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
+ unlink_or_warn(tmp_file.buf);
+ goto cleanup;
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
+}
+
int write_object_file_flags(const void *buf, unsigned long len,
- const char *type, struct object_id *oid,
+ enum object_type type, struct object_id *oid,
unsigned flags)
## object-store.h ##
@@ object-store.h: struct object_directory {
struct object_directory *, 1, fspathhash, fspatheq)
@@ object-store.h: static inline int write_object_file(const void *buf, unsigned long len,
- return write_object_file_flags(buf, len, type, oid, 0);
- }
-
+ int write_object_file_literally(const void *buf, unsigned long len,
+ const char *type, struct object_id *oid,
+ unsigned flags);
+int stream_loose_object(struct input_stream *in_stream, size_t len,
+ struct object_id *oid);
-+
- int hash_object_file_literally(const void *buf, unsigned long len,
- const char *type, struct object_id *oid,
- unsigned flags);
+
+ /*
+ * Add an object file to the in-memory object store, without writing it
-: ----------- > 5: 4550f3a2745 core doc: modernize core.bigFileThreshold documentation 4: 6bcba6bce66 ! 6: 6a70e49a346 unpack-objects: unpack_non_delta_entry() read data in a stream
@@ Metadata
Author: Han Xin <[email protected]>
## Commit message ##
- unpack-objects: unpack_non_delta_entry() read data in a stream
+ unpack-objects: use stream_loose_object() to unpack large objects
- We used to call "get_data()" in "unpack_non_delta_entry()" to read the
- entire contents of a blob object, no matter how big it is. This
- implementation may consume all the memory and cause OOM.
+ Make use of the stream_loose_object() function introduced in the
+ preceding commit to unpack large objects. Before this we'd need to
+ malloc() the size of the blob before unpacking it, which could cause
+ OOM with very large blobs.
- By implementing a zstream version of input_stream interface, we can use
- a small fixed buffer for "unpack_non_delta_entry()". However, unpack
- non-delta objects from a stream instead of from an entrie buffer will
- have 10% performance penalty.
+ We could use this new interface to unpack all blobs, but doing so
+ would result in a performance penalty of around 10%, as the below
+ "hyperfine" benchmark will show. We therefore limit this to files
+ larger than "core.bigFileThreshold":
$ hyperfine \
--setup \
@@ Commit message
-c core.bigFileThreshold=16k unpack-objects <small.pack' in 'HEAD~1'
- Therefore, only unpack objects larger than the "core.bigFileThreshold"
- in zstream. Until now, the config variable has been used in the
- following cases, and our new case belongs to the packfile category.
+ An earlier version of this patch introduced a new
+ "core.bigFileStreamingThreshold" instead of re-using the existing
+ "core.bigFileThreshold" variable[1]. As noted in a detailed overview
+ of its users in [2] using it has several different meanings.
- * Archive:
+ Still, we consider it good enough to simply re-use it. While it's
+ possible that someone might want to e.g. consider objects "small" for
+ the purposes of diffing but "big" for the purposes of writing them
+ such use-cases are probably too obscure to worry about. We can always
+ split up "core.bigFileThreshold" in the future if there's a need for
+ that.
- + archive.c: write_entry(): write large blob entries to archive in
- stream.
-
- * Loose objects:
-
- + object-file.c: index_fd(): when hashing large files in worktree,
- read files in a stream, and create one packfile per large blob if
- want to save files to git object store.
-
- + object-file.c: read_loose_object(): when checking loose objects
- using "git-fsck", do not read full content of large loose objects.
-
- * Packfile:
-
- + fast-import.c: parse_and_store_blob(): streaming large blob from
- foreign source to packfile.
-
- + index-pack.c: check_collison(): read and check large blob in stream.
-
- + index-pack.c: unpack_entry_data(): do not return the entire
- contents of the big blob from packfile, but uses a fixed buf to
- perform some integrity checks on the object.
-
- + pack-check.c: verify_packfile(): used by "git-fsck" and will call
- check_object_signature() to check large blob in pack with the
- streaming interface.
-
- + pack-objects.c: get_object_details(): set "no_try_delta" for large
- blobs when counting objects.
-
- + pack-objects.c: write_no_reuse_object(): streaming large blob to
- pack.
-
- + unpack-objects.c: unpack_non_delta_entry(): unpack large blob in
- stream from packfile.
-
- * Others:
-
- + diff.c: diff_populate_filespec(): treat large blob file as binary.
-
- + streaming.c: istream_source(): as a helper of "open_istream()" to
- select proper streaming interface to read large blob from packfile.
+ 1. https://lore.kernel.org/git/[email protected]/
+ 2. https://lore.kernel.org/git/[email protected]/
+ ## Documentation/config/core.txt ##
+@@ Documentation/config/core.txt: usage, at the slight expense of increased disk usage.
+ * Will be generally be streamed when written, which avoids excessive
+ memory usage, at the cost of some fixed overhead. Commands that make
+ use of this include linkgit:git-archive[1],
+-linkgit:git-fast-import[1], linkgit:git-index-pack[1] and +-linkgit:git-fsck[1].
++linkgit:git-fast-import[1], linkgit:git-index-pack[1], ++linkgit:git-unpack-objects[1] and linkgit:git-fsck[1].
+
+ core.excludesFile::
+ Specifies the pathname to the file that contains patterns to
+
## builtin/unpack-objects.c ##
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type type,
}
5: 1bfaf89ee0b < -: ----------- object-file API: add a format_object_header() function
Signature · 1 line
2.35.1.940.ge7a5b4b05f2
PATCH v10 1/6 unpack-objects: low memory footprint for get_data() in dry_run mode
From: Han Xin <[email protected]>
As the name implies, "get_data(size)" will allocate and return a given amount of memory. Allocating memory for a large blob object may cause the system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint for dry_run mode.
Because in dry_run mode, "get_data()" is only used to check the integrity of data, and the returned buffer is not used at all, we can allocate a smaller buffer and reuse it as zstream output. Therefore, in dry_run mode, "get_data()" will release the allocated buffer and return NULL instead of returning garbage data.
The "find [...]objects/?? -type f | wc -l" test idiom being used here is adapted from the same "find" use added to another test in d9545c7f465 (fast-import: implement unpack limit, 2016-04-25).
--- builtin/unpack-objects.c | 39 ++++++++++++++++++++-------- t/t5328-unpack-large-objects.sh | 45 +++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 11 deletions(-) create mode 100755 t/t5328-unpack-large-objects.sh
builtin/unpack-objects.c +28 -11
@@ -96,15 +96,31 @@ static void use(int bytes)display_throughput (progress, consumed_bytes); } +/* +* Decompress zstream from stdin and return specific size of data. +* The caller is responsible to free the returned buffer. +* +* But for dry_run mode, "get_data()" is only used to check the +* integrity of data, and the returned buffer is not used at all. +* Therefore, in dry_run mode, "get_data()" will release the small +* allocated buffer which is reused to hold temporary zstream output +* and return NULL instead of returning garbage data. +*/ static void *get_data (unsigned long size) {git_zstream stream; -void *buf =xmallocz (size); +unsigned long bufsize; +void *buf;memset (&stream,0 ,sizeof (stream)); +if (dry_run && size >8192 ) + bufsize =8192 ; +else + bufsize = size; + buf =xmallocz (bufsize); stream.next_out = buf; - stream.avail_out = size; + stream.avail_out = bufsize; stream.next_in =fill (1 ); stream.avail_in = len;git_inflate_init (&stream);
@@ -124,8 +140,15 @@ static void *get_data(unsigned long size) } stream.next_in =fill (1 ); stream.avail_in = len; +if (dry_run) { +/* reuse the buffer in dry_run mode */ + stream.next_out = buf; + stream.avail_out = bufsize; + } }git_inflate_end (&stream); +if (dry_run) +FREE_AND_NULL (buf);return buf; }
@@ -325,10 +348,8 @@ static void unpack_non_delta_entry(enum object_type type, unsigned long size, {void *buf =get_data (size); -if (!dry_run && buf) +if (buf)write_object (nr, type, buf, size); -else -free (buf); } static int resolve_against_held(unsigned nr, const struct object_id *base,
@@ -358,10 +379,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,oidread (&base_oid,fill (the_hash_algo->rawsz));use (the_hash_algo->rawsz); delta_data =get_data (delta_size); -if (dry_run || !delta_data) { -free (delta_data); +if (!delta_data)return ; - }if (has_object_file (&base_oid)) ;/* Ok we have this one */ else if (resolve_against_held(nr, &base_oid,
@@ -397,10 +416,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,die ("offset value out of bound for delta base object" ); delta_data =get_data (delta_size); -if (dry_run || !delta_data) { -free (delta_data); +if (!delta_data)return ; - } lo =0 ; hi = nr;while (lo < hi) {
t/t5328-unpack-large-objects.sh new file +45 -0
45 lines not on this page · raw message
Signature · 1 line
2.35.1.940.ge7a5b4b05f2
PATCH v10 2/6 object-file.c: do fsync() and close() before post-write die()
In reply to Ævar Arnfjörð BjarmasonChange write_loose_object() to do an fsync() and close() before the oideq() sanity check at the end. This change re-joins code that was split up by the die() sanity check added in 748af44c63e (sha1_file: be paranoid when creating loose objects, 2010-02-21).
I don't think that this change matters in itself, if we called die() it was possible that our data wouldn't fully make it to disk, but in any case we were writing data that we'd consider corrupted. It's possible that a subsequent "git fsck" will be less confused now.
The real reason to make this change is that in a subsequent commit we'll split this code in write_loose_object() into a utility function, all its callers will want the preceding sanity checks, but not the "oideq" check. By moving the close_loose_object() earlier it'll be easier to reason about the introduction of the utility function.
--- object-file.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-)
object-file.c +11 -2
@@ -2001,12 +2001,21 @@ static int write_loose_object(const struct object_id *oid, char *hdr,die (_ ("deflateEnd on object %s failed (%d)" ),oid_to_hex (oid), ret); the_hash_algo->final_oid_fn (¶no_oid, &c); + +/* +* We already did a write_buffer() to the "fd", let's fsync() +* and close(). +* +* We might still die() on a subsequent sanity check, but +* let's not add to that confusion by not flushing any +* outstanding writes to disk first. +*/ +close_loose_object (fd); +if (!oideq (oid, ¶no_oid))die (_ ("confused by unstable object source data for %s" ),oid_to_hex (oid)); -close_loose_object (fd); -if (mtime) {struct utimbuf utb; utb.actime = mtime;
Signature · 1 line
2.35.1.940.ge7a5b4b05f2
PATCH v10 3/6 object-file.c: refactor write_loose_object() to several steps
In reply to Ævar Arnfjörð BjarmasonFrom: Han Xin <[email protected]>
When writing a large blob using "write_loose_object()", we have to pass a buffer with the whole content of the blob, and this behavior will consume lots of memory and may cause OOM. We will introduce a stream version function ("stream_loose_object()") in later commit to resolve this issue.
Before introducing that streaming function, do some refactoring on "write_loose_object()" to reuse code for both versions.
Rewrite "write_loose_object()" as follows:
1. Figure out a path for the (temp) object file. This step is only used in "write_loose_object()".
2. Move common steps for starting to write loose objects into a new function "start_loose_object_common()".
3. Compress data.
4. Move common steps for ending zlib stream into a new function
"end_loose_object_common()".
5. Close fd and finalize the object file.
--- object-file.c | 129 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 89 insertions(+), 40 deletions(-)
object-file.c +89 -40
@@ -1943,6 +1943,87 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)return fd; } +/** +* Common steps for loose object writers to start writing loose +* objects: +* +* - Create tmpfile for the loose object. +* - Setup zlib stream for compression. +* - Start to feed header to zlib stream. +* +* Returns a "fd", which should later be provided to +* end_loose_object_common(). +*/ +static int start_loose_object_common (struct strbuf *tmp_file, +const char *filename,unsigned flags, +git_zstream *stream, +unsigned char *buf,size_t buflen, +git_hash_ctx *c, +char *hdr,int hdrlen) +{ +int fd; + + fd =create_tmpfile (tmp_file, filename); +if (fd <0 ) { +if (flags &HASH_SILENT ) +return -1 ; +else if (errno ==EACCES ) +return error (_ ("insufficient permission for adding " +"an object to repository database %s" ), +get_object_directory ()); +else +return error_errno ( +_ ("unable to create temporary file" )); + } + +/* Setup zlib stream for compression */ +git_deflate_init (stream, zlib_compression_level); + stream->next_out = buf; + stream->avail_out = buflen; + the_hash_algo->init_fn (c); + +/* Start to feed header to zlib stream */ + stream->next_in = (unsigned char *)hdr; + stream->avail_in = hdrlen; +while (git_deflate (stream,0 ) ==Z_OK ) + ;/* nothing */ + the_hash_algo->update_fn (c, hdr, hdrlen); + +return fd; +} + +/** +* Common steps for loose object writers to end writing loose objects: +* +* - End the compression of zlib stream. +* - Get the calculated oid to "parano_oid". +* - fsync() and close() the "fd" +*/ +static void end_loose_object_common (int fd,int ret,git_hash_ctx *c, +git_zstream *stream, +struct object_id *parano_oid, +const struct object_id *expected_oid, +const char *die_msg1_fmt, +const char *die_msg2_fmt) +{ +if (ret !=Z_STREAM_END ) +die (_ (die_msg1_fmt), ret, expected_oid); + ret =git_deflate_end_gently (stream); +if (ret !=Z_OK ) +die (_ (die_msg2_fmt), ret, expected_oid); + the_hash_algo->final_oid_fn (parano_oid, c); + +/* +* We already did a write_buffer() to the "fd", let's fsync() +* and close(). +* +* We might still die() on a subsequent sanity check, but +* let's not add to that confusion by not flushing any +* outstanding writes to disk first. +*/ +close_loose_object (fd); +} +static int write_loose_object (const struct object_id *oid,char *hdr,int hdrlen,const void *buf,unsigned long len,time_t mtime,unsigned flags)
@@ -1957,28 +2038,11 @@ static int write_loose_object(const struct object_id *oid, char *hdr,loose_object_path (the_repository, &filename, oid); - fd =create_tmpfile (&tmp_file, filename.buf); -if (fd <0 ) { -if (flags &HASH_SILENT ) -return -1 ; -else if (errno ==EACCES ) -return error (_ ("insufficient permission for adding an object to repository database %s" ),get_object_directory ()); -else -return error_errno (_ ("unable to create temporary file" )); - } - -/* Set it up */ -git_deflate_init (&stream, zlib_compression_level); - stream.next_out = compressed; - stream.avail_out =sizeof (compressed); - the_hash_algo->init_fn (&c); - -/* First header.. */ - stream.next_in = (unsigned char *)hdr; - stream.avail_in = hdrlen; -while (git_deflate (&stream,0 ) ==Z_OK ) - ;/* nothing */ - the_hash_algo->update_fn (&c, hdr, hdrlen); + fd =start_loose_object_common (&tmp_file, filename.buf, flags, + &stream, compressed,sizeof (compressed), + &c, hdr, hdrlen); +if (fd <0 ) +return -1 ;/* Then the data itself.. */ stream.next_in = (void *)buf;
@@ -1993,24 +2057,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr, stream.avail_out =sizeof (compressed); }while (ret ==Z_OK ); -if (ret !=Z_STREAM_END ) -die (_ ("unable to deflate new object %s (%d)" ),oid_to_hex (oid), - ret); - ret =git_deflate_end_gently (&stream); -if (ret !=Z_OK ) -die (_ ("deflateEnd on object %s failed (%d)" ),oid_to_hex (oid), - ret); - the_hash_algo->final_oid_fn (¶no_oid, &c); - -/* -* We already did a write_buffer() to the "fd", let's fsync() -* and close(). -* -* We might still die() on a subsequent sanity check, but -* let's not add to that confusion by not flushing any -* outstanding writes to disk first. -*/ -close_loose_object (fd); +end_loose_object_common (fd, ret, &c, &stream, ¶no_oid, oid, +N_ ("unable to deflate new object %s (%d)" ), +N_ ("deflateEnd on object %s failed (%d)" ));if (!oideq (oid, ¶no_oid)) die(_("confused by unstable object source data for %s"),
Signature · 1 line
2.35.1.940.ge7a5b4b05f2
PATCH v10 4/6 object-file.c: add "stream_loose_object()" to handle large object
In reply to Ævar Arnfjörð BjarmasonFrom: Han Xin <[email protected]>
If we want unpack and write a loose object using "write_loose_object", we have to feed it with a buffer with the same size of the object, which will consume lots of memory and may cause OOM. This can be improved by feeding data to "stream_loose_object()" in a stream.
Add a new function "stream_loose_object()", which is a stream version of "write_loose_object()" but with a low memory footprint. We will use this function to unpack large blob object in later commit.
Another difference with "write_loose_object()" is that we have no chance to run "write_object_file_prepare()" to calculate the oid in advance. In "write_loose_object()", we know the oid and we can write the temporary file in the same directory as the final object, but for an object with an undetermined oid, we don't know the exact directory for the object.
Still, we need to save the temporary file we're preparing somewhere. We'll do that in the top-level ".git/objects/" directory (or whatever "GIT_OBJECT_DIRECTORY" is set to). Once we've streamed it we'll know the OID, and will move it to its canonical path.
"freshen_packed_object()" or "freshen_loose_object()" will be called inside "stream_loose_object()" after obtaining the "oid".
--- object-file.c | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++ object-store.h | 8 ++++ 2 files changed, 107 insertions(+)
object-file.c +99 -0
@@ -2095,6 +2095,105 @@ static int freshen_packed_object(const struct object_id *oid)return 1 ; } +int stream_loose_object (struct input_stream *in_stream,size_t len, +struct object_id *oid) +{ +int fd, ret, err =0 , flush =0 ; +unsigned char compressed[4096 ]; +git_zstream stream; +git_hash_ctx c; +struct strbuf tmp_file =STRBUF_INIT ; +struct strbuf filename =STRBUF_INIT ; +int dirlen; +char hdr[MAX_HEADER_LEN ]; +int hdrlen; + +/* Since oid is not determined, save tmp file to odb path. */ +strbuf_addf (&filename,"%s/" ,get_object_directory ()); + hdrlen =format_object_header (hdr,sizeof (hdr),OBJ_BLOB , len); + +/* Common steps for write_loose_object and stream_loose_object to +* start writing loose oject: +* +* - Create tmpfile for the loose object. +* - Setup zlib stream for compression. +* - Start to feed header to zlib stream. +*/ + fd =start_loose_object_common (&tmp_file, filename.buf,0 , + &stream, compressed,sizeof (compressed), + &c, hdr, hdrlen); +if (fd <0 ) { + err =-1 ; + goto cleanup; + } + +/* Then the data itself.. */ +do { +unsigned char *in0 = stream.next_in; +if (!stream.avail_in && !in_stream->is_finished) { +const void *in = in_stream->read (in_stream, &stream.avail_in); + stream.next_in = (void *)in; + in0 = (unsigned char *)in; +/* All data has been read. */ +if (in_stream->is_finished) + flush =Z_FINISH ; + } + ret =git_deflate (&stream, flush); + the_hash_algo->update_fn (&c, in0, stream.next_in - in0); +if (write_buffer (fd, compressed, stream.next_out - compressed) <0 ) +die (_ ("unable to write loose object file" )); + stream.next_out = compressed; + stream.avail_out =sizeof (compressed); +/* +* Unlike write_loose_object(), we do not have the entire +* buffer. If we get Z_BUF_ERROR due to too few input bytes, +* then we'll replenish them in the next input_stream->read() +* call when we loop. +*/ + }while (ret ==Z_OK || ret ==Z_BUF_ERROR ); + +if (stream.total_in != len + hdrlen) +die (_ ("write stream object %ld != %" PRIuMAX), stream.total_in, + (uintmax_t )len + hdrlen); + +/* Common steps for write_loose_object and stream_loose_object to +* end writing loose oject: +* +* - End the compression of zlib stream. +* - Get the calculated oid. +*/ +end_loose_object_common (fd, ret, &c, &stream, oid,NULL , +N_ ("unable to stream deflate new object (%d)" ), +N_ ("deflateEnd on stream object failed (%d)" )); + +if (freshen_packed_object (oid) ||freshen_loose_object (oid)) { +unlink_or_warn (tmp_file.buf); + goto cleanup; + } + +loose_object_path (the_repository, &filename, oid); + +/* We finally know the object path, and create the missing dir. */ + dirlen =directory_size (filename.buf); +if (dirlen) { +struct strbuf dir =STRBUF_INIT ; +strbuf_add (&dir, filename.buf, dirlen); + +if (mkdir_in_gitdir (dir.buf) && errno !=EEXIST ) { + err =error_errno (_ ("unable to create directory %s" ), dir.buf); +strbuf_release (&dir); + goto cleanup; + } +strbuf_release (&dir); + } + + err =finalize_object_file (tmp_file.buf, filename.buf); +cleanup: +strbuf_release (&tmp_file); +strbuf_release (&filename); +return err; +} +int write_object_file_flags (const void *buf,unsigned long len,enum object_type type,struct object_id *oid,unsigned flags)
object-store.h +8 -0
@@ -46,6 +46,12 @@ struct object_directory {char *path; }; +struct input_stream { +const void *(*read)(struct input_stream *,unsigned long *len); +void *data; +int is_finished; +}; + KHASH_INIT(odb_path_map, const char * /* key: odb_path */, struct object_directory *, 1, fspathhash, fspatheq)
@@ -261,6 +267,8 @@ static inline int write_object_file(const void *buf, unsigned long len,int write_object_file_literally (const void *buf,unsigned long len,const char *type,struct object_id *oid,unsigned flags); +int stream_loose_object (struct input_stream *in_stream,size_t len, +struct object_id *oid);/* * Add an object file to the in-memory object store, without writing it
Signature · 1 line
2.35.1.940.ge7a5b4b05f2
PATCH v10 5/6 core doc: modernize core.bigFileThreshold documentation
In reply to Ævar Arnfjörð BjarmasonThe core.bigFileThreshold documentation has been largely unchanged since 5eef828bc03 (fast-import: Stream very large blobs directly to pack, 2010-02-01).
But since then this setting has been expanded to affect a lot more than that description indicated. Most notably in how "git diff" treats them, see 6bf3b813486 (diff --stat: mark any file larger than core.bigfilethreshold binary, 2014-08-16).
In addition to that, numerous commands and APIs make use of a streaming mode for files above this threshold.
So let's attempt to summarize 12 years of changes in behavior, which can be seen with:
git log --oneline -Gbig_file_thre 5eef828bc03.. -- '*.c'
To do that turn this into a bullet-point list. The summary Han Xin produced in [1] helped a lot, but is a bit too detailed for documentation aimed at users. Let's instead summarize how user-observable behavior differs, and generally describe how we tend to stream these files in various commands.
1. https://lore.kernel.org/git/[email protected]/
--- Documentation/config/core.txt | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-)
Documentation/config/core.txt +24 -9
@@ -412,17 +412,32 @@ You probably do not need to adjust this value. Common unit suffixes of 'k', 'm', or 'g' are supported. core.bigFileThreshold:: - Files larger than this size are stored deflated, without - attempting delta compression. Storing large files without - delta compression avoids excessive memory usage, at the - slight expense of increased disk usage. Additionally files - larger than this size are always treated as binary. + The size of files considered "big", which as discussed below + changes the behavior of numerous git commands, as well as how + such files are stored within the repository. The default is + 512 MiB. Common unit suffixes of 'k', 'm', or 'g' are + supported. + -Default is 512 MiB on all platforms. This should be reasonable -for most projects as source code and other text files can still -be delta compressed, but larger binary media files won't be. +Files above the configured limit will be: + -Common unit suffixes of 'k', 'm', or 'g' are supported. +* Stored deflated, without attempting delta compression. ++ +The default limit is primarily set with this use-case in mind. With it +most projects will have their source code and other text files delta +compressed, but not larger binary media files. ++ +Storing large files without delta compression avoids excessive memory +usage, at the slight expense of increased disk usage. ++ +* Will be treated as if though they were labeled "binary" (see + linkgit:gitattributes[5]). This means that e.g. linkgit:git-log[1] + and linkgit:git-diff[1] will not diffs for files above this limit. ++ +* Will be generally be streamed when written, which avoids excessive +memory usage, at the cost of some fixed overhead. Commands that make +use of this include linkgit:git-archive[1], +linkgit:git-fast-import[1], linkgit:git-index-pack[1] and +linkgit:git-fsck[1]. core.excludesFile:: Specifies the pathname to the file that contains patterns to
Signature · 1 line
2.35.1.940.ge7a5b4b05f2
PATCH v10 6/6 unpack-objects: use stream_loose_object() to unpack large objects
In reply to Ævar Arnfjörð BjarmasonFrom: Han Xin <[email protected]>
Make use of the stream_loose_object() function introduced in the preceding commit to unpack large objects. Before this we'd need to malloc() the size of the blob before unpacking it, which could cause OOM with very large blobs.
We could use this new interface to unpack all blobs, but doing so would result in a performance penalty of around 10%, as the below "hyperfine" benchmark will show. We therefore limit this to files larger than "core.bigFileThreshold":
$ hyperfine \
--setup \
'if ! test -d scalar.git; then git clone --bare
https://github.com/microsoft/scalar.git;
cp scalar.git/objects/pack/*.pack small.pack; fi' \
--prepare 'rm -rf dest.git && git init --bare dest.git' \
... Summary
'./git -C dest.git -c core.bigFileThreshold=512m
unpack-objects <small.pack' in 'origin/master'
1.01 ± 0.04 times faster than './git -C dest.git
-c core.bigFileThreshold=512m unpack-objects
<small.pack' in 'HEAD~1'
1.01 ± 0.04 times faster than './git -C dest.git
-c core.bigFileThreshold=512m unpack-objects
<small.pack' in 'HEAD~0'
1.03 ± 0.10 times faster than './git -C dest.git
-c core.bigFileThreshold=16k unpack-objects
<small.pack' in 'origin/master'
1.02 ± 0.07 times faster than './git -C dest.git
-c core.bigFileThreshold=16k unpack-objects
<small.pack' in 'HEAD~0'
1.10 ± 0.04 times faster than './git -C dest.git
-c core.bigFileThreshold=16k unpack-objects
<small.pack' in 'HEAD~1'An earlier version of this patch introduced a new "core.bigFileStreamingThreshold" instead of re-using the existing "core.bigFileThreshold" variable[1]. As noted in a detailed overview of its users in [2] using it has several different meanings.
Still, we consider it good enough to simply re-use it. While it's possible that someone might want to e.g. consider objects "small" for the purposes of diffing but "big" for the purposes of writing them such use-cases are probably too obscure to worry about. We can always split up "core.bigFileThreshold" in the future if there's a need for that.
1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/
--- Documentation/config/core.txt | 4 +- builtin/unpack-objects.c | 71 ++++++++++++++++++++++++++++++++- t/t5328-unpack-large-objects.sh | 23 +++++++++-- 3 files changed, 92 insertions(+), 6 deletions(-)
Documentation/config/core.txt +2 -2
@@ -436,8 +436,8 @@ usage, at the slight expense of increased disk usage. * Will be generally be streamed when written, which avoids excessive memory usage, at the cost of some fixed overhead. Commands that make use of this include linkgit:git-archive[1], -linkgit:git-fast-import[1], linkgit:git-index-pack[1] and -linkgit:git-fsck[1]. +linkgit:git-fast-import[1], linkgit:git-index-pack[1], +linkgit:git-unpack-objects[1] and linkgit:git-fsck[1]. core.excludesFile:: Specifies the pathname to the file that contains patterns to
builtin/unpack-objects.c +70 -1
81 lines not on this page · raw message
t/t5328-unpack-large-objects.sh +20 -3
@@ -9,7 +9,11 @@ test_description='git unpack-objects with large objects' prepare_dest () { test_when_finished "rm -rf dest.git" && - git init --bare dest.git + git init --bare dest.git && + if test -n "$1" + then + git -C dest.git config core.bigFileThreshold $1 + fi } test_no_loose () {
@@ -30,16 +34,29 @@ test_expect_success 'set memory limitation to 1MB' ' ' test_expect_success 'unpack-objects failed under memory limitation' ' - prepare_dest && + prepare_dest 2m && test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err && grep "fatal: attempting to allocate" err ' test_expect_success 'unpack-objects works with memory limitation in dry-run mode' ' - prepare_dest && + prepare_dest 2m && git -C dest.git unpack-objects -n <test-$PACK.pack && test_no_loose && test_dir_is_empty dest.git/objects/pack ' +test_expect_success 'unpack big object in stream' ' + prepare_dest 1m && + git -C dest.git unpack-objects <test-$PACK.pack && + test_dir_is_empty dest.git/objects/pack +' + +test_expect_success 'do not unpack existing large objects' ' + prepare_dest 1m && + git -C dest.git index-pack --stdin <test-$PACK.pack && + git -C dest.git unpack-objects <test-$PACK.pack && + test_no_loose +' + test_done
Signature · 1 line
2.35.1.940.ge7a5b4b05f2
v11 cover unpack-objects: support streaming blobs to disk
In reply to Ævar Arnfjörð BjarmasonThis series by Han Xin was waiting on some in-flight patches that landed in 430883a70c7 (Merge branch 'ab/object-file-api-updates', 2022-03-16).
This series teaches "git unpack-objects" to stream objects larger than core.bigFileThreshold to disk. As 8/8 shows streaming e.g. a 100MB blob now uses ~5MB of memory instead of ~105MB. This streaming method is slower if you've got memory to handle the blobs in-core, but if you don't it allows you to unpack objects at all, as you might otherwise OOM.
Changes since v10:
* Renamed the new test file, its number conflicted with a since-landed commit-graph test.
* Some minor code changes to make diffs to the pre-image smaller (e.g. the top of the range-diff below)
* The whole "find dest.git" to see if we have loose objects is now either a test for "do we have objects at all?" (--dry-run mode), or uses a simpler implementation. We could use "test_stdout_line_count" for that.
* We also test that as we use "unpack-objects" to stream directly to a pack that the result is byte-for-byte the same as the source.
* A new 4/8 that I added allows for more code sharing in object-file.c, our two end-state functions now share more logic.
* Minor typo/grammar/comment etc. fixes throughout.
* Updated 8/8 with benchmarks, somewhere along the line we lost the code to run the benchmark mentioned in the commit message...
1. https://lore.kernel.org/git/[email protected]/
Han Xin (4):
unpack-objects: low memory footprint for get_data() in dry_run mode object-file.c: refactor write_loose_object() to several steps object-file.c: add "stream_loose_object()" to handle large object
unpack-objects: use stream_loose_object() to unpack large objects
Ævar Arnfjörð Bjarmason (4):
object-file.c: do fsync() and close() before post-write die() object-file.c: factor out deflate part of write_loose_object() core doc: modernize core.bigFileThreshold documentation
unpack-objects: refactor away unpack_non_delta_entry()
Documentation/config/core.txt | 33 +++-- builtin/unpack-objects.c | 109 +++++++++++--- object-file.c | 250 +++++++++++++++++++++++++++----- object-store.h | 8 + t/t5351-unpack-large-objects.sh | 61 ++++++++ 5 files changed, 397 insertions(+), 64 deletions(-) create mode 100755 t/t5351-unpack-large-objects.sh
Range-diff against v10:
1: e46eb75b98f ! 1: 2103d5bfd96 unpack-objects: low memory footprint for get_data() in dry_run mode
@@ builtin/unpack-objects.c: static void use(int bytes)
{
git_zstream stream;
- void *buf = xmallocz(size);
-+ unsigned long bufsize;
-+ void *buf;
++ unsigned long bufsize = dry_run && size > 8192 ? 8192 : size;
++ void *buf = xmallocz(bufsize);
memset(&stream, 0, sizeof(stream));
-+ if (dry_run && size > 8192)
-+ bufsize = 8192;
-+ else
-+ bufsize = size;
-+ buf = xmallocz(bufsize);
stream.next_out = buf;
- stream.avail_out = size;
@@ builtin/unpack-objects.c: static void unpack_delta_entry(enum object_type type,
hi = nr;
while (lo < hi) {
- ## t/t5328-unpack-large-objects.sh (new) ##
+ ## t/t5351-unpack-large-objects.sh (new) ##
@@
+#!/bin/sh
+#
@@ t/t5328-unpack-large-objects.sh (new)
+ git init --bare dest.git
+}
+
-+test_no_loose () {
-+ test $(find dest.git/objects/?? -type f | wc -l) = 0
-+}
-+
+test_expect_success "create large objects (1.5 MB) and PACK" '
+ test-tool genrandom foo 1500000 >big-blob &&
+ test_commit --append foo big-blob &&
+ test-tool genrandom bar 1500000 >big-blob &&
+ test_commit --append bar big-blob &&
-+ PACK=$(echo HEAD | git pack-objects --revs test)
++ PACK=$(echo HEAD | git pack-objects --revs pack)
+'
+
+test_expect_success 'set memory limitation to 1MB' '
@@ t/t5328-unpack-large-objects.sh (new)
+
+test_expect_success 'unpack-objects failed under memory limitation' '
+ prepare_dest &&
-+ test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err &&
++ test_must_fail git -C dest.git unpack-objects <pack-$PACK.pack 2>err &&
+ grep "fatal: attempting to allocate" err
+'
+
+test_expect_success 'unpack-objects works with memory limitation in dry-run mode' '
+ prepare_dest &&
-+ git -C dest.git unpack-objects -n <test-$PACK.pack &&
-+ test_no_loose &&
++ git -C dest.git unpack-objects -n <pack-$PACK.pack &&
++ test_stdout_line_count = 0 find dest.git/objects -type f &&
+ test_dir_is_empty dest.git/objects/pack
+'
+
2: 48bf9090058 = 2: 6acd8759772 object-file.c: do fsync() and close() before post-write die()
3: 0e33d2a6e35 = 3: f7b02c307fc object-file.c: refactor write_loose_object() to several steps
-: ----------- > 4: 20d97cc2605 object-file.c: factor out deflate part of write_loose_object()
4: 9644df5c744 ! 5: db40f4160c4 object-file.c: add "stream_loose_object()" to handle large object
@@ Commit message
Helped-by: Ævar Arnfjörð Bjarmason <[email protected]>
Helped-by: Jiang Xin <[email protected]>
Signed-off-by: Han Xin <[email protected]>
+ Signed-off-by: Ævar Arnfjörð Bjarmason <[email protected]>
## object-file.c ##
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
+ strbuf_addf(&filename, "%s/", get_object_directory());
+ hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
+
-+ /* Common steps for write_loose_object and stream_loose_object to
-+ * start writing loose oject:
++ /*
++ * Common steps for write_loose_object and stream_loose_object to
++ * start writing loose objects:
+ *
+ * - Create tmpfile for the loose object.
+ * - Setup zlib stream for compression.
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
+ /* Then the data itself.. */
+ do {
+ unsigned char *in0 = stream.next_in;
++
+ if (!stream.avail_in && !in_stream->is_finished) {
+ const void *in = in_stream->read(in_stream, &stream.avail_in);
+ stream.next_in = (void *)in;
+ in0 = (unsigned char *)in;
+ /* All data has been read. */
+ if (in_stream->is_finished)
-+ flush = Z_FINISH;
++ flush = 1;
+ }
-+ ret = git_deflate(&stream, flush);
-+ the_hash_algo->update_fn(&c, in0, stream.next_in - in0);
-+ if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
-+ die(_("unable to write loose object file"));
-+ stream.next_out = compressed;
-+ stream.avail_out = sizeof(compressed);
++ ret = write_loose_object_common(&c, &stream, flush, in0, fd,
++ compressed, sizeof(compressed));
+ /*
+ * Unlike write_loose_object(), we do not have the entire
+ * buffer. If we get Z_BUF_ERROR due to too few input bytes,
5: 4550f3a2745 = 6: d8ae2eadb98 core doc: modernize core.bigFileThreshold documentation
-: ----------- > 7: 2b403e7cd9c unpack-objects: refactor away unpack_non_delta_entry()
6: 6a70e49a346 ! 8: 5eded902496 unpack-objects: use stream_loose_object() to unpack large objects @@ Commit message
malloc() the size of the blob before unpacking it, which could cause OOM with very large blobs.
- We could use this new interface to unpack all blobs, but doing so
- would result in a performance penalty of around 10%, as the below
- "hyperfine" benchmark will show. We therefore limit this to files
- larger than "core.bigFileThreshold":
-
- $ hyperfine \
- --setup \
- 'if ! test -d scalar.git; then git clone --bare
- https://github.com/microsoft/scalar.git;
- cp scalar.git/objects/pack/*.pack small.pack; fi' \
- --prepare 'rm -rf dest.git && git init --bare dest.git' \
- ...
-
- Summary
- './git -C dest.git -c core.bigFileThreshold=512m
- unpack-objects <small.pack' in 'origin/master'
- 1.01 ± 0.04 times faster than './git -C dest.git
- -c core.bigFileThreshold=512m unpack-objects
- <small.pack' in 'HEAD~1'
- 1.01 ± 0.04 times faster than './git -C dest.git
- -c core.bigFileThreshold=512m unpack-objects
- <small.pack' in 'HEAD~0'
- 1.03 ± 0.10 times faster than './git -C dest.git
- -c core.bigFileThreshold=16k unpack-objects
- <small.pack' in 'origin/master'
- 1.02 ± 0.07 times faster than './git -C dest.git
- -c core.bigFileThreshold=16k unpack-objects
- <small.pack' in 'HEAD~0'
- 1.10 ± 0.04 times faster than './git -C dest.git
- -c core.bigFileThreshold=16k unpack-objects
- <small.pack' in 'HEAD~1'
+ We could use the new streaming interface to unpack all blobs, but
+ doing so would be much slower, as demonstrated e.g. with this
+ benchmark using git-hyperfine[0]:
+
+ rm -rf /tmp/scalar.git &&
+ git clone --bare https://github.com/Microsoft/scalar.git /tmp/scalar.git &&
+ mv /tmp/scalar.git/objects/pack/*.pack /tmp/scalar.git/my.pack &&
+ git hyperfine \
+ -r 2 --warmup 1 \
+ -L rev origin/master,HEAD -L v "10,512,1k,1m" \
+ -s 'make' \
+ -p 'git init --bare dest.git' \
+ -c 'rm -rf dest.git' \
+ './git -C dest.git -c core.bigFileThreshold={v} unpack-objects </tmp/scalar.git/my.pack'
+
+ Here we'll perform worse with lower core.bigFileThreshold settings
+ with this change in terms of speed, but we're getting lower memory use
+ in return:
+
+ Summary
+ './git -C dest.git -c core.bigFileThreshold=10 unpack-objects </tmp/scalar.git/my.pack' in 'origin/master' ran
+ 1.01 ± 0.01 times faster than './git -C dest.git -c core.bigFileThreshold=1k unpack-objects </tmp/scalar.git/my.pack' in 'origin/master'
+ 1.01 ± 0.01 times faster than './git -C dest.git -c core.bigFileThreshold=1m unpack-objects </tmp/scalar.git/my.pack' in 'origin/master'
+ 1.01 ± 0.02 times faster than './git -C dest.git -c core.bigFileThreshold=1m unpack-objects </tmp/scalar.git/my.pack' in 'HEAD'
+ 1.02 ± 0.00 times faster than './git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/scalar.git/my.pack' in 'origin/master'
+ 1.09 ± 0.01 times faster than './git -C dest.git -c core.bigFileThreshold=1k unpack-objects </tmp/scalar.git/my.pack' in 'HEAD'
+ 1.10 ± 0.00 times faster than './git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/scalar.git/my.pack' in 'HEAD'
+ 1.11 ± 0.00 times faster than './git -C dest.git -c core.bigFileThreshold=10 unpack-objects </tmp/scalar.git/my.pack' in 'HEAD'
+
+ A better benchmark to demonstrate the benefits of that this one, which
+ creates an artificial repo with a 1, 25, 50, 75 and 100MB blob:
+
+ rm -rf /tmp/repo &&
+ git init /tmp/repo &&
+ (
+ cd /tmp/repo &&
+ for i in 1 25 50 75 100
+ do
+ dd if=/dev/urandom of=blob.$i count=$(($i*1024)) bs=1024
+ done &&
+ git add blob.* &&
+ git commit -mblobs &&
+ git gc &&
+ PACK=$(echo .git/objects/pack/pack-*.pack) &&
+ cp "$PACK" my.pack
+ ) &&
+ git hyperfine \
+ --show-output \
+ -L rev origin/master,HEAD -L v "512,50m,100m" \
+ -s 'make' \
+ -p 'git init --bare dest.git' \
+ -c 'rm -rf dest.git' \
+ '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold={v} unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum'
+
+ Using this test we'll always use >100MB of memory on
+ origin/master (around ~105MB), but max out at e.g. ~55MB if we set
+ core.bigFileThreshold=50m.
+
+ The relevant "Maximum resident set size" lines were manually added
+ below the relevant benchmark:
+
+ '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=50m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'origin/master' ran
+ Maximum resident set size (kbytes): 107080
+ 1.02 ± 0.78 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'origin/master'
+ Maximum resident set size (kbytes): 106968
+ 1.09 ± 0.79 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=100m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'origin/master'
+ Maximum resident set size (kbytes): 107032
+ 1.42 ± 1.07 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=100m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'HEAD'
+ Maximum resident set size (kbytes): 107072
+ 1.83 ± 1.02 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=50m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'HEAD'
+ Maximum resident set size (kbytes): 55704
+ 2.16 ± 1.19 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'HEAD'
+ Maximum resident set size (kbytes): 4564
+
+ This shows that if you have enough memory this new streaming method is
+ slower the lower you set the streaming threshold, but the benefit is
+ more bounded memory use.
An earlier version of this patch introduced a new "core.bigFileStreamingThreshold" instead of re-using the existing
@@ Commit message
split up "core.bigFileThreshold" in the future if there's a need for that.
+ 0. https://github.com/avar/git-hyperfine/
1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/
@@ Commit message
+ Signed-off-by: Ævar Arnfjörð Bjarmason <[email protected]>
## Documentation/config/core.txt ##
@@ Documentation/config/core.txt: usage, at the slight expense of increased disk usage.
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ return data->buf;
+}
+
-+static void write_stream_blob(unsigned nr, size_t size)
++static void stream_blob(unsigned long size, unsigned nr)
+{
+ git_zstream zstream = { 0 };
+ struct input_zstream_data data = { 0 };
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ .read = feed_input_zstream,
+ .data = &data,
+ };
++ struct obj_info *info = &obj_list[nr];
+
+ data.zstream = &zstream;
+ git_inflate_init(&zstream);
+
-+ if (stream_loose_object(&in_stream, size, &obj_list[nr].oid))
++ if (stream_loose_object(&in_stream, size, &info->oid))
+ die(_("failed to write object in stream"));
+
+ if (data.status != Z_STREAM_END)
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
+ git_inflate_end(&zstream);
+
+ if (strict) {
-+ struct blob *blob =
-+ lookup_blob(the_repository, &obj_list[nr].oid);
-+ if (blob)
-+ blob->object.flags |= FLAG_WRITTEN;
-+ else
++ struct blob *blob = lookup_blob(the_repository, &info->oid);
++
++ if (!blob)
+ die(_("invalid blob object from stream"));
++ blob->object.flags |= FLAG_WRITTEN;
+ }
-+ obj_list[nr].obj = NULL;
++ info->obj = NULL;
+}
+
- static void unpack_non_delta_entry(enum object_type type, unsigned long size,
- unsigned nr)
+ static int resolve_against_held(unsigned nr, const struct object_id *base,
+ void *delta_data, unsigned long delta_size)
{
-- void *buf = get_data(size);
-+ void *buf;
-+
-+ /* Write large blob in stream without allocating full buffer. */
-+ if (!dry_run && type == OBJ_BLOB && size > big_file_threshold) {
-+ write_stream_blob(nr, size);
-+ return;
-+ }
+@@ builtin/unpack-objects.c: static void unpack_one(unsigned nr)
-+ buf = get_data(size);
- if (buf)
- write_object(nr, type, buf, size);
- }
+ switch (type) {
+ case OBJ_BLOB:
++ if (!dry_run && size > big_file_threshold) {
++ stream_blob(size, nr);
++ return;
++ }
++ /* fallthrough */
+ case OBJ_COMMIT:
+ case OBJ_TREE:
+ case OBJ_TAG:
- ## t/t5328-unpack-large-objects.sh ##
-@@ t/t5328-unpack-large-objects.sh: test_description='git unpack-objects with large objects'
+ ## t/t5351-unpack-large-objects.sh ##
+@@ t/t5351-unpack-large-objects.sh: test_description='git unpack-objects with large objects'
prepare_dest () {
test_when_finished "rm -rf dest.git" &&
- git init --bare dest.git
+ git init --bare dest.git &&
-+ if test -n "$1"
-+ then
-+ git -C dest.git config core.bigFileThreshold $1
-+ fi
++ git -C dest.git config core.bigFileThreshold "$1"
}
- test_no_loose () {
-@@ t/t5328-unpack-large-objects.sh: test_expect_success 'set memory limitation to 1MB' '
+ test_expect_success "create large objects (1.5 MB) and PACK" '
+@@ t/t5351-unpack-large-objects.sh: test_expect_success 'set memory limitation to 1MB' '
'
test_expect_success 'unpack-objects failed under memory limitation' '
- prepare_dest &&
+ prepare_dest 2m &&
- test_must_fail git -C dest.git unpack-objects <test-$PACK.pack 2>err &&
+ test_must_fail git -C dest.git unpack-objects <pack-$PACK.pack 2>err &&
grep "fatal: attempting to allocate" err
'
test_expect_success 'unpack-objects works with memory limitation in dry-run mode' '
- prepare_dest &&
+ prepare_dest 2m &&
- git -C dest.git unpack-objects -n <test-$PACK.pack &&
- test_no_loose &&
+ git -C dest.git unpack-objects -n <pack-$PACK.pack &&
+ test_stdout_line_count = 0 find dest.git/objects -type f &&
test_dir_is_empty dest.git/objects/pack
'
+test_expect_success 'unpack big object in stream' '
+ prepare_dest 1m &&
-+ git -C dest.git unpack-objects <test-$PACK.pack &&
++ git -C dest.git unpack-objects <pack-$PACK.pack &&
+ test_dir_is_empty dest.git/objects/pack
+'
+
+test_expect_success 'do not unpack existing large objects' '
+ prepare_dest 1m &&
-+ git -C dest.git index-pack --stdin <test-$PACK.pack &&
-+ git -C dest.git unpack-objects <test-$PACK.pack &&
-+ test_no_loose
++ git -C dest.git index-pack --stdin <pack-$PACK.pack &&
++ git -C dest.git unpack-objects <pack-$PACK.pack &&
++
++ # The destination came up with the exact same pack...
++ DEST_PACK=$(echo dest.git/objects/pack/pack-*.pack) &&
++ test_cmp pack-$PACK.pack $DEST_PACK &&
++
++ # ...and wrote no loose objects
++ test_stdout_line_count = 0 find dest.git/objects -type f ! -name "pack-*"
+'
+
test_done
Signature · 1 line
2.35.1.1438.g8874c8eeb35
PATCH v11 1/8 unpack-objects: low memory footprint for get_data() in dry_run mode
From: Han Xin <[email protected]>
As the name implies, "get_data(size)" will allocate and return a given amount of memory. Allocating memory for a large blob object may cause the system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint for dry_run mode.
Because in dry_run mode, "get_data()" is only used to check the integrity of data, and the returned buffer is not used at all, we can allocate a smaller buffer and reuse it as zstream output. Therefore, in dry_run mode, "get_data()" will release the allocated buffer and return NULL instead of returning garbage data.
The "find [...]objects/?? -type f | wc -l" test idiom being used here is adapted from the same "find" use added to another test in d9545c7f465 (fast-import: implement unpack limit, 2016-04-25).
--- builtin/unpack-objects.c | 34 ++++++++++++++++++--------- t/t5351-unpack-large-objects.sh | 41 +++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 11 deletions(-) create mode 100755 t/t5351-unpack-large-objects.sh
builtin/unpack-objects.c +23 -11
@@ -96,15 +96,26 @@ static void use(int bytes)display_throughput (progress, consumed_bytes); } +/* +* Decompress zstream from stdin and return specific size of data. +* The caller is responsible to free the returned buffer. +* +* But for dry_run mode, "get_data()" is only used to check the +* integrity of data, and the returned buffer is not used at all. +* Therefore, in dry_run mode, "get_data()" will release the small +* allocated buffer which is reused to hold temporary zstream output +* and return NULL instead of returning garbage data. +*/ static void *get_data (unsigned long size) {git_zstream stream; -void *buf =xmallocz (size); +unsigned long bufsize = dry_run && size >8192 ?8192 : size; +void *buf =xmallocz (bufsize);memset (&stream,0 ,sizeof (stream)); stream.next_out = buf; - stream.avail_out = size; + stream.avail_out = bufsize; stream.next_in =fill (1 ); stream.avail_in = len;git_inflate_init (&stream);
@@ -124,8 +135,15 @@ static void *get_data(unsigned long size) } stream.next_in =fill (1 ); stream.avail_in = len; +if (dry_run) { +/* reuse the buffer in dry_run mode */ + stream.next_out = buf; + stream.avail_out = bufsize; + } }git_inflate_end (&stream); +if (dry_run) +FREE_AND_NULL (buf);return buf; }
@@ -325,10 +343,8 @@ static void unpack_non_delta_entry(enum object_type type, unsigned long size, {void *buf =get_data (size); -if (!dry_run && buf) +if (buf)write_object (nr, type, buf, size); -else -free (buf); } static int resolve_against_held(unsigned nr, const struct object_id *base,
@@ -358,10 +374,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,oidread (&base_oid,fill (the_hash_algo->rawsz));use (the_hash_algo->rawsz); delta_data =get_data (delta_size); -if (dry_run || !delta_data) { -free (delta_data); +if (!delta_data)return ; - }if (has_object_file (&base_oid)) ;/* Ok we have this one */ else if (resolve_against_held(nr, &base_oid,
@@ -397,10 +411,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,die ("offset value out of bound for delta base object" ); delta_data =get_data (delta_size); -if (dry_run || !delta_data) { -free (delta_data); +if (!delta_data)return ; - } lo =0 ; hi = nr;while (lo < hi) {
t/t5351-unpack-large-objects.sh new file +41 -0
41 lines not on this page · raw message
Signature · 1 line
2.35.1.1438.g8874c8eeb35
PATCH v11 2/8 object-file.c: do fsync() and close() before post-write die()
In reply to Ævar Arnfjörð BjarmasonChange write_loose_object() to do an fsync() and close() before the oideq() sanity check at the end. This change re-joins code that was split up by the die() sanity check added in 748af44c63e (sha1_file: be paranoid when creating loose objects, 2010-02-21).
I don't think that this change matters in itself, if we called die() it was possible that our data wouldn't fully make it to disk, but in any case we were writing data that we'd consider corrupted. It's possible that a subsequent "git fsck" will be less confused now.
The real reason to make this change is that in a subsequent commit we'll split this code in write_loose_object() into a utility function, all its callers will want the preceding sanity checks, but not the "oideq" check. By moving the close_loose_object() earlier it'll be easier to reason about the introduction of the utility function.
--- object-file.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-)
object-file.c +11 -2
@@ -2001,12 +2001,21 @@ static int write_loose_object(const struct object_id *oid, char *hdr,die (_ ("deflateEnd on object %s failed (%d)" ),oid_to_hex (oid), ret); the_hash_algo->final_oid_fn (¶no_oid, &c); + +/* +* We already did a write_buffer() to the "fd", let's fsync() +* and close(). +* +* We might still die() on a subsequent sanity check, but +* let's not add to that confusion by not flushing any +* outstanding writes to disk first. +*/ +close_loose_object (fd); +if (!oideq (oid, ¶no_oid))die (_ ("confused by unstable object source data for %s" ),oid_to_hex (oid)); -close_loose_object (fd); -if (mtime) {struct utimbuf utb; utb.actime = mtime;
Signature · 1 line
2.35.1.1438.g8874c8eeb35
PATCH v11 3/8 object-file.c: refactor write_loose_object() to several steps
In reply to Ævar Arnfjörð BjarmasonFrom: Han Xin <[email protected]>
When writing a large blob using "write_loose_object()", we have to pass a buffer with the whole content of the blob, and this behavior will consume lots of memory and may cause OOM. We will introduce a stream version function ("stream_loose_object()") in later commit to resolve this issue.
Before introducing that streaming function, do some refactoring on "write_loose_object()" to reuse code for both versions.
Rewrite "write_loose_object()" as follows:
1. Figure out a path for the (temp) object file. This step is only used in "write_loose_object()".
2. Move common steps for starting to write loose objects into a new function "start_loose_object_common()".
3. Compress data.
4. Move common steps for ending zlib stream into a new function
"end_loose_object_common()".
5. Close fd and finalize the object file.
--- object-file.c | 129 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 89 insertions(+), 40 deletions(-)
object-file.c +89 -40
@@ -1943,6 +1943,87 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)return fd; } +/** +* Common steps for loose object writers to start writing loose +* objects: +* +* - Create tmpfile for the loose object. +* - Setup zlib stream for compression. +* - Start to feed header to zlib stream. +* +* Returns a "fd", which should later be provided to +* end_loose_object_common(). +*/ +static int start_loose_object_common (struct strbuf *tmp_file, +const char *filename,unsigned flags, +git_zstream *stream, +unsigned char *buf,size_t buflen, +git_hash_ctx *c, +char *hdr,int hdrlen) +{ +int fd; + + fd =create_tmpfile (tmp_file, filename); +if (fd <0 ) { +if (flags &HASH_SILENT ) +return -1 ; +else if (errno ==EACCES ) +return error (_ ("insufficient permission for adding " +"an object to repository database %s" ), +get_object_directory ()); +else +return error_errno ( +_ ("unable to create temporary file" )); + } + +/* Setup zlib stream for compression */ +git_deflate_init (stream, zlib_compression_level); + stream->next_out = buf; + stream->avail_out = buflen; + the_hash_algo->init_fn (c); + +/* Start to feed header to zlib stream */ + stream->next_in = (unsigned char *)hdr; + stream->avail_in = hdrlen; +while (git_deflate (stream,0 ) ==Z_OK ) + ;/* nothing */ + the_hash_algo->update_fn (c, hdr, hdrlen); + +return fd; +} + +/** +* Common steps for loose object writers to end writing loose objects: +* +* - End the compression of zlib stream. +* - Get the calculated oid to "parano_oid". +* - fsync() and close() the "fd" +*/ +static void end_loose_object_common (int fd,int ret,git_hash_ctx *c, +git_zstream *stream, +struct object_id *parano_oid, +const struct object_id *expected_oid, +const char *die_msg1_fmt, +const char *die_msg2_fmt) +{ +if (ret !=Z_STREAM_END ) +die (_ (die_msg1_fmt), ret, expected_oid); + ret =git_deflate_end_gently (stream); +if (ret !=Z_OK ) +die (_ (die_msg2_fmt), ret, expected_oid); + the_hash_algo->final_oid_fn (parano_oid, c); + +/* +* We already did a write_buffer() to the "fd", let's fsync() +* and close(). +* +* We might still die() on a subsequent sanity check, but +* let's not add to that confusion by not flushing any +* outstanding writes to disk first. +*/ +close_loose_object (fd); +} +static int write_loose_object (const struct object_id *oid,char *hdr,int hdrlen,const void *buf,unsigned long len,time_t mtime,unsigned flags)
@@ -1957,28 +2038,11 @@ static int write_loose_object(const struct object_id *oid, char *hdr,loose_object_path (the_repository, &filename, oid); - fd =create_tmpfile (&tmp_file, filename.buf); -if (fd <0 ) { -if (flags &HASH_SILENT ) -return -1 ; -else if (errno ==EACCES ) -return error (_ ("insufficient permission for adding an object to repository database %s" ),get_object_directory ()); -else -return error_errno (_ ("unable to create temporary file" )); - } - -/* Set it up */ -git_deflate_init (&stream, zlib_compression_level); - stream.next_out = compressed; - stream.avail_out =sizeof (compressed); - the_hash_algo->init_fn (&c); - -/* First header.. */ - stream.next_in = (unsigned char *)hdr; - stream.avail_in = hdrlen; -while (git_deflate (&stream,0 ) ==Z_OK ) - ;/* nothing */ - the_hash_algo->update_fn (&c, hdr, hdrlen); + fd =start_loose_object_common (&tmp_file, filename.buf, flags, + &stream, compressed,sizeof (compressed), + &c, hdr, hdrlen); +if (fd <0 ) +return -1 ;/* Then the data itself.. */ stream.next_in = (void *)buf;
@@ -1993,24 +2057,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr, stream.avail_out =sizeof (compressed); }while (ret ==Z_OK ); -if (ret !=Z_STREAM_END ) -die (_ ("unable to deflate new object %s (%d)" ),oid_to_hex (oid), - ret); - ret =git_deflate_end_gently (&stream); -if (ret !=Z_OK ) -die (_ ("deflateEnd on object %s failed (%d)" ),oid_to_hex (oid), - ret); - the_hash_algo->final_oid_fn (¶no_oid, &c); - -/* -* We already did a write_buffer() to the "fd", let's fsync() -* and close(). -* -* We might still die() on a subsequent sanity check, but -* let's not add to that confusion by not flushing any -* outstanding writes to disk first. -*/ -close_loose_object (fd); +end_loose_object_common (fd, ret, &c, &stream, ¶no_oid, oid, +N_ ("unable to deflate new object %s (%d)" ), +N_ ("deflateEnd on object %s failed (%d)" ));if (!oideq (oid, ¶no_oid)) die(_("confused by unstable object source data for %s"),
Signature · 1 line
2.35.1.1438.g8874c8eeb35
Re: [PATCH v11 3/8] object-file.c: refactor write_loose_object() to several steps
Am 19.03.22 um 01:23 schrieb Ævar Arnfjörð Bjarmason:
Quoted 100 lines
From: Han Xin <[email protected]>
When writing a large blob using "
write_loose_object()", we have to pass a buffer with the whole content of the blob, and this behavior will consume lots of memory and may cause OOM. We will introduce a stream version function ("stream_loose_object()") in later commit to resolve this issue.Before introducing that streaming function, do some refactoring on "
write_loose_object()" to reuse code for both versions.Rewrite "
write_loose_object()" as follows:1. Figure out a path for the (temp) object file. This step is only used in "
write_loose_object()".2. Move common steps for starting to write loose objects into a new function "
start_loose_object_common()".3. Compress data.
4. Move common steps for ending zlib stream into a new function
"end_loose_object_common()".5. Close fd and finalize the object file.
Helped-by Jiang Xin <[email protected]>Signed-off-by Han Xin <[email protected]>Signed-off-by Ævar Arnfjörð Bjarmason <[email protected]>--- object-file.c | 129 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 89 insertions(+), 40 deletions(-)object-file.c +58
-0@@ -1943,6 +1943,87 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)return fd; } +/** +* Common steps for loose object writers to start writing loose +* objects: +* +* - Create tmpfile for the loose object. +* - Setup zlib stream for compression. +* - Start to feed header to zlib stream. +* +* Returns a "fd", which should later be provided to +* end_loose_object_common(). +*/ +static int start_loose_object_common (struct strbuf *tmp_file, +const char *filename,unsigned flags, +git_zstream *stream, +unsigned char *buf,size_t buflen, +git_hash_ctx *c, +char *hdr,int hdrlen) +{ +int fd; + + fd =create_tmpfile (tmp_file, filename); +if (fd <0 ) { +if (flags &HASH_SILENT ) +return -1 ; +else if (errno ==EACCES ) +return error (_ ("insufficient permission for adding " +"an object to repository database %s" ), +get_object_directory ()); +else +return error_errno ( +_ ("unable to create temporary file" )); + } + +/* Setup zlib stream for compression */ +git_deflate_init (stream, zlib_compression_level); + stream->next_out = buf; + stream->avail_out = buflen; + the_hash_algo->init_fn (c); + +/* Start to feed header to zlib stream */ + stream->next_in = (unsigned char *)hdr; + stream->avail_in = hdrlen; +while (git_deflate (stream,0 ) ==Z_OK ) + ;/* nothing */ + the_hash_algo->update_fn (c, hdr, hdrlen); + +return fd; +} + +/** +* Common steps for loose object writers to end writing loose objects: +* +* - End the compression of zlib stream. +* - Get the calculated oid to "parano_oid". +* - fsync() and close() the "fd" +*/ +static void end_loose_object_common(int fd, int ret, git_hash_ctx *c, + git_zstream *stream,
object-file.c static int create_tmpfile(struct strbuf *tmp, const char *filename) · line 2013
Quoted 10 lines
object-file.c +10
-0+struct object_id *parano_oid, + const struct object_id *expected_oid, + const char *die_msg1_fmt, + const char *die_msg2_fmt) +{ + if (ret != Z_STREAM_END) + die(_(die_msg1_fmt), ret, expected_oid); + ret =git_deflate_end_gently (stream); +if (ret !=Z_OK ) +die (_ (die_msg2_fmt), ret, expected_oid);
stream_loose_object(), added in patch 5, passes NULL as expected_oid, but these die() messages need a valid value. end_loose_object_common() has more parameters than lines of code in its body. Inline it to allow fully custom messages?
Quoted 78 lines
object-file.c +21
-40+ the_hash_algo->final_oid_fn (parano_oid, c); + +/* +* We already did a write_buffer() to the "fd", let's fsync() +* and close(). +* +* We might still die() on a subsequent sanity check, but +* let's not add to that confusion by not flushing any +* outstanding writes to disk first. +*/ +close_loose_object (fd); +} +static int write_loose_object (const struct object_id *oid,char *hdr,int hdrlen,const void *buf,unsigned long len,time_t mtime,unsigned flags)@@ -1957,28 +2038,11 @@ static int write_loose_object(const struct object_id *oid, char *hdr,loose_object_path (the_repository, &filename, oid); - fd =create_tmpfile (&tmp_file, filename.buf); -if (fd <0 ) { -if (flags &HASH_SILENT ) -return -1 ; -else if (errno ==EACCES ) -return error (_ ("insufficient permission for adding an object to repository database %s" ),get_object_directory ()); -else -return error_errno (_ ("unable to create temporary file" )); - } - -/* Set it up */ -git_deflate_init (&stream, zlib_compression_level); - stream.next_out = compressed; - stream.avail_out =sizeof (compressed); - the_hash_algo->init_fn (&c); - -/* First header.. */ - stream.next_in = (unsigned char *)hdr; - stream.avail_in = hdrlen; -while (git_deflate (&stream,0 ) ==Z_OK ) - ;/* nothing */ - the_hash_algo->update_fn (&c, hdr, hdrlen); + fd =start_loose_object_common (&tmp_file, filename.buf, flags, + &stream, compressed,sizeof (compressed), + &c, hdr, hdrlen); +if (fd <0 ) +return -1 ;/* Then the data itself.. */ stream.next_in = (void *)buf;@@ -1993,24 +2057,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr, stream.avail_out =sizeof (compressed); }while (ret ==Z_OK ); -if (ret !=Z_STREAM_END ) -die (_ ("unable to deflate new object %s (%d)" ),oid_to_hex (oid), - ret); - ret =git_deflate_end_gently (&stream); -if (ret !=Z_OK ) -die (_ ("deflateEnd on object %s failed (%d)" ),oid_to_hex (oid), - ret); - the_hash_algo->final_oid_fn (¶no_oid, &c); - -/* -* We already did a write_buffer() to the "fd", let's fsync() -* and close(). -* -* We might still die() on a subsequent sanity check, but -* let's not add to that confusion by not flushing any -* outstanding writes to disk first. -*/ -close_loose_object (fd); +end_loose_object_common (fd, ret, &c, &stream, ¶no_oid, oid, +N_ ("unable to deflate new object %s (%d)" ), +N_ ("deflateEnd on object %s failed (%d)" ));if (!oideq (oid, ¶no_oid)) die(_("confused by unstable object source data for %s"),
PATCH v11 4/8 object-file.c: factor out deflate part of write_loose_object()
In reply to Ævar Arnfjörð BjarmasonSplit out the part of write_loose_object() that deals with calling git_deflate() into a utility function, a subsequent commit will introduce another function that'll make use of it.
--- object-file.c | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-)
object-file.c +25 -6
@@ -1992,6 +1992,28 @@ static int start_loose_object_common(struct strbuf *tmp_file,return fd; } +/** +* Common steps for the inner git_deflate() loop for writing loose +* objects. Returns what git_deflate() returns. +*/ +static int write_loose_object_common (git_hash_ctx *c, +git_zstream *stream,const int flush, +unsigned char *in0,const int fd, +unsigned char *compressed, +const size_t compressed_len) +{ +int ret; + + ret =git_deflate (stream, flush ?Z_FINISH :0 ); + the_hash_algo->update_fn (c, in0, stream->next_in - in0); +if (write_buffer (fd, compressed, stream->next_out - compressed) <0 ) +die (_ ("unable to write loose object file" )); + stream->next_out = compressed; + stream->avail_out = compressed_len; + +return ret; +} +/** * Common steps for loose object writers to end writing loose objects: *
@@ -2049,12 +2071,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr, stream.avail_in = len;do {unsigned char *in0 = stream.next_in; - ret =git_deflate (&stream,Z_FINISH ); - the_hash_algo->update_fn (&c, in0, stream.next_in - in0); -if (write_buffer (fd, compressed, stream.next_out - compressed) <0 ) -die (_ ("unable to write loose object file" )); - stream.next_out = compressed; - stream.avail_out =sizeof (compressed); + + ret =write_loose_object_common (&c, &stream,1 , in0, fd, + compressed,sizeof (compressed)); }while (ret ==Z_OK ); end_loose_object_common(fd, ret, &c, &stream, ¶no_oid, oid,
Signature · 1 line
2.35.1.1438.g8874c8eeb35
PATCH v11 5/8 object-file.c: add "stream_loose_object()" to handle large object
In reply to Ævar Arnfjörð BjarmasonFrom: Han Xin <[email protected]>
If we want unpack and write a loose object using "write_loose_object", we have to feed it with a buffer with the same size of the object, which will consume lots of memory and may cause OOM. This can be improved by feeding data to "stream_loose_object()" in a stream.
Add a new function "stream_loose_object()", which is a stream version of "write_loose_object()" but with a low memory footprint. We will use this function to unpack large blob object in later commit.
Another difference with "write_loose_object()" is that we have no chance to run "write_object_file_prepare()" to calculate the oid in advance. In "write_loose_object()", we know the oid and we can write the temporary file in the same directory as the final object, but for an object with an undetermined oid, we don't know the exact directory for the object.
Still, we need to save the temporary file we're preparing somewhere. We'll do that in the top-level ".git/objects/" directory (or whatever "GIT_OBJECT_DIRECTORY" is set to). Once we've streamed it we'll know the OID, and will move it to its canonical path.
"freshen_packed_object()" or "freshen_loose_object()" will be called inside "stream_loose_object()" after obtaining the "oid".
--- object-file.c | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++ object-store.h | 8 +++++ 2 files changed, 105 insertions(+)
object-file.c +97 -0
@@ -2114,6 +2114,103 @@ static int freshen_packed_object(const struct object_id *oid)return 1 ; } +int stream_loose_object (struct input_stream *in_stream,size_t len, +struct object_id *oid) +{ +int fd, ret, err =0 , flush =0 ; +unsigned char compressed[4096 ]; +git_zstream stream; +git_hash_ctx c; +struct strbuf tmp_file =STRBUF_INIT ; +struct strbuf filename =STRBUF_INIT ; +int dirlen; +char hdr[MAX_HEADER_LEN ]; +int hdrlen; + +/* Since oid is not determined, save tmp file to odb path. */ +strbuf_addf (&filename,"%s/" ,get_object_directory ()); + hdrlen =format_object_header (hdr,sizeof (hdr),OBJ_BLOB , len); + +/* +* Common steps for write_loose_object and stream_loose_object to +* start writing loose objects: +* +* - Create tmpfile for the loose object. +* - Setup zlib stream for compression. +* - Start to feed header to zlib stream. +*/ + fd =start_loose_object_common (&tmp_file, filename.buf,0 , + &stream, compressed,sizeof (compressed), + &c, hdr, hdrlen); +if (fd <0 ) { + err =-1 ; + goto cleanup; + } + +/* Then the data itself.. */ +do { +unsigned char *in0 = stream.next_in; + +if (!stream.avail_in && !in_stream->is_finished) { +const void *in = in_stream->read (in_stream, &stream.avail_in); + stream.next_in = (void *)in; + in0 = (unsigned char *)in; +/* All data has been read. */ +if (in_stream->is_finished) + flush =1 ; + } + ret =write_loose_object_common (&c, &stream, flush, in0, fd, + compressed,sizeof (compressed)); +/* +* Unlike write_loose_object(), we do not have the entire +* buffer. If we get Z_BUF_ERROR due to too few input bytes, +* then we'll replenish them in the next input_stream->read() +* call when we loop. +*/ + }while (ret ==Z_OK || ret ==Z_BUF_ERROR ); + +if (stream.total_in != len + hdrlen) +die (_ ("write stream object %ld != %" PRIuMAX), stream.total_in, + (uintmax_t )len + hdrlen); + +/* Common steps for write_loose_object and stream_loose_object to +* end writing loose oject: +* +* - End the compression of zlib stream. +* - Get the calculated oid. +*/ +end_loose_object_common (fd, ret, &c, &stream, oid,NULL , +N_ ("unable to stream deflate new object (%d)" ), +N_ ("deflateEnd on stream object failed (%d)" )); + +if (freshen_packed_object (oid) ||freshen_loose_object (oid)) { +unlink_or_warn (tmp_file.buf); + goto cleanup; + } + +loose_object_path (the_repository, &filename, oid); + +/* We finally know the object path, and create the missing dir. */ + dirlen =directory_size (filename.buf); +if (dirlen) { +struct strbuf dir =STRBUF_INIT ; +strbuf_add (&dir, filename.buf, dirlen); + +if (mkdir_in_gitdir (dir.buf) && errno !=EEXIST ) { + err =error_errno (_ ("unable to create directory %s" ), dir.buf); +strbuf_release (&dir); + goto cleanup; + } +strbuf_release (&dir); + } + + err =finalize_object_file (tmp_file.buf, filename.buf); +cleanup: +strbuf_release (&tmp_file); +strbuf_release (&filename); +return err; +} +int write_object_file_flags (const void *buf,unsigned long len,enum object_type type,struct object_id *oid,unsigned flags)
object-store.h +8 -0
@@ -46,6 +46,12 @@ struct object_directory {char *path; }; +struct input_stream { +const void *(*read)(struct input_stream *,unsigned long *len); +void *data; +int is_finished; +}; + KHASH_INIT(odb_path_map, const char * /* key: odb_path */, struct object_directory *, 1, fspathhash, fspatheq)
@@ -261,6 +267,8 @@ static inline int write_object_file(const void *buf, unsigned long len,int write_object_file_literally (const void *buf,unsigned long len,const char *type,struct object_id *oid,unsigned flags); +int stream_loose_object (struct input_stream *in_stream,size_t len, +struct object_id *oid);/* * Add an object file to the in-memory object store, without writing it
Signature · 1 line
2.35.1.1438.g8874c8eeb35
PATCH v11 6/8 core doc: modernize core.bigFileThreshold documentation
In reply to Ævar Arnfjörð BjarmasonThe core.bigFileThreshold documentation has been largely unchanged since 5eef828bc03 (fast-import: Stream very large blobs directly to pack, 2010-02-01).
But since then this setting has been expanded to affect a lot more than that description indicated. Most notably in how "git diff" treats them, see 6bf3b813486 (diff --stat: mark any file larger than core.bigfilethreshold binary, 2014-08-16).
In addition to that, numerous commands and APIs make use of a streaming mode for files above this threshold.
So let's attempt to summarize 12 years of changes in behavior, which can be seen with:
git log --oneline -Gbig_file_thre 5eef828bc03.. -- '*.c'
To do that turn this into a bullet-point list. The summary Han Xin produced in [1] helped a lot, but is a bit too detailed for documentation aimed at users. Let's instead summarize how user-observable behavior differs, and generally describe how we tend to stream these files in various commands.
1. https://lore.kernel.org/git/[email protected]/
--- Documentation/config/core.txt | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-)
Documentation/config/core.txt +24 -9
@@ -412,17 +412,32 @@ You probably do not need to adjust this value. Common unit suffixes of 'k', 'm', or 'g' are supported. core.bigFileThreshold:: - Files larger than this size are stored deflated, without - attempting delta compression. Storing large files without - delta compression avoids excessive memory usage, at the - slight expense of increased disk usage. Additionally files - larger than this size are always treated as binary. + The size of files considered "big", which as discussed below + changes the behavior of numerous git commands, as well as how + such files are stored within the repository. The default is + 512 MiB. Common unit suffixes of 'k', 'm', or 'g' are + supported. + -Default is 512 MiB on all platforms. This should be reasonable -for most projects as source code and other text files can still -be delta compressed, but larger binary media files won't be. +Files above the configured limit will be: + -Common unit suffixes of 'k', 'm', or 'g' are supported. +* Stored deflated, without attempting delta compression. ++ +The default limit is primarily set with this use-case in mind. With it +most projects will have their source code and other text files delta +compressed, but not larger binary media files. ++ +Storing large files without delta compression avoids excessive memory +usage, at the slight expense of increased disk usage. ++ +* Will be treated as if though they were labeled "binary" (see + linkgit:gitattributes[5]). This means that e.g. linkgit:git-log[1] + and linkgit:git-diff[1] will not diffs for files above this limit. ++ +* Will be generally be streamed when written, which avoids excessive +memory usage, at the cost of some fixed overhead. Commands that make +use of this include linkgit:git-archive[1], +linkgit:git-fast-import[1], linkgit:git-index-pack[1] and +linkgit:git-fsck[1]. core.excludesFile:: Specifies the pathname to the file that contains patterns to
Signature · 1 line
2.35.1.1438.g8874c8eeb35
PATCH v11 7/8 unpack-objects: refactor away unpack_non_delta_entry()
In reply to Ævar Arnfjörð BjarmasonThe unpack_one() function will call either a non-trivial unpack_delta_entry() or a trivial unpack_non_delta_entry(). Let's inline the latter in the only caller.
Since 21666f1aae4 (convert object type handling from a string to a number, 2007-02-26) the unpack_non_delta_entry() function has been rather trivial, and in a preceding commit the "dry_run" condition it was handling went away.
This is not done as an optimization, as the compiler will easily discover that it can do the same, rather this makes a subsequent commit easier to reason about. As it'll be handling "OBJ_BLOB" in a special manner let's re-arrange that "case" in preparation for that change.
--- builtin/unpack-objects.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-)
builtin/unpack-objects.c +7 -11
@@ -338,15 +338,6 @@ static void added_object(unsigned nr, enum object_type type, } } -static void unpack_non_delta_entry (enum object_type type,unsigned long size, -unsigned nr) -{ -void *buf =get_data (size); - -if (buf) -write_object (nr, type, buf, size); -} -static int resolve_against_held (unsigned nr,const struct object_id *base,void *delta_data,unsigned long delta_size) {
@@ -479,12 +470,17 @@ static void unpack_one(unsigned nr) }switch (type) { +case OBJ_BLOB :case OBJ_COMMIT :case OBJ_TREE : -case OBJ_BLOB :case OBJ_TAG : -unpack_non_delta_entry (type, size, nr); + { +void *buf =get_data (size); + +if (buf) +write_object (nr, type, buf, size);return ; + }case OBJ_REF_DELTA :case OBJ_OFS_DELTA :unpack_delta_entry (type, size, nr);
Signature · 1 line
2.35.1.1438.g8874c8eeb35
PATCH v11 8/8 unpack-objects: use stream_loose_object() to unpack large objects
In reply to Ævar Arnfjörð BjarmasonFrom: Han Xin <[email protected]>
Make use of the stream_loose_object() function introduced in the preceding commit to unpack large objects. Before this we'd need to malloc() the size of the blob before unpacking it, which could cause OOM with very large blobs.
We could use the new streaming interface to unpack all blobs, but doing so would be much slower, as demonstrated e.g. with this benchmark using git-hyperfine[0]:
rm -rf /tmp/scalar.git && git clone --bare https://github.com/Microsoft/scalar.git /tmp/scalar.git && mv /tmp/scalar.git/objects/pack/*.pack /tmp/scalar.git/my.pack && git hyperfine \ -r 2 --warmup 1 \ -L rev origin/master,HEAD -L v "10,512,1k,1m" \ -s 'make' \ -p 'git init --bare dest.git' \ -c 'rm -rf dest.git' \ './git -C dest.git -c core.bigFileThreshold={v} unpack-objects </tmp/scalar.git/my.pack'
Here we'll perform worse with lower core.bigFileThreshold settings with this change in terms of speed, but we're getting lower memory use in return:
Summary
'./git -C dest.git -c core.bigFileThreshold=10 unpack-objects </tmp/scalar.git/my.pack' in 'origin/master' ran
1.01 ± 0.01 times faster than './git -C dest.git -c core.bigFileThreshold=1k unpack-objects </tmp/scalar.git/my.pack' in 'origin/master' 1.01 ± 0.01 times faster than './git -C dest.git -c core.bigFileThreshold=1m unpack-objects </tmp/scalar.git/my.pack' in 'origin/master' 1.01 ± 0.02 times faster than './git -C dest.git -c core.bigFileThreshold=1m unpack-objects </tmp/scalar.git/my.pack' in 'HEAD' 1.02 ± 0.00 times faster than './git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/scalar.git/my.pack' in 'origin/master' 1.09 ± 0.01 times faster than './git -C dest.git -c core.bigFileThreshold=1k unpack-objects </tmp/scalar.git/my.pack' in 'HEAD' 1.10 ± 0.00 times faster than './git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/scalar.git/my.pack' in 'HEAD' 1.11 ± 0.00 times faster than './git -C dest.git -c core.bigFileThreshold=10 unpack-objects </tmp/scalar.git/my.pack' in 'HEAD'
A better benchmark to demonstrate the benefits of that this one, which creates an artificial repo with a 1, 25, 50, 75 and 100MB blob:
rm -rf /tmp/repo &&
git init /tmp/repo &&
(
cd /tmp/repo &&
for i in 1 25 50 75 100
do
dd if=/dev/urandom of=blob.$i count=$(($i*1024)) bs=1024
done &&
git add blob.* &&
git commit -mblobs &&
git gc &&
PACK=$(echo .git/objects/pack/pack-*.pack) &&
cp "$PACK" my.pack
) &&
git hyperfine \
--show-output \
-L rev origin/master,HEAD -L v "512,50m,100m" \
-s 'make' \
-p 'git init --bare dest.git' \
-c 'rm -rf dest.git' \
'/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold={v} unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum'Using this test we'll always use >100MB of memory on origin/master (around ~105MB), but max out at e.g. ~55MB if we set core.bigFileThreshold=50m.
The relevant "Maximum resident set size" lines were manually added below the relevant benchmark:
'/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=50m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'origin/master' ran
Maximum resident set size (kbytes): 107080
1.02 ± 0.78 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'origin/master'
Maximum resident set size (kbytes): 106968
1.09 ± 0.79 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=100m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'origin/master'
Maximum resident set size (kbytes): 107032
1.42 ± 1.07 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=100m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'HEAD'
Maximum resident set size (kbytes): 107072
1.83 ± 1.02 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=50m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'HEAD'
Maximum resident set size (kbytes): 55704
2.16 ± 1.19 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'HEAD'
Maximum resident set size (kbytes): 4564
This shows that if you have enough memory this new streaming method is slower the lower you set the streaming threshold, but the benefit is more bounded memory use.
An earlier version of this patch introduced a new "core.bigFileStreamingThreshold" instead of re-using the existing "core.bigFileThreshold" variable[1]. As noted in a detailed overview of its users in [2] using it has several different meanings.
Still, we consider it good enough to simply re-use it. While it's possible that someone might want to e.g. consider objects "small" for the purposes of diffing but "big" for the purposes of writing them such use-cases are probably too obscure to worry about. We can always split up "core.bigFileThreshold" in the future if there's a need for that.
0. https://github.com/avar/git-hyperfine/
1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/
--- Documentation/config/core.txt | 4 +- builtin/unpack-objects.c | 67 +++++++++++++++++++++++++++++++++ t/t5351-unpack-large-objects.sh | 26 +++++++++++-- 3 files changed, 92 insertions(+), 5 deletions(-)
Documentation/config/core.txt +2 -2
@@ -436,8 +436,8 @@ usage, at the slight expense of increased disk usage. * Will be generally be streamed when written, which avoids excessive memory usage, at the cost of some fixed overhead. Commands that make use of this include linkgit:git-archive[1], -linkgit:git-fast-import[1], linkgit:git-index-pack[1] and -linkgit:git-fsck[1]. +linkgit:git-fast-import[1], linkgit:git-index-pack[1], +linkgit:git-unpack-objects[1] and linkgit:git-fsck[1]. core.excludesFile:: Specifies the pathname to the file that contains patterns to
builtin/unpack-objects.c +67 -0
79 lines not on this page · raw message
t/t5351-unpack-large-objects.sh +23 -3
@@ -9,7 +9,8 @@ test_description='git unpack-objects with large objects' prepare_dest () { test_when_finished "rm -rf dest.git" && - git init --bare dest.git + git init --bare dest.git && + git -C dest.git config core.bigFileThreshold "$1" } test_expect_success "create large objects (1.5 MB) and PACK" '
@@ -26,16 +27,35 @@ test_expect_success 'set memory limitation to 1MB' ' ' test_expect_success 'unpack-objects failed under memory limitation' ' - prepare_dest && + prepare_dest 2m && test_must_fail git -C dest.git unpack-objects <pack-$PACK.pack 2>err && grep "fatal: attempting to allocate" err ' test_expect_success 'unpack-objects works with memory limitation in dry-run mode' ' - prepare_dest && + prepare_dest 2m && git -C dest.git unpack-objects -n <pack-$PACK.pack && test_stdout_line_count = 0 find dest.git/objects -type f && test_dir_is_empty dest.git/objects/pack ' +test_expect_success 'unpack big object in stream' ' + prepare_dest 1m && + git -C dest.git unpack-objects <pack-$PACK.pack && + test_dir_is_empty dest.git/objects/pack +' + +test_expect_success 'do not unpack existing large objects' ' + prepare_dest 1m && + git -C dest.git index-pack --stdin <pack-$PACK.pack && + git -C dest.git unpack-objects <pack-$PACK.pack && + + # The destination came up with the exact same pack... + DEST_PACK=$(echo dest.git/objects/pack/pack-*.pack) && + test_cmp pack-$PACK.pack $DEST_PACK && + + # ...and wrote no loose objects + test_stdout_line_count = 0 find dest.git/objects -type f ! -name "pack-*" +' + test_done
Signature · 1 line
2.35.1.1438.g8874c8eeb35
v12 cover unpack-objects: support streaming blobs to disk
In reply to Ævar Arnfjörð BjarmasonThis series by Han Xin was waiting on some in-flight patches that landed in 430883a70c7 (Merge branch 'ab/object-file-api-updates', 2022-03-16).
This series teaches "git unpack-objects" to stream objects larger than core.bigFileThreshold to disk. As 8/8 shows streaming e.g. a 100MB blob now uses ~5MB of memory instead of ~105MB. This streaming method is slower if you've got memory to handle the blobs in-core, but if you don't it allows you to unpack objects at all, as you might otherwise OOM.
Changes since v10[1]:
* René rightly spotted that the end_loose_object_common() function was feeding NULL to a format. That's now fixed, and parts of that function were pulled out into the two callers to make the trade-off of factoring that logic out worth it.
* This topic conflicts with ns/batch-fsync in "seen" (see below). I moved an inline comment on close_loose_object() around to make the conflict easier (and it's better placed with the function anyway, as we'll get two callers of it).
Conflict this is the --remerge-diff with "seen" after resolving the conflict. Botht textual and semantic (there's a new caller in this topic) conflicts are caught by the compiler:
diff --git a/object-file.c b/object-file.c
remerge CONFLICT (content): Merge conflict in object-file.c
index 9c640f1f39d..6068f8ec6c4 100644
--- a/object-file.c
+++ b/object-file.c
@@ -1887,7 +1887,6 @@ void hash_object_file(const struct git_hash_algo *algo, const void *buf,
hash_object_file_literally(algo, buf, len, type_name(type), oid);
}
-<<<<<<< 34ee6a28a54 (unpack-objects: use stream_loose_object() to unpack large objects)
/*
* We already did a write_buffer() to the "fd", let's fsync()
* and close().
@@ -1896,11 +1895,7 @@ void hash_object_file(const struct git_hash_algo *algo, const void *buf,
* subsequent sanity check, but let's not add to that confusion by not
* flushing any outstanding writes to disk first.
*/
-static void close_loose_object(int fd)
-=======
-/* Finalize a file on disk, and close it. */
static void close_loose_object(int fd, const char *filename)
->>>>>>> b1423c89b5a (Merge branch 'ab/reftable-aix-xlc-12' into seen)
{
if (the_repository->objects->odb->will_destroy)
goto out;
@@ -2093,17 +2088,12 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
ret = end_loose_object_common(&c, &stream, ¶no_oid);
if (ret != Z_OK)
die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid), ret);
- close_loose_object(fd);
+ close_loose_object(fd, tmp_file.buf);
if (!oideq(oid, ¶no_oid))
die(_("confused by unstable object source data for %s"),
oid_to_hex(oid));
-<<<<<<< 34ee6a28a54 (unpack-objects: use stream_loose_object() to unpack large objects)
-=======
- close_loose_object(fd, tmp_file.buf);
-
->>>>>>> b1423c89b5a (Merge branch 'ab/reftable-aix-xlc-12' into seen)
if (mtime) {
struct utimbuf utb;
utb.actime = mtime;
@@ -2206,7 +2196,7 @@ int stream_loose_object(struct input_stream *in_stream, size_t len,
ret = end_loose_object_common(&c, &stream, oid);
if (ret != Z_OK)
die(_("deflateEnd on stream object failed (%d)"), ret);
- close_loose_object(fd);
+ close_loose_object(fd, tmp_file.buf);
if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
unlink_or_warn(tmp_file.buf);1. https://lore.kernel.org/git/[email protected]/
Han Xin (4):
unpack-objects: low memory footprint for get_data() in dry_run mode object-file.c: refactor write_loose_object() to several steps object-file.c: add "stream_loose_object()" to handle large object
unpack-objects: use stream_loose_object() to unpack large objects
Ævar Arnfjörð Bjarmason (4):
object-file.c: do fsync() and close() before post-write die() object-file.c: factor out deflate part of write_loose_object() core doc: modernize core.bigFileThreshold documentation
unpack-objects: refactor away unpack_non_delta_entry()
Documentation/config/core.txt | 33 +++-- builtin/unpack-objects.c | 109 +++++++++++--- object-file.c | 246 +++++++++++++++++++++++++++----- object-store.h | 8 ++ t/t5351-unpack-large-objects.sh | 61 ++++++++ 5 files changed, 396 insertions(+), 61 deletions(-) create mode 100755 t/t5351-unpack-large-objects.sh
Range-diff against v11:
1: 2103d5bfd96 = 1: e95f6a1cfb6 unpack-objects: low memory footprint for get_data() in dry_run mode 2: 6acd8759772 ! 2: 54060eb8c6b object-file.c: do fsync() and close() before post-write die()
@@ Commit message
## object-file.c ##
+@@ object-file.c: void hash_object_file(const struct git_hash_algo *algo, const void *buf,
+ hash_object_file_literally(algo, buf, len, type_name(type), oid);
+ }
+
+-/* Finalize a file on disk, and close it. */
++/*
++ * We already did a write_buffer() to the "fd", let's fsync()
++ * and close().
++ *
++ * Finalize a file on disk, and close it. We might still die() on a
++ * subsequent sanity check, but let's not add to that confusion by not
++ * flushing any outstanding writes to disk first.
++ */
+ static void close_loose_object(int fd)
+ {
+ if (the_repository->objects->odb->will_destroy)
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *hdr,
die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
ret);
the_hash_algo->final_oid_fn(¶no_oid, &c);
-+
-+ /*
-+ * We already did a write_buffer() to the "fd", let's fsync()
-+ * and close().
-+ *
-+ * We might still die() on a subsequent sanity check, but
-+ * let's not add to that confusion by not flushing any
-+ * outstanding writes to disk first.
-+ */
+ close_loose_object(fd);
+
if (!oideq(oid, ¶no_oid))
3: f7b02c307fc ! 3: 3dcaa5d6589 object-file.c: refactor write_loose_object() to several steps
@@ object-file.c: static int create_tmpfile(struct strbuf *tmp, const char *filenam
+ * Common steps for loose object writers to end writing loose objects:
+ *
+ * - End the compression of zlib stream.
-+ * - Get the calculated oid to "parano_oid".
++ * - Get the calculated oid to "oid".
+ * - fsync() and close() the "fd"
+ */
-+static void end_loose_object_common(int fd, int ret, git_hash_ctx *c,
-+ git_zstream *stream,
-+ struct object_id *parano_oid,
-+ const struct object_id *expected_oid,
-+ const char *die_msg1_fmt,
-+ const char *die_msg2_fmt)
++static int end_loose_object_common(git_hash_ctx *c, git_zstream *stream,
++ struct object_id *oid)
+{
-+ if (ret != Z_STREAM_END)
-+ die(_(die_msg1_fmt), ret, expected_oid);
++ int ret;
++
+ ret = git_deflate_end_gently(stream);
+ if (ret != Z_OK)
-+ die(_(die_msg2_fmt), ret, expected_oid);
-+ the_hash_algo->final_oid_fn(parano_oid, c);
++ return ret;
++ the_hash_algo->final_oid_fn(oid, c);
+
-+ /*
-+ * We already did a write_buffer() to the "fd", let's fsync()
-+ * and close().
-+ *
-+ * We might still die() on a subsequent sanity check, but
-+ * let's not add to that confusion by not flushing any
-+ * outstanding writes to disk first.
-+ */
-+ close_loose_object(fd);
++ return Z_OK;
+}
+
static int write_loose_object(const struct object_id *oid, char *hdr,
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
/* Then the data itself.. */
stream.next_in = (void *)buf;
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *hdr,
- stream.avail_out = sizeof(compressed);
- } while (ret == Z_OK);
-
-- if (ret != Z_STREAM_END)
-- die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
-- ret);
+ if (ret != Z_STREAM_END)
+ die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
+ ret);
- ret = git_deflate_end_gently(&stream);
-- if (ret != Z_OK)
++ ret = end_loose_object_common(&c, &stream, ¶no_oid);
+ if (ret != Z_OK)
- die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
- ret);
- the_hash_algo->final_oid_fn(¶no_oid, &c);
--
-- /*
-- * We already did a write_buffer() to the "fd", let's fsync()
-- * and close().
-- *
-- * We might still die() on a subsequent sanity check, but
-- * let's not add to that confusion by not flushing any
-- * outstanding writes to disk first.
-- */
-- close_loose_object(fd);
-+ end_loose_object_common(fd, ret, &c, &stream, ¶no_oid, oid,
-+ N_("unable to deflate new object %s (%d)"),
-+ N_("deflateEnd on object %s failed (%d)"));
++ die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid), ret);
+ close_loose_object(fd);
if (!oideq(oid, ¶no_oid))
- die(_("confused by unstable object source data for %s"),4: 20d97cc2605 ! 4: 03f4e91ac89 object-file.c: factor out deflate part of write_loose_object()
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
+ compressed, sizeof(compressed));
} while (ret == Z_OK);
- end_loose_object_common(fd, ret, &c, &stream, ¶no_oid, oid,
+ if (ret != Z_STREAM_END)
5: db40f4160c4 ! 5: 3d64cf1cf33 object-file.c: add "stream_loose_object()" to handle large object
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
+ * - End the compression of zlib stream.
+ * - Get the calculated oid.
+ */
-+ end_loose_object_common(fd, ret, &c, &stream, oid, NULL,
-+ N_("unable to stream deflate new object (%d)"),
-+ N_("deflateEnd on stream object failed (%d)"));
++ if (ret != Z_STREAM_END)
++ die(_("unable to stream deflate new object (%d)"), ret);
++ ret = end_loose_object_common(&c, &stream, oid);
++ if (ret != Z_OK)
++ die(_("deflateEnd on stream object failed (%d)"), ret);
++ close_loose_object(fd);
+
+ if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
+ unlink_or_warn(tmp_file.buf);
6: d8ae2eadb98 = 6: 33ffcbbc1f0 core doc: modernize core.bigFileThreshold documentation
7: 2b403e7cd9c = 7: 11f7aa026b4 unpack-objects: refactor away unpack_non_delta_entry()
8: 5eded902496 = 8: 34ee6a28a54 unpack-objects: use stream_loose_object() to unpack large objectsSignature · 1 line
2.35.1.1548.g36973b18e52
PATCH v12 1/8 unpack-objects: low memory footprint for get_data() in dry_run mode
From: Han Xin <[email protected]>
As the name implies, "get_data(size)" will allocate and return a given amount of memory. Allocating memory for a large blob object may cause the system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint for dry_run mode.
Because in dry_run mode, "get_data()" is only used to check the integrity of data, and the returned buffer is not used at all, we can allocate a smaller buffer and reuse it as zstream output. Therefore, in dry_run mode, "get_data()" will release the allocated buffer and return NULL instead of returning garbage data.
The "find [...]objects/?? -type f | wc -l" test idiom being used here is adapted from the same "find" use added to another test in d9545c7f465 (fast-import: implement unpack limit, 2016-04-25).
--- builtin/unpack-objects.c | 34 ++++++++++++++++++--------- t/t5351-unpack-large-objects.sh | 41 +++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 11 deletions(-) create mode 100755 t/t5351-unpack-large-objects.sh
builtin/unpack-objects.c +23 -11
@@ -96,15 +96,26 @@ static void use(int bytes)display_throughput (progress, consumed_bytes); } +/* +* Decompress zstream from stdin and return specific size of data. +* The caller is responsible to free the returned buffer. +* +* But for dry_run mode, "get_data()" is only used to check the +* integrity of data, and the returned buffer is not used at all. +* Therefore, in dry_run mode, "get_data()" will release the small +* allocated buffer which is reused to hold temporary zstream output +* and return NULL instead of returning garbage data. +*/ static void *get_data (unsigned long size) {git_zstream stream; -void *buf =xmallocz (size); +unsigned long bufsize = dry_run && size >8192 ?8192 : size; +void *buf =xmallocz (bufsize);memset (&stream,0 ,sizeof (stream)); stream.next_out = buf; - stream.avail_out = size; + stream.avail_out = bufsize; stream.next_in =fill (1 ); stream.avail_in = len;git_inflate_init (&stream);
@@ -124,8 +135,15 @@ static void *get_data(unsigned long size) } stream.next_in =fill (1 ); stream.avail_in = len; +if (dry_run) { +/* reuse the buffer in dry_run mode */ + stream.next_out = buf; + stream.avail_out = bufsize; + } }git_inflate_end (&stream); +if (dry_run) +FREE_AND_NULL (buf);return buf; }
@@ -325,10 +343,8 @@ static void unpack_non_delta_entry(enum object_type type, unsigned long size, {void *buf =get_data (size); -if (!dry_run && buf) +if (buf)write_object (nr, type, buf, size); -else -free (buf); } static int resolve_against_held(unsigned nr, const struct object_id *base,
@@ -358,10 +374,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,oidread (&base_oid,fill (the_hash_algo->rawsz));use (the_hash_algo->rawsz); delta_data =get_data (delta_size); -if (dry_run || !delta_data) { -free (delta_data); +if (!delta_data)return ; - }if (has_object_file (&base_oid)) ;/* Ok we have this one */ else if (resolve_against_held(nr, &base_oid,
@@ -397,10 +411,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,die ("offset value out of bound for delta base object" ); delta_data =get_data (delta_size); -if (dry_run || !delta_data) { -free (delta_data); +if (!delta_data)return ; - } lo =0 ; hi = nr;while (lo < hi) {
t/t5351-unpack-large-objects.sh new file +41 -0
41 lines not on this page · raw message
Signature · 1 line
2.35.1.1548.g36973b18e52
PATCH v12 2/8 object-file.c: do fsync() and close() before post-write die()
In reply to Ævar Arnfjörð BjarmasonChange write_loose_object() to do an fsync() and close() before the oideq() sanity check at the end. This change re-joins code that was split up by the die() sanity check added in 748af44c63e (sha1_file: be paranoid when creating loose objects, 2010-02-21).
I don't think that this change matters in itself, if we called die() it was possible that our data wouldn't fully make it to disk, but in any case we were writing data that we'd consider corrupted. It's possible that a subsequent "git fsck" will be less confused now.
The real reason to make this change is that in a subsequent commit we'll split this code in write_loose_object() into a utility function, all its callers will want the preceding sanity checks, but not the "oideq" check. By moving the close_loose_object() earlier it'll be easier to reason about the introduction of the utility function.
--- object-file.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-)
object-file.c +10 -3
@@ -1886,7 +1886,14 @@ void hash_object_file(const struct git_hash_algo *algo, const void *buf,hash_object_file_literally (algo, buf, len,type_name (type), oid); } -/* Finalize a file on disk, and close it. */ +/* +* We already did a write_buffer() to the "fd", let's fsync() +* and close(). +* +* Finalize a file on disk, and close it. We might still die() on a +* subsequent sanity check, but let's not add to that confusion by not +* flushing any outstanding writes to disk first. +*/ static void close_loose_object (int fd) {if (the_repository->objects->odb->will_destroy)
@@ -2006,12 +2013,12 @@ static int write_loose_object(const struct object_id *oid, char *hdr,die (_ ("deflateEnd on object %s failed (%d)" ),oid_to_hex (oid), ret); the_hash_algo->final_oid_fn (¶no_oid, &c); +close_loose_object (fd); +if (!oideq (oid, ¶no_oid))die (_ ("confused by unstable object source data for %s" ),oid_to_hex (oid)); -close_loose_object (fd); -if (mtime) {struct utimbuf utb; utb.actime = mtime;
Signature · 1 line
2.35.1.1548.g36973b18e52
PATCH v12 3/8 object-file.c: refactor write_loose_object() to several steps
In reply to Ævar Arnfjörð BjarmasonFrom: Han Xin <[email protected]>
When writing a large blob using "write_loose_object()", we have to pass a buffer with the whole content of the blob, and this behavior will consume lots of memory and may cause OOM. We will introduce a stream version function ("stream_loose_object()") in later commit to resolve this issue.
Before introducing that streaming function, do some refactoring on "write_loose_object()" to reuse code for both versions.
Rewrite "write_loose_object()" as follows:
1. Figure out a path for the (temp) object file. This step is only used in "write_loose_object()".
2. Move common steps for starting to write loose objects into a new function "start_loose_object_common()".
3. Compress data.
4. Move common steps for ending zlib stream into a new function
"end_loose_object_common()".
5. Close fd and finalize the object file.
--- object-file.c | 102 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 76 insertions(+), 26 deletions(-)
object-file.c +76 -26
@@ -1955,6 +1955,75 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)return fd; } +/** +* Common steps for loose object writers to start writing loose +* objects: +* +* - Create tmpfile for the loose object. +* - Setup zlib stream for compression. +* - Start to feed header to zlib stream. +* +* Returns a "fd", which should later be provided to +* end_loose_object_common(). +*/ +static int start_loose_object_common (struct strbuf *tmp_file, +const char *filename,unsigned flags, +git_zstream *stream, +unsigned char *buf,size_t buflen, +git_hash_ctx *c, +char *hdr,int hdrlen) +{ +int fd; + + fd =create_tmpfile (tmp_file, filename); +if (fd <0 ) { +if (flags &HASH_SILENT ) +return -1 ; +else if (errno ==EACCES ) +return error (_ ("insufficient permission for adding " +"an object to repository database %s" ), +get_object_directory ()); +else +return error_errno ( +_ ("unable to create temporary file" )); + } + +/* Setup zlib stream for compression */ +git_deflate_init (stream, zlib_compression_level); + stream->next_out = buf; + stream->avail_out = buflen; + the_hash_algo->init_fn (c); + +/* Start to feed header to zlib stream */ + stream->next_in = (unsigned char *)hdr; + stream->avail_in = hdrlen; +while (git_deflate (stream,0 ) ==Z_OK ) + ;/* nothing */ + the_hash_algo->update_fn (c, hdr, hdrlen); + +return fd; +} + +/** +* Common steps for loose object writers to end writing loose objects: +* +* - End the compression of zlib stream. +* - Get the calculated oid to "oid". +* - fsync() and close() the "fd" +*/ +static int end_loose_object_common (git_hash_ctx *c,git_zstream *stream, +struct object_id *oid) +{ +int ret; + + ret =git_deflate_end_gently (stream); +if (ret !=Z_OK ) +return ret; + the_hash_algo->final_oid_fn (oid, c); + +return Z_OK ; +} +static int write_loose_object (const struct object_id *oid,char *hdr,int hdrlen,const void *buf,unsigned long len,time_t mtime,unsigned flags)
@@ -1969,28 +2038,11 @@ static int write_loose_object(const struct object_id *oid, char *hdr,loose_object_path (the_repository, &filename, oid); - fd =create_tmpfile (&tmp_file, filename.buf); -if (fd <0 ) { -if (flags &HASH_SILENT ) -return -1 ; -else if (errno ==EACCES ) -return error (_ ("insufficient permission for adding an object to repository database %s" ),get_object_directory ()); -else -return error_errno (_ ("unable to create temporary file" )); - } - -/* Set it up */ -git_deflate_init (&stream, zlib_compression_level); - stream.next_out = compressed; - stream.avail_out =sizeof (compressed); - the_hash_algo->init_fn (&c); - -/* First header.. */ - stream.next_in = (unsigned char *)hdr; - stream.avail_in = hdrlen; -while (git_deflate (&stream,0 ) ==Z_OK ) - ;/* nothing */ - the_hash_algo->update_fn (&c, hdr, hdrlen); + fd =start_loose_object_common (&tmp_file, filename.buf, flags, + &stream, compressed,sizeof (compressed), + &c, hdr, hdrlen); +if (fd <0 ) +return -1 ;/* Then the data itself.. */ stream.next_in = (void *)buf;
@@ -2008,11 +2060,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr,if (ret !=Z_STREAM_END )die (_ ("unable to deflate new object %s (%d)" ),oid_to_hex (oid), ret); - ret =git_deflate_end_gently (&stream); + ret =end_loose_object_common (&c, &stream, ¶no_oid);if (ret !=Z_OK ) -die (_ ("deflateEnd on object %s failed (%d)" ),oid_to_hex (oid), - ret); - the_hash_algo->final_oid_fn (¶no_oid, &c); +die (_ ("deflateEnd on object %s failed (%d)" ),oid_to_hex (oid), ret);close_loose_object (fd);if (!oideq (oid, ¶no_oid))
Signature · 1 line
2.35.1.1548.g36973b18e52
Re: [PATCH v12 3/8] object-file.c: refactor write_loose_object() to several steps
Quoted 7 lines · Ævar Arnfjörð Bjarmason
+/**
+ * Common steps for loose object writers to end writing loose objects:
+ *
+ * - End the compression of zlib stream.
+ * - Get the calculated oid to "oid".
+ * -fsync()andclose()the "fd"
Since we removed close_loose_object() from end_loose_object_common() , I think this comment should also be removed.
Thanks.
-Han Xin
Quoted 14 lines
+ */
object-file.c +13
-0+static int end_loose_object_common (git_hash_ctx *c,git_zstream *stream, +struct object_id *oid) +{ +int ret; + + ret =git_deflate_end_gently (stream); +if (ret !=Z_OK ) +return ret; + the_hash_algo->final_oid_fn (oid, c); + +return Z_OK ; +} +
Re: [PATCH v12 3/8] object-file.c: refactor write_loose_object() to several steps
Quoted 11 lines · Han Xin
Quoted 7 lines · Ævar Arnfjörð Bjarmason
+/**
+ * Common steps for loose object writers to end writing loose objects:
+ *
+ * - End the compression of zlib stream.
+ * - Get the calculated oid to "oid".
+ * -fsync()andclose()the "fd"Since we removed
close_loose_object()fromend_loose_object_common(), I think this comment should also be removed.
You're right. I adjusted it for the "parano_oid" in this v12, but managed to miss that somehow.
Will submit a re-roll with those changes, but will wait a bit more to see if there's any other comments on this v12 first. Thanks!
PATCH v12 4/8 object-file.c: factor out deflate part of write_loose_object()
In reply to Ævar Arnfjörð BjarmasonSplit out the part of write_loose_object() that deals with calling git_deflate() into a utility function, a subsequent commit will introduce another function that'll make use of it.
--- object-file.c | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-)
object-file.c +25 -6
@@ -2004,6 +2004,28 @@ static int start_loose_object_common(struct strbuf *tmp_file,return fd; } +/** +* Common steps for the inner git_deflate() loop for writing loose +* objects. Returns what git_deflate() returns. +*/ +static int write_loose_object_common (git_hash_ctx *c, +git_zstream *stream,const int flush, +unsigned char *in0,const int fd, +unsigned char *compressed, +const size_t compressed_len) +{ +int ret; + + ret =git_deflate (stream, flush ?Z_FINISH :0 ); + the_hash_algo->update_fn (c, in0, stream->next_in - in0); +if (write_buffer (fd, compressed, stream->next_out - compressed) <0 ) +die (_ ("unable to write loose object file" )); + stream->next_out = compressed; + stream->avail_out = compressed_len; + +return ret; +} +/** * Common steps for loose object writers to end writing loose objects: *
@@ -2049,12 +2071,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr, stream.avail_in = len;do {unsigned char *in0 = stream.next_in; - ret =git_deflate (&stream,Z_FINISH ); - the_hash_algo->update_fn (&c, in0, stream.next_in - in0); -if (write_buffer (fd, compressed, stream.next_out - compressed) <0 ) -die (_ ("unable to write loose object file" )); - stream.next_out = compressed; - stream.avail_out =sizeof (compressed); + + ret =write_loose_object_common (&c, &stream,1 , in0, fd, + compressed,sizeof (compressed)); }while (ret ==Z_OK );if (ret !=Z_STREAM_END )
Signature · 1 line
2.35.1.1548.g36973b18e52
PATCH v12 5/8 object-file.c: add "stream_loose_object()" to handle large object
In reply to Ævar Arnfjörð BjarmasonFrom: Han Xin <[email protected]>
If we want unpack and write a loose object using "write_loose_object", we have to feed it with a buffer with the same size of the object, which will consume lots of memory and may cause OOM. This can be improved by feeding data to "stream_loose_object()" in a stream.
Add a new function "stream_loose_object()", which is a stream version of "write_loose_object()" but with a low memory footprint. We will use this function to unpack large blob object in later commit.
Another difference with "write_loose_object()" is that we have no chance to run "write_object_file_prepare()" to calculate the oid in advance. In "write_loose_object()", we know the oid and we can write the temporary file in the same directory as the final object, but for an object with an undetermined oid, we don't know the exact directory for the object.
Still, we need to save the temporary file we're preparing somewhere. We'll do that in the top-level ".git/objects/" directory (or whatever "GIT_OBJECT_DIRECTORY" is set to). Once we've streamed it we'll know the OID, and will move it to its canonical path.
"freshen_packed_object()" or "freshen_loose_object()" will be called inside "stream_loose_object()" after obtaining the "oid".
--- object-file.c | 100 +++++++++++++++++++++++++++++++++++++++++++++++++ object-store.h | 8 ++++ 2 files changed, 108 insertions(+)
object-file.c +100 -0
@@ -2118,6 +2118,106 @@ static int freshen_packed_object(const struct object_id *oid)return 1 ; } +int stream_loose_object (struct input_stream *in_stream,size_t len, +struct object_id *oid) +{ +int fd, ret, err =0 , flush =0 ; +unsigned char compressed[4096 ]; +git_zstream stream; +git_hash_ctx c; +struct strbuf tmp_file =STRBUF_INIT ; +struct strbuf filename =STRBUF_INIT ; +int dirlen; +char hdr[MAX_HEADER_LEN ]; +int hdrlen; + +/* Since oid is not determined, save tmp file to odb path. */ +strbuf_addf (&filename,"%s/" ,get_object_directory ()); + hdrlen =format_object_header (hdr,sizeof (hdr),OBJ_BLOB , len); + +/* +* Common steps for write_loose_object and stream_loose_object to +* start writing loose objects: +* +* - Create tmpfile for the loose object. +* - Setup zlib stream for compression. +* - Start to feed header to zlib stream. +*/ + fd =start_loose_object_common (&tmp_file, filename.buf,0 , + &stream, compressed,sizeof (compressed), + &c, hdr, hdrlen); +if (fd <0 ) { + err =-1 ; + goto cleanup; + } + +/* Then the data itself.. */ +do { +unsigned char *in0 = stream.next_in; + +if (!stream.avail_in && !in_stream->is_finished) { +const void *in = in_stream->read (in_stream, &stream.avail_in); + stream.next_in = (void *)in; + in0 = (unsigned char *)in; +/* All data has been read. */ +if (in_stream->is_finished) + flush =1 ; + } + ret =write_loose_object_common (&c, &stream, flush, in0, fd, + compressed,sizeof (compressed)); +/* +* Unlike write_loose_object(), we do not have the entire +* buffer. If we get Z_BUF_ERROR due to too few input bytes, +* then we'll replenish them in the next input_stream->read() +* call when we loop. +*/ + }while (ret ==Z_OK || ret ==Z_BUF_ERROR ); + +if (stream.total_in != len + hdrlen) +die (_ ("write stream object %ld != %" PRIuMAX), stream.total_in, + (uintmax_t )len + hdrlen); + +/* Common steps for write_loose_object and stream_loose_object to +* end writing loose oject: +* +* - End the compression of zlib stream. +* - Get the calculated oid. +*/ +if (ret !=Z_STREAM_END ) +die (_ ("unable to stream deflate new object (%d)" ), ret); + ret =end_loose_object_common (&c, &stream, oid); +if (ret !=Z_OK ) +die (_ ("deflateEnd on stream object failed (%d)" ), ret); +close_loose_object (fd); + +if (freshen_packed_object (oid) ||freshen_loose_object (oid)) { +unlink_or_warn (tmp_file.buf); + goto cleanup; + } + +loose_object_path (the_repository, &filename, oid); + +/* We finally know the object path, and create the missing dir. */ + dirlen =directory_size (filename.buf); +if (dirlen) { +struct strbuf dir =STRBUF_INIT ; +strbuf_add (&dir, filename.buf, dirlen); + +if (mkdir_in_gitdir (dir.buf) && errno !=EEXIST ) { + err =error_errno (_ ("unable to create directory %s" ), dir.buf); +strbuf_release (&dir); + goto cleanup; + } +strbuf_release (&dir); + } + + err =finalize_object_file (tmp_file.buf, filename.buf); +cleanup: +strbuf_release (&tmp_file); +strbuf_release (&filename); +return err; +} +int write_object_file_flags (const void *buf,unsigned long len,enum object_type type,struct object_id *oid,unsigned flags)
object-store.h +8 -0
@@ -46,6 +46,12 @@ struct object_directory {char *path; }; +struct input_stream { +const void *(*read)(struct input_stream *,unsigned long *len); +void *data; +int is_finished; +}; + KHASH_INIT(odb_path_map, const char * /* key: odb_path */, struct object_directory *, 1, fspathhash, fspatheq)
@@ -261,6 +267,8 @@ static inline int write_object_file(const void *buf, unsigned long len,int write_object_file_literally (const void *buf,unsigned long len,const char *type,struct object_id *oid,unsigned flags); +int stream_loose_object (struct input_stream *in_stream,size_t len, +struct object_id *oid);/* * Add an object file to the in-memory object store, without writing it
Signature · 1 line
2.35.1.1548.g36973b18e52
Re: [PATCH v12 5/8] object-file.c: add "stream_loose_object()" to handle large object
Quoted 11 lines · Ævar Arnfjörð Bjarmason
From: Han Xin <[email protected]>
If we want unpack and write a loose object using "
write_loose_object", we have to feed it with a buffer with the same size of the object, which will consume lots of memory and may cause OOM. This can be improved by feeding data to "stream_loose_object()" in a stream.Add a new function "
stream_loose_object()", which is a stream version of "write_loose_object()" but with a low memory footprint. We will use this function to unpack large blob object in later commit.
Just a thought for optimization which you might want to try on top of this series:
try using mmap on both the source and target files of your stream. Use a big 'window' for the mmap (multiple MB) to reduce the TLB flush costs. TLB flush costs should be minimal anyway if Git is single-threaded.
If you can set the source and target buffers of zlib to the source and dest mappings respectively, you'd eliminate two copies of data into Git's stack buffers. You might need to over-allocate the dst file if you don't know the size up front, but doing an over-allocate and truncate should be pretty cheap if you're working with a big file.
Thanks,
Neeraj
PATCH v12 6/8 core doc: modernize core.bigFileThreshold documentation
In reply to Ævar Arnfjörð BjarmasonThe core.bigFileThreshold documentation has been largely unchanged since 5eef828bc03 (fast-import: Stream very large blobs directly to pack, 2010-02-01).
But since then this setting has been expanded to affect a lot more than that description indicated. Most notably in how "git diff" treats them, see 6bf3b813486 (diff --stat: mark any file larger than core.bigfilethreshold binary, 2014-08-16).
In addition to that, numerous commands and APIs make use of a streaming mode for files above this threshold.
So let's attempt to summarize 12 years of changes in behavior, which can be seen with:
git log --oneline -Gbig_file_thre 5eef828bc03.. -- '*.c'
To do that turn this into a bullet-point list. The summary Han Xin produced in [1] helped a lot, but is a bit too detailed for documentation aimed at users. Let's instead summarize how user-observable behavior differs, and generally describe how we tend to stream these files in various commands.
1. https://lore.kernel.org/git/[email protected]/
--- Documentation/config/core.txt | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-)
Documentation/config/core.txt +24 -9
@@ -412,17 +412,32 @@ You probably do not need to adjust this value. Common unit suffixes of 'k', 'm', or 'g' are supported. core.bigFileThreshold:: - Files larger than this size are stored deflated, without - attempting delta compression. Storing large files without - delta compression avoids excessive memory usage, at the - slight expense of increased disk usage. Additionally files - larger than this size are always treated as binary. + The size of files considered "big", which as discussed below + changes the behavior of numerous git commands, as well as how + such files are stored within the repository. The default is + 512 MiB. Common unit suffixes of 'k', 'm', or 'g' are + supported. + -Default is 512 MiB on all platforms. This should be reasonable -for most projects as source code and other text files can still -be delta compressed, but larger binary media files won't be. +Files above the configured limit will be: + -Common unit suffixes of 'k', 'm', or 'g' are supported. +* Stored deflated, without attempting delta compression. ++ +The default limit is primarily set with this use-case in mind. With it +most projects will have their source code and other text files delta +compressed, but not larger binary media files. ++ +Storing large files without delta compression avoids excessive memory +usage, at the slight expense of increased disk usage. ++ +* Will be treated as if though they were labeled "binary" (see + linkgit:gitattributes[5]). This means that e.g. linkgit:git-log[1] + and linkgit:git-diff[1] will not diffs for files above this limit. ++ +* Will be generally be streamed when written, which avoids excessive +memory usage, at the cost of some fixed overhead. Commands that make +use of this include linkgit:git-archive[1], +linkgit:git-fast-import[1], linkgit:git-index-pack[1] and +linkgit:git-fsck[1]. core.excludesFile:: Specifies the pathname to the file that contains patterns to
Signature · 1 line
2.35.1.1548.g36973b18e52
PATCH v12 7/8 unpack-objects: refactor away unpack_non_delta_entry()
In reply to Ævar Arnfjörð BjarmasonThe unpack_one() function will call either a non-trivial unpack_delta_entry() or a trivial unpack_non_delta_entry(). Let's inline the latter in the only caller.
Since 21666f1aae4 (convert object type handling from a string to a number, 2007-02-26) the unpack_non_delta_entry() function has been rather trivial, and in a preceding commit the "dry_run" condition it was handling went away.
This is not done as an optimization, as the compiler will easily discover that it can do the same, rather this makes a subsequent commit easier to reason about. As it'll be handling "OBJ_BLOB" in a special manner let's re-arrange that "case" in preparation for that change.
--- builtin/unpack-objects.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-)
builtin/unpack-objects.c +7 -11
@@ -338,15 +338,6 @@ static void added_object(unsigned nr, enum object_type type, } } -static void unpack_non_delta_entry (enum object_type type,unsigned long size, -unsigned nr) -{ -void *buf =get_data (size); - -if (buf) -write_object (nr, type, buf, size); -} -static int resolve_against_held (unsigned nr,const struct object_id *base,void *delta_data,unsigned long delta_size) {
@@ -479,12 +470,17 @@ static void unpack_one(unsigned nr) }switch (type) { +case OBJ_BLOB :case OBJ_COMMIT :case OBJ_TREE : -case OBJ_BLOB :case OBJ_TAG : -unpack_non_delta_entry (type, size, nr); + { +void *buf =get_data (size); + +if (buf) +write_object (nr, type, buf, size);return ; + }case OBJ_REF_DELTA :case OBJ_OFS_DELTA :unpack_delta_entry (type, size, nr);
Signature · 1 line
2.35.1.1548.g36973b18e52
Re: [PATCH v12 7/8] unpack-objects: refactor away unpack_non_delta_entry()
Am 29.03.22 um 15:56 schrieb Ævar Arnfjörð Bjarmason:
Quoted 4 lines
The
unpack_one()function will call either a non-trivialunpack_delta_entry()or a trivialunpack_non_delta_entry(). Let's inline the latter in the only caller.
Quoted 8 lines
Since
21666f1aae4(convert object type handling from a string to a number, 2007-02-26) theunpack_non_delta_entry()function has been rather trivial, and in a preceding commit the "dry_run" condition it was handling went away.This is not done as an optimization, as the compiler will easily discover that it can do the same, rather this makes a subsequent commit easier to reason about.
How exactly does inlining the function make the next patch easier to understand or discuss? Plugging in the stream_blob() call to handle the big blobs looks the same with or without the unpack_non_delta_entry() call to me.
Quoted 8 lines
As it'll be handling "
OBJ_BLOB" in a
special manner let's re-arrange that "case" in preparation for that change.Signed-off-by Ævar Arnfjörð Bjarmason <[email protected]>--- builtin/unpack-objects.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-)
Reducing the number of lines can be an advantage. *shrug*
Quoted 41 lines
builtin/unpack-objects.c +7
-11@@ -338,15 +338,6 @@ static void added_object(unsigned nr, enum object_type type, } } -static void unpack_non_delta_entry (enum object_type type,unsigned long size, -unsigned nr) -{ -void *buf =get_data (size); - -if (buf) -write_object (nr, type, buf, size); -} -static int resolve_against_held (unsigned nr,const struct object_id *base,void *delta_data,unsigned long delta_size) {@@ -479,12 +470,17 @@ static void unpack_one(unsigned nr) }switch (type) { +case OBJ_BLOB :case OBJ_COMMIT :case OBJ_TREE : -case OBJ_BLOB :case OBJ_TAG : -unpack_non_delta_entry (type, size, nr); + { +void *buf =get_data (size); + +if (buf) +write_object (nr, type, buf, size);return ; + }case OBJ_REF_DELTA :case OBJ_OFS_DELTA :unpack_delta_entry (type, size, nr);
Re: [PATCH v12 7/8] unpack-objects: refactor away unpack_non_delta_entry()
Quoted 10 lines · René Scharfe
Am 29.03.22 um 15:56 schrieb Ævar Arnfjörð Bjarmason:
Quoted 9 lines
The
unpack_one()function will call either a non-trivialunpack_delta_entry()or a trivialunpack_non_delta_entry(). Let's inline the latter in the only caller.Since
21666f1aae4(convert object type handling from a string to a number, 2007-02-26) theunpack_non_delta_entry()function has been rather trivial, and in a preceding commit the "dry_run" condition it was handling went away.
Quoted 8 lines
Quoted 3 lines
This is not done as an optimization, as the compiler will easily discover that it can do the same, rather this makes a subsequent commit easier to reason about.
How exactly does inlining the function make the next patch easier to understand or discuss? Plugging in the
stream_blob()call to handle the big blobs looks the same with or without theunpack_non_delta_entry()call to me.
The earlier version of it without this prep cleanup can be seen at
https://lore.kernel.org/git/[email protected]/
So yes, this could be skipped, but I tought with this step it was easier to understand.
We previously had to change "void *buf = get_data(size);" in the function to just "void *buf", and do the assignment after the condition that's being checked here.
I think it's also more obvious in terms of control flow if we're checking OBJ_BLOB here to not call a function which has a special-case just for OBJ_BLOB, we can just do that here instead.
Quoted 10 lines
Quoted 8 lines
As it'll be handling "
OBJ_BLOB" in a
special manner let's re-arrange that "case" in preparation for that change.Signed-off-by Ævar Arnfjörð Bjarmason <[email protected]>--- builtin/unpack-objects.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-)Reducing the number of lines can be an advantage. *shrug*
There was also the (admittedly rather small) knock-on-effect on 8/8. Before this it was 8 lines added / 1 removed when it came to the code impacted by this change, now it's a 5 added/0 removed in the below "switch".
So I think it's worth keeping.
Re: [PATCH v12 7/8] unpack-objects: refactor away unpack_non_delta_entry()
Am 31.03.22 um 14:42 schrieb Ævar Arnfjörð Bjarmason:
Quoted 17 lines · René Scharfe
Quoted 15 lines
Am 29.03.22 um 15:56 schrieb Ævar Arnfjörð Bjarmason:
Quoted 12 lines
The
unpack_one()function will call either a non-trivialunpack_delta_entry()or a trivialunpack_non_delta_entry(). Let's inline the latter in the only caller.Since
21666f1aae4(convert object type handling from a string to a number, 2007-02-26) theunpack_non_delta_entry()function has been rather trivial, and in a preceding commit the "dry_run" condition it was handling went away.This is not done as an optimization, as the compiler will easily discover that it can do the same, rather this makes a subsequent commit easier to reason about.
Quoted 7 lines
Quoted 4 lines
How exactly does inlining the function make the next patch easier to understand or discuss? Plugging in the
stream_blob()call to handle the big blobs looks the same with or without theunpack_non_delta_entry()call to me.The earlier version of it without this prep cleanup can be seen at
https://lore.kernel.org/git/[email protected]/
This plugged the special case into unpack_non_delta_entry(). The alternative I had in mind was to plug it into the switch statement as the current patch does, just without inlining unpack_non_delta_entry().
Quoted 28 lines
So yes, this could be skipped, but I tought with this step it was easier to understand.
We previously had to change "void *buf =
get_data(size);" in the function to just "void *buf", and do the assignment after the condition that's being checked here.I think it's also more obvious in terms of control flow if we're checking
OBJ_BLOBhere to not call a function which has a special-case just forOBJ_BLOB, we can just do that here instead.Quoted 10 lines
Quoted 8 lines
As it'll be handling "
OBJ_BLOB" in a
special manner let's re-arrange that "case" in preparation for that change.Signed-off-by Ævar Arnfjörð Bjarmason <[email protected]>--- builtin/unpack-objects.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-)Reducing the number of lines can be an advantage. *shrug*
There was also the (admittedly rather small) knock-on-effect on 8/8. Before this it was 8 lines added / 1 removed when it came to the code impacted by this change, now it's a 5 added/0 removed in the below "switch".
So I think it's worth keeping.
PATCH v12 8/8 unpack-objects: use stream_loose_object() to unpack large objects
In reply to Ævar Arnfjörð BjarmasonFrom: Han Xin <[email protected]>
Make use of the stream_loose_object() function introduced in the preceding commit to unpack large objects. Before this we'd need to malloc() the size of the blob before unpacking it, which could cause OOM with very large blobs.
We could use the new streaming interface to unpack all blobs, but doing so would be much slower, as demonstrated e.g. with this benchmark using git-hyperfine[0]:
rm -rf /tmp/scalar.git && git clone --bare https://github.com/Microsoft/scalar.git /tmp/scalar.git && mv /tmp/scalar.git/objects/pack/*.pack /tmp/scalar.git/my.pack && git hyperfine \ -r 2 --warmup 1 \ -L rev origin/master,HEAD -L v "10,512,1k,1m" \ -s 'make' \ -p 'git init --bare dest.git' \ -c 'rm -rf dest.git' \ './git -C dest.git -c core.bigFileThreshold={v} unpack-objects </tmp/scalar.git/my.pack'
Here we'll perform worse with lower core.bigFileThreshold settings with this change in terms of speed, but we're getting lower memory use in return:
Summary
'./git -C dest.git -c core.bigFileThreshold=10 unpack-objects </tmp/scalar.git/my.pack' in 'origin/master' ran
1.01 ± 0.01 times faster than './git -C dest.git -c core.bigFileThreshold=1k unpack-objects </tmp/scalar.git/my.pack' in 'origin/master' 1.01 ± 0.01 times faster than './git -C dest.git -c core.bigFileThreshold=1m unpack-objects </tmp/scalar.git/my.pack' in 'origin/master' 1.01 ± 0.02 times faster than './git -C dest.git -c core.bigFileThreshold=1m unpack-objects </tmp/scalar.git/my.pack' in 'HEAD' 1.02 ± 0.00 times faster than './git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/scalar.git/my.pack' in 'origin/master' 1.09 ± 0.01 times faster than './git -C dest.git -c core.bigFileThreshold=1k unpack-objects </tmp/scalar.git/my.pack' in 'HEAD' 1.10 ± 0.00 times faster than './git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/scalar.git/my.pack' in 'HEAD' 1.11 ± 0.00 times faster than './git -C dest.git -c core.bigFileThreshold=10 unpack-objects </tmp/scalar.git/my.pack' in 'HEAD'
A better benchmark to demonstrate the benefits of that this one, which creates an artificial repo with a 1, 25, 50, 75 and 100MB blob:
rm -rf /tmp/repo &&
git init /tmp/repo &&
(
cd /tmp/repo &&
for i in 1 25 50 75 100
do
dd if=/dev/urandom of=blob.$i count=$(($i*1024)) bs=1024
done &&
git add blob.* &&
git commit -mblobs &&
git gc &&
PACK=$(echo .git/objects/pack/pack-*.pack) &&
cp "$PACK" my.pack
) &&
git hyperfine \
--show-output \
-L rev origin/master,HEAD -L v "512,50m,100m" \
-s 'make' \
-p 'git init --bare dest.git' \
-c 'rm -rf dest.git' \
'/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold={v} unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum'Using this test we'll always use >100MB of memory on origin/master (around ~105MB), but max out at e.g. ~55MB if we set core.bigFileThreshold=50m.
The relevant "Maximum resident set size" lines were manually added below the relevant benchmark:
'/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=50m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'origin/master' ran
Maximum resident set size (kbytes): 107080
1.02 ± 0.78 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'origin/master'
Maximum resident set size (kbytes): 106968
1.09 ± 0.79 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=100m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'origin/master'
Maximum resident set size (kbytes): 107032
1.42 ± 1.07 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=100m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'HEAD'
Maximum resident set size (kbytes): 107072
1.83 ± 1.02 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=50m unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'HEAD'
Maximum resident set size (kbytes): 55704
2.16 ± 1.19 times faster than '/usr/bin/time -v ./git -C dest.git -c core.bigFileThreshold=512 unpack-objects </tmp/repo/my.pack 2>&1 | grep Maximum' in 'HEAD'
Maximum resident set size (kbytes): 4564
This shows that if you have enough memory this new streaming method is slower the lower you set the streaming threshold, but the benefit is more bounded memory use.
An earlier version of this patch introduced a new "core.bigFileStreamingThreshold" instead of re-using the existing "core.bigFileThreshold" variable[1]. As noted in a detailed overview of its users in [2] using it has several different meanings.
Still, we consider it good enough to simply re-use it. While it's possible that someone might want to e.g. consider objects "small" for the purposes of diffing but "big" for the purposes of writing them such use-cases are probably too obscure to worry about. We can always split up "core.bigFileThreshold" in the future if there's a need for that.
0. https://github.com/avar/git-hyperfine/
1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/
--- Documentation/config/core.txt | 4 +- builtin/unpack-objects.c | 67 +++++++++++++++++++++++++++++++++ t/t5351-unpack-large-objects.sh | 26 +++++++++++-- 3 files changed, 92 insertions(+), 5 deletions(-)
Documentation/config/core.txt +2 -2
@@ -436,8 +436,8 @@ usage, at the slight expense of increased disk usage. * Will be generally be streamed when written, which avoids excessive memory usage, at the cost of some fixed overhead. Commands that make use of this include linkgit:git-archive[1], -linkgit:git-fast-import[1], linkgit:git-index-pack[1] and -linkgit:git-fsck[1]. +linkgit:git-fast-import[1], linkgit:git-index-pack[1], +linkgit:git-unpack-objects[1] and linkgit:git-fsck[1]. core.excludesFile:: Specifies the pathname to the file that contains patterns to
builtin/unpack-objects.c +67 -0
79 lines not on this page · raw message
t/t5351-unpack-large-objects.sh +23 -3
@@ -9,7 +9,8 @@ test_description='git unpack-objects with large objects' prepare_dest () { test_when_finished "rm -rf dest.git" && - git init --bare dest.git + git init --bare dest.git && + git -C dest.git config core.bigFileThreshold "$1" } test_expect_success "create large objects (1.5 MB) and PACK" '
@@ -26,16 +27,35 @@ test_expect_success 'set memory limitation to 1MB' ' ' test_expect_success 'unpack-objects failed under memory limitation' ' - prepare_dest && + prepare_dest 2m && test_must_fail git -C dest.git unpack-objects <pack-$PACK.pack 2>err && grep "fatal: attempting to allocate" err ' test_expect_success 'unpack-objects works with memory limitation in dry-run mode' ' - prepare_dest && + prepare_dest 2m && git -C dest.git unpack-objects -n <pack-$PACK.pack && test_stdout_line_count = 0 find dest.git/objects -type f && test_dir_is_empty dest.git/objects/pack ' +test_expect_success 'unpack big object in stream' ' + prepare_dest 1m && + git -C dest.git unpack-objects <pack-$PACK.pack && + test_dir_is_empty dest.git/objects/pack +' + +test_expect_success 'do not unpack existing large objects' ' + prepare_dest 1m && + git -C dest.git index-pack --stdin <pack-$PACK.pack && + git -C dest.git unpack-objects <pack-$PACK.pack && + + # The destination came up with the exact same pack... + DEST_PACK=$(echo dest.git/objects/pack/pack-*.pack) && + test_cmp pack-$PACK.pack $DEST_PACK && + + # ...and wrote no loose objects + test_stdout_line_count = 0 find dest.git/objects -type f ! -name "pack-*" +' + test_done
Signature · 1 line
2.35.1.1548.g36973b18e52
v13 cover unpack-objects: support streaming blobs to disk
In reply to Ævar Arnfjörð BjarmasonThis series makes "unpack-objects" capable of streaming large objects to disk.
As 7/7 shows streaming e.g. a 100MB blob now uses ~5MB of memory instead of ~105MB. This streaming method is slower if you've got memory to handle the blobs in-core, but if you don't it allows you to unpack objects at all, as you might otherwise OOM.
This series by Han Xin was originally waiting on some in-flight patches that landed in 430883a70c7 (Merge branch 'ab/object-file-api-updates', 2022-03-16), and until yesterday with 83937e95928 (Merge branch 'ns/batch-fsync', 2022-06-03) had a textual and semantic conflict with "master".
Changes since v12:
* Since v12 Han Xin submitted 1/1 here as
https://lore.kernel.org/git/[email protected]/;
I think this is better off reviewed as a whole, and hopefully will be picked up as such.
* Dropped the previous 7/8, which was a refactoring to make 8/8 slightly smaller. Per dicsussion with René it's better to leave it out.
* The rest (especially 2/8) is due to rebasing on ns/batch-fsync.
Han Xin (4):
unpack-objects: low memory footprint for get_data() in dry_run mode object-file.c: refactor write_loose_object() to several steps object-file.c: add "stream_loose_object()" to handle large object
unpack-objects: use stream_loose_object() to unpack large objects
Ævar Arnfjörð Bjarmason (3):
object-file.c: do fsync() and close() before post-write die() object-file.c: factor out deflate part of write_loose_object() core doc: modernize core.bigFileThreshold documentation
Documentation/config/core.txt | 33 +++-- builtin/unpack-objects.c | 103 ++++++++++++-- object-file.c | 237 +++++++++++++++++++++++++++----- object-store.h | 8 ++ t/t5351-unpack-large-objects.sh | 61 ++++++++ 5 files changed, 387 insertions(+), 55 deletions(-) create mode 100755 t/t5351-unpack-large-objects.sh
Range-diff against v12:
1: e95f6a1cfb6 = 1: 12873fc9915 unpack-objects: low memory footprint for get_data() in dry_run mode
2: 54060eb8c6b ! 2: b3568f0c5c0 object-file.c: do fsync() and close() before post-write die()
@@ Commit message
Signed-off-by: Ævar Arnfjörð Bjarmason <[email protected]>
## object-file.c ##
-@@ object-file.c: void hash_object_file(const struct git_hash_algo *algo, const void *buf,
- hash_object_file_literally(algo, buf, len, type_name(type), oid);
- }
-
--/* Finalize a file on disk, and close it. */
-+/*
-+ * We already did a write_buffer() to the "fd", let's fsync()
-+ * and close().
-+ *
-+ * Finalize a file on disk, and close it. We might still die() on a
-+ * subsequent sanity check, but let's not add to that confusion by not
-+ * flushing any outstanding writes to disk first.
-+ */
- static void close_loose_object(int fd)
- {
- if (the_repository->objects->odb->will_destroy)
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *hdr,
die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
ret);
the_hash_algo->final_oid_fn(¶no_oid, &c);
-+ close_loose_object(fd);
++ close_loose_object(fd, tmp_file.buf);
+
if (!oideq(oid, ¶no_oid))
die(_("confused by unstable object source data for %s"),
oid_to_hex(oid));
-- close_loose_object(fd);
+- close_loose_object(fd, tmp_file.buf);
-
if (mtime) {
struct utimbuf utb;
3: 3dcaa5d6589 ! 3: 9dc0f56878a object-file.c: refactor write_loose_object() to several steps
@@ object-file.c: static int write_loose_object(const struct object_id *oid, char *
- ret);
- the_hash_algo->final_oid_fn(¶no_oid, &c);
+ die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid), ret);
- close_loose_object(fd);
+ close_loose_object(fd, tmp_file.buf);
if (!oideq(oid, ¶no_oid))
4: 03f4e91ac89 = 4: a0434835fe7 object-file.c: factor out deflate part of write_loose_object()
5: 3d64cf1cf33 ! 5: 0b07b29836b object-file.c: add "stream_loose_object()" to handle large object
@@ object-file.c: static int freshen_packed_object(const struct object_id *oid)
+ ret = end_loose_object_common(&c, &stream, oid);
+ if (ret != Z_OK)
+ die(_("deflateEnd on stream object failed (%d)"), ret);
-+ close_loose_object(fd);
++ close_loose_object(fd, tmp_file.buf);
+
+ if (freshen_packed_object(oid) || freshen_loose_object(oid)) {
+ unlink_or_warn(tmp_file.buf);
6: 33ffcbbc1f0 = 6: 5ed79c58b18 core doc: modernize core.bigFileThreshold documentation
7: 11f7aa026b4 < -: ----------- unpack-objects: refactor away unpack_non_delta_entry()
8: 34ee6a28a54 ! 7: 5bc8fa9bc8d unpack-objects: use stream_loose_object() to unpack large objects @@ Documentation/config/core.txt: usage, at the slight expense of increased disk us
Specifies the pathname to the file that contains patterns to
## builtin/unpack-objects.c ##
-@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type type,
- }
+@@ builtin/unpack-objects.c: static void unpack_non_delta_entry(enum object_type type, unsigned long size,
+ write_object(nr, type, buf, size);
}
+struct input_zstream_data {
@@ builtin/unpack-objects.c: static void added_object(unsigned nr, enum object_type
void *delta_data, unsigned long delta_size)
{
@@ builtin/unpack-objects.c: static void unpack_one(unsigned nr)
+ }
switch (type) {
- case OBJ_BLOB:
++ case OBJ_BLOB:
+ if (!dry_run && size > big_file_threshold) {
+ stream_blob(size, nr);
+ return;
@@ builtin/unpack-objects.c: static void unpack_one(unsigned nr)
+ /* fallthrough */
case OBJ_COMMIT:
case OBJ_TREE:
+- case OBJ_BLOB:
case OBJ_TAG:
+ unpack_non_delta_entry(type, size, nr);
+ return;
## t/t5351-unpack-large-objects.sh ##
@@ t/t5351-unpack-large-objects.sh: test_description='git unpack-objects with large objects'
Signature · 1 line
2.36.1.1124.g52838f02905
PATCH v13 1/7 unpack-objects: low memory footprint for get_data() in dry_run mode
From: Han Xin <[email protected]>
As the name implies, "get_data(size)" will allocate and return a given amount of memory. Allocating memory for a large blob object may cause the system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint for dry_run mode.
Because in dry_run mode, "get_data()" is only used to check the integrity of data, and the returned buffer is not used at all, we can allocate a smaller buffer and reuse it as zstream output. Therefore, in dry_run mode, "get_data()" will release the allocated buffer and return NULL instead of returning garbage data.
The "find [...]objects/?? -type f | wc -l" test idiom being used here is adapted from the same "find" use added to another test in d9545c7f465 (fast-import: implement unpack limit, 2016-04-25).
--- builtin/unpack-objects.c | 34 ++++++++++++++++++--------- t/t5351-unpack-large-objects.sh | 41 +++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 11 deletions(-) create mode 100755 t/t5351-unpack-large-objects.sh
builtin/unpack-objects.c +23 -11
@@ -97,15 +97,26 @@ static void use(int bytes)display_throughput (progress, consumed_bytes); } +/* +* Decompress zstream from stdin and return specific size of data. +* The caller is responsible to free the returned buffer. +* +* But for dry_run mode, "get_data()" is only used to check the +* integrity of data, and the returned buffer is not used at all. +* Therefore, in dry_run mode, "get_data()" will release the small +* allocated buffer which is reused to hold temporary zstream output +* and return NULL instead of returning garbage data. +*/ static void *get_data (unsigned long size) {git_zstream stream; -void *buf =xmallocz (size); +unsigned long bufsize = dry_run && size >8192 ?8192 : size; +void *buf =xmallocz (bufsize);memset (&stream,0 ,sizeof (stream)); stream.next_out = buf; - stream.avail_out = size; + stream.avail_out = bufsize; stream.next_in =fill (1 ); stream.avail_in = len;git_inflate_init (&stream);
@@ -125,8 +136,15 @@ static void *get_data(unsigned long size) } stream.next_in =fill (1 ); stream.avail_in = len; +if (dry_run) { +/* reuse the buffer in dry_run mode */ + stream.next_out = buf; + stream.avail_out = bufsize; + } }git_inflate_end (&stream); +if (dry_run) +FREE_AND_NULL (buf);return buf; }
@@ -326,10 +344,8 @@ static void unpack_non_delta_entry(enum object_type type, unsigned long size, {void *buf =get_data (size); -if (!dry_run && buf) +if (buf)write_object (nr, type, buf, size); -else -free (buf); } static int resolve_against_held(unsigned nr, const struct object_id *base,
@@ -359,10 +375,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,oidread (&base_oid,fill (the_hash_algo->rawsz));use (the_hash_algo->rawsz); delta_data =get_data (delta_size); -if (dry_run || !delta_data) { -free (delta_data); +if (!delta_data)return ; - }if (has_object_file (&base_oid)) ;/* Ok we have this one */ else if (resolve_against_held(nr, &base_oid,
@@ -398,10 +412,8 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,die ("offset value out of bound for delta base object" ); delta_data =get_data (delta_size); -if (dry_run || !delta_data) { -free (delta_data); +if (!delta_data)return ; - } lo =0 ; hi = nr;while (lo < hi) {
t/t5351-unpack-large-objects.sh new file +41 -0
41 lines not on this page · raw message
Signature · 1 line
2.36.1.1124.g52838f02905
Re: [PATCH v13 1/7] unpack-objects: low memory footprint for get_data() in dry_run mode
Quoted 11 lines · Ævar Arnfjörð Bjarmason
From: Han Xin <[email protected]>
As the name implies, "
get_data(size)" will allocate and return a given amount of memory. Allocating memory for a large blob object may cause the system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint fordry_runmode.Because in
dry_runmode, "get_data()" is only used to check the integrity of data, and the returned buffer is not used at all, we can allocate a smaller buffer and reuse it as zstream output. Therefore,
"reuse" -> "use"
Quoted 2 lines
in
dry_runmode, "get_data()" will release the allocated buffer and return NULL instead of returning garbage data.
It makes it sound as if we used to return garbage data, but I do not think that is what happened in reality. Perhaps rewrite the last sentence like
Make the function return NULL in the dry-run mode, as no callers use the returned buffer.
or something?
The overall logic sounds quite sensible.
Quoted 7 lines
The "find [...]objects/?? -type f | wc -l" test idiom being used here is adapted from the same "find" use added to another test in
d9545c7f465(fast-import: implement unpack limit, 2016-04-25).+/*
+ * Decompress zstream from stdin and return specific size of data.
"specific size"? The caller specifies the size of data (because it knows a-priori how many bytes the zstream should inflate to), so
Decompress zstream from the standard input into a newly allocated buffer of specified size and return the buffer.
or something, perhaps. In any case, it needs to say that the caller is responsible for giving the "right" size.
Quoted 10 lines
+ * The caller is responsible to free the returned buffer.
+ *
+ * But fordry_runmode, "get_data()" is only used to check the
+ * integrity of data, and the returned buffer is not used at all.
+ * Therefore, indry_runmode, "get_data()" will release the small
+ * allocated buffer which is reused to hold temporary zstream output
+ * and return NULL instead of returning garbage data.
+ */
static void *get_data(unsigned long size)
{
builtin/unpack-objects.c static void use(int bytes) · line 114
Quoted 4 lines
builtin/unpack-objects.c +2
-1git_zstream stream; -void *buf =xmallocz (size); +unsigned long bufsize = dry_run && size >8192 ?8192 : size; +void *buf =xmallocz (bufsize);
OK.
builtin/unpack-objects.c static void *get_data(unsigned long size)
Quoted 9 lines
builtin/unpack-objects.c +1
-1memset (&stream,0 ,sizeof (stream)); stream.next_out = buf; - stream.avail_out = size; + stream.avail_out = bufsize; stream.next_in =fill (1 ); stream.avail_in = len;git_inflate_init (&stream);@@ -125,8 +136,15 @@ static void *get_data(unsigned long size)
What's hidden in the pre-context is this bit:
int ret = git_inflate(&stream, 0);
use(len - stream.avail_in);
if (stream.total_out == size && ret == Z_STREAM_END)
break;
if (ret != Z_OK) {
error("inflate returned %d", ret);
FREE_AND_NULL(buf);
if (!recover)
exit(1);
has_errors = 1;
break;
}and it is correct to use "size", not "bufsize", for this check. Unless we receive exactly the caller-specified "size" bytes from the inflated zstream with Z_STREAM_END, we want to detect an error and bail out.
I am not sure if this is not loosening the error checking in the dry-run case, though. In the original code, we set the avail_out to the total expected size so
(1) if the caller gives too small a size, git_inflate() would stop at stream.total_out with ret that is not STREAM_END nor OK, bypassing the "break", and we catch the error.
(2) if the caller gives too large a size, git_inflate() would stop at the true size of inflated zstream, with STREAM_END and would not hit this "break", and we catch the error.
With the new code, since we keep refreshing avail_out (see below), git_inflate() does not even learn how many bytes we are _expecting_ to see. Is the error checking in the loop, with the updated code, catch the mismatch between expected and actual size (plausibly caused by a corrupted zstream) the same way as we do in the non dry-run code path?
Quoted 14 lines
builtin/unpack-objects.c +7
-0} stream.next_in =fill (1 ); stream.avail_in = len; +if (dry_run) { +/* reuse the buffer in dry_run mode */ + stream.next_out = buf; + stream.avail_out = bufsize; + } }git_inflate_end (&stream); +if (dry_run) +FREE_AND_NULL (buf);return buf; }
Re: [PATCH v13 1/7] unpack-objects: low memory footprint for get_data() in dry_run mode
Quoted 107 lines · Junio C Hamano
Quoted 11 lines · Ævar Arnfjörð Bjarmason
From: Han Xin <[email protected]>
As the name implies, "
get_data(size)" will allocate and return a given amount of memory. Allocating memory for a large blob object may cause the system to run out of memory. Before preparing to replace calling of "get_data()" to unpack large blob objects in latter commits, refactor "get_data()" to reduce memory footprint fordry_runmode.Because in
dry_runmode, "get_data()" is only used to check the integrity of data, and the returned buffer is not used at all, we can allocate a smaller buffer and reuse it as zstream output. Therefore,"reuse" -> "use"
Quoted 2 lines
in
dry_runmode, "get_data()" will release the allocated buffer and return NULL instead of returning garbage data.It makes it sound as if we used to return garbage data, but I do not think that is what happened in reality. Perhaps rewrite the last sentence like
Make the function return NULL in the dry-run mode, as no callers use the returned buffer.
or something?
The overall logic sounds quite sensible.
Quoted 7 lines
The "find [...]objects/?? -type f | wc -l" test idiom being used here is adapted from the same "find" use added to another test in
d9545c7f465(fast-import: implement unpack limit, 2016-04-25).+/*
+ * Decompress zstream from stdin and return specific size of data."specific size"? The caller specifies the size of data (because it knows a-priori how many bytes the zstream should inflate to), so
Decompress zstream from the standard input into a newly allocated buffer of specified size and return the buffer.
or something, perhaps. In any case, it needs to say that the caller is responsible for giving the "right" size.
Quoted 11 lines
+ * The caller is responsible to free the returned buffer.
+ *
+ * But fordry_runmode, "get_data()" is only used to check the
+ * integrity of data, and the returned buffer is not used at all.
+ * Therefore, indry_runmode, "get_data()" will release the small
+ * allocated buffer which is reused to hold temporary zstream output
+ * and return NULL instead of returning garbage data.
+ */
static void *get_data(unsigned long size){ git_zstream stream;builtin/unpack-objects.c static void use(int bytes) · line 114
Quoted 3 lines
builtin/unpack-objects.c +2
-1-void *buf =xmallocz (size); +unsigned long bufsize = dry_run && size >8192 ?8192 : size; +void *buf =xmallocz (bufsize);OK.
builtin/unpack-objects.c static void *get_data(unsigned long size)
Quoted 9 lines
memset(&stream, 0, sizeof(stream)); stream.next_out = buf;builtin/unpack-objects.c +1
-1- stream.avail_out = size; + stream.avail_out = bufsize; stream.next_in =fill (1 ); stream.avail_in = len;git_inflate_init (&stream);@@ -125,8 +136,15 @@ static void *get_data(unsigned long size)What's hidden in the pre-context is this bit:
int ret = git_inflate(&stream, 0); use(len - stream.avail_in); if (stream.total_out == size && ret == Z_STREAM_END) break; if (ret != Z_OK) { error("inflate returned %d", ret); FREE_AND_NULL(buf); if (!recover) exit(1); has_errors = 1; break; }and it is correct to use "size", not "bufsize", for this check. Unless we receive exactly the caller-specified "size" bytes from the inflated zstream with
Z_STREAM_END, we want to detect an error and bail out.I am not sure if this is not loosening the error checking in the dry-run case, though. In the original code, we set the
avail_outto the total expected size so(1) if the caller gives too small a size,
git_inflate()would stop at stream.total_out with ret that is notSTREAM_ENDnor OK, bypassing the "break", and we catch the error.(2) if the caller gives too large a size,
git_inflate()would stop at the true size of inflated zstream, withSTREAM_ENDand would not hit this "break", and we catch the error.
Quoted 7 lines
With the new code, since we keep refreshing
avail_out(see below),git_inflate()does not even learn how many bytes we are _expecting_ to see. Is the error checking in the loop, with the updated code, catch the mismatch between expected and actual size (plausibly caused by a corrupted zstream) the same way as we do in the non dry-run code path?
Unlike the original implementation, if we get a corrupted zstream, we won't break at Z_BUFFER_ERROR, maybe until we've read all the input. I think it can still catch the mismatch between expected and actual size when "fill(1)" gets an EOF, if it's not too late.
Thanks.
-Han Xin
Quoted 14 lines
Quoted 14 lines
builtin/unpack-objects.c +7
-0} stream.next_in =fill (1 ); stream.avail_in = len; +if (dry_run) { +/* reuse the buffer in dry_run mode */ + stream.next_out = buf; + stream.avail_out = bufsize; + } }git_inflate_end (&stream); +if (dry_run) +FREE_AND_NULL (buf);return buf; }
Re: [PATCH v13 1/7] unpack-objects: low memory footprint for get_data() in dry_run mode
Quoted 20 lines · Han Xin
Quoted 12 lines
I am not sure if this is not loosening the error checking in the dry-run case, though. In the original code, we set the
avail_outto the total expected size so(1) if the caller gives too small a size,
git_inflate()would stop at stream.total_out with ret that is notSTREAM_ENDnor OK, bypassing the "break", and we catch the error.(2) if the caller gives too large a size,
git_inflate()would stop at the true size of inflated zstream, withSTREAM_ENDand would not hit this "break", and we catch the error.Quoted 7 lines
With the new code, since we keep refreshing
avail_out(see below),git_inflate()does not even learn how many bytes we are _expecting_ to see. Is the error checking in the loop, with the updated code, catch the mismatch between expected and actual size (plausibly caused by a corrupted zstream) the same way as we do in the non dry-run code path?
Quoted 4 lines
Unlike the original implementation, if we get a corrupted zstream, we won't break at
Z_BUFFER_ERROR, maybe until we've read all the input. I think it can still catch the mismatch between expected and actual size when "fill(1)" gets an EOF, if it's not too late.
That is only one half of the two possible failure cases, i.e. input is shorter than the expected size. If the caller specified size is smaller than what the stream inflates to, I do not see the new code to be limiting the .avail_out near the end of the iteration, which would be necessary to catch such an error, even if we are not interested in using the inflated contents, no?
Re: [PATCH v13 1/7] unpack-objects: low memory footprint for get_data() in dry_run mode
Quoted 28 lines · Junio C Hamano
Quoted 20 lines · Han Xin
Quoted 12 lines
I am not sure if this is not loosening the error checking in the dry-run case, though. In the original code, we set the
avail_outto the total expected size so(1) if the caller gives too small a size,
git_inflate()would stop at stream.total_out with ret that is notSTREAM_ENDnor OK, bypassing the "break", and we catch the error.(2) if the caller gives too large a size,
git_inflate()would stop at the true size of inflated zstream, withSTREAM_ENDand would not hit this "break", and we catch the error.Quoted 7 lines
With the new code, since we keep refreshing
avail_out(see below),git_inflate()does not even learn how many bytes we are _expecting_ to see. Is the error checking in the loop, with the updated code, catch the mismatch between expected and actual size (plausibly caused by a corrupted zstream) the same way as we do in the non dry-run code path?Quoted 4 lines
Unlike the original implementation, if we get a corrupted zstream, we won't break at
Z_BUFFER_ERROR, maybe until we've read all the input. I think it can still catch the mismatch between expected and actual size when "fill(1)" gets an EOF, if it's not too late.
Quoted 7 lines
That is only one half of the two possible failure cases, i.e. input is shorter than the expected size. If the caller specified size is smaller than what the stream inflates to, I do not see the new code to be limiting the .avail_out near the end of the iteration, which would be necessary to catch such an error, even if we are not interested in using the inflated contents, no?
Yes, you are right.
Instead of always using a fixed "bufsize" even if there is not enough expected output remaining, we can get a more accurate one by comparing "total_out" to "size", so we can catch problems early by getting Z_BUFFER_ERROR.
builtin/unpack-objects.c +2 -1
@@ -139,7 +139,8 @@ static void *get_data(unsigned long size)if (dry_run) {/* reuse the buffer in dry_run mode */ stream.next_out = buf; - stream.avail_out = bufsize; + stream.avail_out = bufsize > size - stream.total_out ? + size - stream.total_out : bufsize; } }git_inflate_end (&stream);
Thanks
-Han Xin
Re: [PATCH v13 1/7] unpack-objects: low memory footprint for get_data() in dry_run mode
Quoted 59 lines · Han Xin
Quoted 28 lines · Junio C Hamano
Quoted 20 lines · Han Xin
Quoted 12 lines
I am not sure if this is not loosening the error checking in the dry-run case, though. In the original code, we set the
avail_outto the total expected size so(1) if the caller gives too small a size,
git_inflate()would stop at stream.total_out with ret that is notSTREAM_ENDnor OK, bypassing the "break", and we catch the error.(2) if the caller gives too large a size,
git_inflate()would stop at the true size of inflated zstream, withSTREAM_ENDand would not hit this "break", and we catch the error.Quoted 7 lines
With the new code, since we keep refreshing
avail_out(see below),git_inflate()does not even learn how many bytes we are _expecting_ to see. Is the error checking in the loop, with the updated code, catch the mismatch between expected and actual size (plausibly caused by a corrupted zstream) the same way as we do in the non dry-run code path?Quoted 4 lines
Unlike the original implementation, if we get a corrupted zstream, we won't break at
Z_BUFFER_ERROR, maybe until we've read all the input. I think it can still catch the mismatch between expected and actual size when "fill(1)" gets an EOF, if it's not too late.Quoted 7 lines
That is only one half of the two possible failure cases, i.e. input is shorter than the expected size. If the caller specified size is smaller than what the stream inflates to, I do not see the new code to be limiting the .avail_out near the end of the iteration, which would be necessary to catch such an error, even if we are not interested in using the inflated contents, no?
Yes, you are right.
Instead of always using a fixed "bufsize" even if there is not enough expected output remaining, we can get a more accurate one by comparing "
total_out" to "size", so we can catch problems early by gettingZ_BUFFER_ERROR.builtin/unpack-objects.c +2
-1@@ -139,7 +139,8 @@ static void *get_data(unsigned long size)if (dry_run) {/* reuse the buffer in dry_run mode */ stream.next_out = buf; - stream.avail_out = bufsize; + stream.avail_out = bufsize > size - stream.total_out ? + size - stream.total_out : bufsize; } }git_inflate_end (&stream);
Quoted 2 lines
Thanks
-Han Xin
Han, do you want to pick this up again for a v14? It looks like you're very on top of it already, and I re-sent your patches because I saw that your
https://lore.kernel.org/git/[email protected]/
wasn't picked up in the interim & you hadn't been active on-list otherwise.
But it looks like there's some interest now, and that you have more time to test & follow-up on this topic than I do at the moment, so if you wanted to do the work of properly rebasing ot in tho recent fsync changes that would be great. Thanks.
Re: [PATCH v13 1/7] unpack-objects: low memory footprint for get_data() in dry_run mode
Quoted 73 lines · Ævar Arnfjörð Bjarmason
Quoted 59 lines · Han Xin
Quoted 28 lines · Junio C Hamano
Quoted 20 lines · Han Xin
Quoted 12 lines
I am not sure if this is not loosening the error checking in the dry-run case, though. In the original code, we set the
avail_outto the total expected size so(1) if the caller gives too small a size,
git_inflate()would stop at stream.total_out with ret that is notSTREAM_ENDnor OK, bypassing the "break", and we catch the error.(2) if the caller gives too large a size,
git_inflate()would stop at the true size of inflated zstream, withSTREAM_ENDand would not hit this "break", and we catch the error.Quoted 7 lines
With the new code, since we keep refreshing
avail_out(see below),git_inflate()does not even learn how many bytes we are _expecting_ to see. Is the error checking in the loop, with the updated code, catch the mismatch between expected and actual size (plausibly caused by a corrupted zstream) the same way as we do in the non dry-run code path?Quoted 4 lines
Unlike the original implementation, if we get a corrupted zstream, we won't break at
Z_BUFFER_ERROR, maybe until we've read all the input. I think it can still catch the mismatch between expected and actual size when "fill(1)" gets an EOF, if it's not too late.Quoted 7 lines
That is only one half of the two possible failure cases, i.e. input is shorter than the expected size. If the caller specified size is smaller than what the stream inflates to, I do not see the new code to be limiting the .avail_out near the end of the iteration, which would be necessary to catch such an error, even if we are not interested in using the inflated contents, no?
Yes, you are right.
Instead of always using a fixed "bufsize" even if there is not enough expected output remaining, we can get a more accurate one by comparing "
total_out" to "size", so we can catch problems early by gettingZ_BUFFER_ERROR.builtin/unpack-objects.c +2
-1@@ -139,7 +139,8 @@ static void *get_data(unsigned long size)if (dry_run) {/* reuse the buffer in dry_run mode */ stream.next_out = buf; - stream.avail_out = bufsize; + stream.avail_out = bufsize > size - stream.total_out ? + size - stream.total_out : bufsize; } }git_inflate_end (&stream);Quoted 2 lines
Thanks
-Han XinHan, do you want to pick this up again for a v14? It looks like you're very on top of it already, and I re-sent your patches because I saw that your
https://lore.kernel.org/git/[email protected]/
wasn't picked up in the interim & you hadn't been active on-list otherwise.
Quoted 4 lines
But it looks like there's some interest now, and that you have more time to test & follow-up on this topic than I do at the moment, so if you wanted to do the work of properly rebasing ot in tho recent fsync changes that would be great. Thanks.
OK, I am glad to do that.
Thank you very much.
-Han Xin
PATCH v13 2/7 object-file.c: do fsync() and close() before post-write die()
In reply to Ævar Arnfjörð BjarmasonChange write_loose_object() to do an fsync() and close() before the oideq() sanity check at the end. This change re-joins code that was split up by the die() sanity check added in 748af44c63e (sha1_file: be paranoid when creating loose objects, 2010-02-21).
I don't think that this change matters in itself, if we called die() it was possible that our data wouldn't fully make it to disk, but in any case we were writing data that we'd consider corrupted. It's possible that a subsequent "git fsck" will be less confused now.
The real reason to make this change is that in a subsequent commit we'll split this code in write_loose_object() into a utility function, all its callers will want the preceding sanity checks, but not the "oideq" check. By moving the close_loose_object() earlier it'll be easier to reason about the introduction of the utility function.
--- object-file.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
object-file.c +2 -2
@@ -2012,12 +2012,12 @@ static int write_loose_object(const struct object_id *oid, char *hdr,die (_ ("deflateEnd on object %s failed (%d)" ),oid_to_hex (oid), ret); the_hash_algo->final_oid_fn (¶no_oid, &c); +close_loose_object (fd, tmp_file.buf); +if (!oideq (oid, ¶no_oid))die (_ ("confused by unstable object source data for %s" ),oid_to_hex (oid)); -close_loose_object (fd, tmp_file.buf); -if (mtime) {struct utimbuf utb; utb.actime = mtime;
Signature · 1 line
2.36.1.1124.g52838f02905
Re: [PATCH v13 2/7] object-file.c: do fsync() and close() before post-write die()
Quoted 9 lines · Ævar Arnfjörð Bjarmason
Change
write_loose_object()to do anfsync()andclose()before theoideq()sanity check at the end. This change re-joins code that was split up by thedie()sanity check added in748af44c63e(sha1_file: be paranoid when creating loose objects, 2010-02-21).I don't think that this change matters in itself, if we called
die()it was possible that our data wouldn't fully make it to disk, but in any case we were writing data that we'd consider corrupted. It's possible that a subsequent "git fsck" will be less confused now.
write_loose_object()
- prepares a temporary file
- deflates into the temporary file
- closes and syncs it
- moves the temporary file to the final locaiton
And any die() inserted in between any of these steps will cause the corrupt temporary file not to become the final loose object file.
So, "git fsck" does not need this change at all.
Quoted 5 lines
The real reason to make this change is that in a subsequent commit we'll split this code in
write_loose_object()into a utility function, all its callers will want the preceding sanity checks, but not the "oideq" check. By moving theclose_loose_object()earlier it'll be easier to reason about the introduction of the utility function.
If a "split" relies on the "close and sync" step being in any particular place, that smells really fishy. Is the series loosening the object integrity check? Are we adding some exploitable hole into our codebase without people knowing, or something? I am not sure if I am following the above logic.
Quoted 24 lines
Signed-off-by Ævar Arnfjörð Bjarmason <[email protected]>--- object-file.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)object-file.c +2
-2@@ -2012,12 +2012,12 @@ static int write_loose_object(const struct object_id *oid, char *hdr,die (_ ("deflateEnd on object %s failed (%d)" ),oid_to_hex (oid), ret); the_hash_algo->final_oid_fn (¶no_oid, &c); +close_loose_object (fd, tmp_file.buf); +if (!oideq (oid, ¶no_oid))die (_ ("confused by unstable object source data for %s" ),oid_to_hex (oid)); -close_loose_object (fd, tmp_file.buf); -if (mtime) {struct utimbuf utb; utb.actime = mtime;
PATCH v13 3/7 object-file.c: refactor write_loose_object() to several steps
In reply to Ævar Arnfjörð BjarmasonFrom: Han Xin <[email protected]>
When writing a large blob using "write_loose_object()", we have to pass a buffer with the whole content of the blob, and this behavior will consume lots of memory and may cause OOM. We will introduce a stream version function ("stream_loose_object()") in later commit to resolve this issue.
Before introducing that streaming function, do some refactoring on "write_loose_object()" to reuse code for both versions.
Rewrite "write_loose_object()" as follows:
1. Figure out a path for the (temp) object file. This step is only used in "write_loose_object()".
2. Move common steps for starting to write loose objects into a new function "start_loose_object_common()".
3. Compress data.
4. Move common steps for ending zlib stream into a new function
"end_loose_object_common()".
5. Close fd and finalize the object file.
--- object-file.c | 102 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 76 insertions(+), 26 deletions(-)
object-file.c +76 -26
@@ -1951,6 +1951,75 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)return fd; } +/** +* Common steps for loose object writers to start writing loose +* objects: +* +* - Create tmpfile for the loose object. +* - Setup zlib stream for compression. +* - Start to feed header to zlib stream. +* +* Returns a "fd", which should later be provided to +* end_loose_object_common(). +*/ +static int start_loose_object_common (struct strbuf *tmp_file, +const char *filename,unsigned flags, +git_zstream *stream, +unsigned char *buf,size_t buflen, +git_hash_ctx *c, +char *hdr,int hdrlen) +{ +int fd; + + fd =create_tmpfile (tmp_file, filename); +if (fd <0 ) { +if (flags &HASH_SILENT ) +return -1 ; +else if (errno ==EACCES ) +return error (_ ("insufficient permission for adding " +"an object to repository database %s" ), +get_object_directory ()); +else +return error_errno ( +_ ("unable to create temporary file" )); + } + +/* Setup zlib stream for compression */ +git_deflate_init (stream, zlib_compression_level); + stream->next_out = buf; + stream->avail_out = buflen; + the_hash_algo->init_fn (c); + +/* Start to feed header to zlib stream */ + stream->next_in = (unsigned char *)hdr; + stream->avail_in = hdrlen; +while (git_deflate (stream,0 ) ==Z_OK ) + ;/* nothing */ + the_hash_algo->update_fn (c, hdr, hdrlen); + +return fd; +} + +/** +* Common steps for loose object writers to end writing loose objects: +* +* - End the compression of zlib stream. +* - Get the calculated oid to "oid". +* - fsync() and close() the "fd" +*/ +static int end_loose_object_common (git_hash_ctx *c,git_zstream *stream, +struct object_id *oid) +{ +int ret; + + ret =git_deflate_end_gently (stream); +if (ret !=Z_OK ) +return ret; + the_hash_algo->final_oid_fn (oid, c); + +return Z_OK ; +} +static int write_loose_object (const struct object_id *oid,char *hdr,int hdrlen,const void *buf,unsigned long len,time_t mtime,unsigned flags)
@@ -1968,28 +2037,11 @@ static int write_loose_object(const struct object_id *oid, char *hdr,loose_object_path (the_repository, &filename, oid); - fd =create_tmpfile (&tmp_file, filename.buf); -if (fd <0 ) { -if (flags &HASH_SILENT ) -return -1 ; -else if (errno ==EACCES ) -return error (_ ("insufficient permission for adding an object to repository database %s" ),get_object_directory ()); -else -return error_errno (_ ("unable to create temporary file" )); - } - -/* Set it up */ -git_deflate_init (&stream, zlib_compression_level); - stream.next_out = compressed; - stream.avail_out =sizeof (compressed); - the_hash_algo->init_fn (&c); - -/* First header.. */ - stream.next_in = (unsigned char *)hdr; - stream.avail_in = hdrlen; -while (git_deflate (&stream,0 ) ==Z_OK ) - ;/* nothing */ - the_hash_algo->update_fn (&c, hdr, hdrlen); + fd =start_loose_object_common (&tmp_file, filename.buf, flags, + &stream, compressed,sizeof (compressed), + &c, hdr, hdrlen); +if (fd <0 ) +return -1 ;/* Then the data itself.. */ stream.next_in = (void *)buf;
@@ -2007,11 +2059,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr,if (ret !=Z_STREAM_END )die (_ ("unable to deflate new object %s (%d)" ),oid_to_hex (oid), ret); - ret =git_deflate_end_gently (&stream); + ret =end_loose_object_common (&c, &stream, ¶no_oid);if (ret !=Z_OK ) -die (_ ("deflateEnd on object %s failed (%d)" ),oid_to_hex (oid), - ret); - the_hash_algo->final_oid_fn (¶no_oid, &c); +die (_ ("deflateEnd on object %s failed (%d)" ),oid_to_hex (oid), ret);close_loose_object (fd, tmp_file.buf);if (!oideq (oid, ¶no_oid))
Signature · 1 line
2.36.1.1124.g52838f02905
PATCH v13 4/7 object-file.c: factor out deflate part of write_loose_object()
In reply to Ævar Arnfjörð BjarmasonSplit out the part of write_loose_object() that deals with calling git_deflate() into a utility function, a subsequent commit will introduce another function that'll make use of it.
--- object-file.c | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-)
object-file.c +25 -6
@@ -2000,6 +2000,28 @@ static int start_loose_object_common(struct strbuf *tmp_file,return fd; } +/** +* Common steps for the inner git_deflate() loop for writing loose +* objects. Returns what git_deflate() returns. +*/ +static int write_loose_object_common (git_hash_ctx *c, +git_zstream *stream,const int flush, +unsigned char *in0,const int fd, +unsigned char *compressed, +const size_t compressed_len) +{ +int ret; + + ret =git_deflate (stream, flush ?Z_FINISH :0 ); + the_hash_algo->update_fn (c, in0, stream->next_in - in0); +if (write_buffer (fd, compressed, stream->next_out - compressed) <0 ) +die (_ ("unable to write loose object file" )); + stream->next_out = compressed; + stream->avail_out = compressed_len; + +return ret; +} +/** * Common steps for loose object writers to end writing loose objects: *
@@ -2048,12 +2070,9 @@ static int write_loose_object(const struct object_id *oid, char *hdr, stream.avail_in = len;do {unsigned char *in0 = stream.next_in; - ret =git_deflate (&stream,Z_FINISH ); - the_hash_algo->update_fn (&c, in0, stream.next_in - in0); -if (write_buffer (fd, compressed, stream.next_out - compressed) <0 ) -die (_ ("unable to write loose object file" )); - stream.next_out = compressed; - stream.avail_out =sizeof (compressed); + + ret =write_loose_object_common (&c, &stream,1 , in0, fd, + compressed,sizeof (compressed)); }while (ret ==Z_OK );if (ret !=Z_STREAM_END )
Signature · 1 line
2.36.1.1124.g52838f02905
PATCH v13 5/7 object-file.c: add "stream_loose_object()" to handle large object
In reply to Ævar Arnfjörð BjarmasonFrom: Han Xin <[email protected]>
If we want unpack and write a loose object using "write_loose_object", we have to feed it with a buffer with the same size of the object, which will consume lots of memory and may cause OOM. This can be improved by feeding data to "stream_loose_object()" in a stream.
Add a new function "stream_loose_object()", which is a stream version of "write_loose_object()" but with a low memory footprint. We will use this function to unpack large blob object in later commit.
Another difference with "write_loose_object()" is that we have no chance to run "write_object_file_prepare()" to calculate the oid in advance. In "write_loose_object()", we know the oid and we can write the temporary file in the same directory as the final object, but for an object with an undetermined oid, we don't know the exact directory for the object.
Still, we need to save the temporary file we're preparing somewhere. We'll do that in the top-level ".git/objects/" directory (or whatever "GIT_OBJECT_DIRECTORY" is set to). Once we've streamed it we'll know the OID, and will move it to its canonical path.
"freshen_packed_object()" or "freshen_loose_object()" will be called inside "stream_loose_object()" after obtaining the "oid".
--- object-file.c | 100 +++++++++++++++++++++++++++++++++++++++++++++++++ object-store.h | 8 ++++ 2 files changed, 108 insertions(+)
object-file.c +100 -0
@@ -2119,6 +2119,106 @@ static int freshen_packed_object(const struct object_id *oid)return 1 ; } +int stream_loose_object (struct input_stream *in_stream,size_t len, +struct object_id *oid) +{ +int fd, ret, err =0 , flush =0 ; +unsigned char compressed[4096 ]; +git_zstream stream; +git_hash_ctx c; +struct strbuf tmp_file =STRBUF_INIT ; +struct strbuf filename =STRBUF_INIT ; +int dirlen; +char hdr[MAX_HEADER_LEN ]; +int hdrlen; + +/* Since oid is not determined, save tmp file to odb path. */ +strbuf_addf (&filename,"%s/" ,get_object_directory ()); + hdrlen =format_object_header (hdr,sizeof (hdr),OBJ_BLOB , len); + +/* +* Common steps for write_loose_object and stream_loose_object to +* start writing loose objects: +* +* - Create tmpfile for the loose object. +* - Setup zlib stream for compression. +* - Start to feed header to zlib stream. +*/ + fd =start_loose_object_common (&tmp_file, filename.buf,0 , + &stream, compressed,sizeof (compressed), + &c, hdr, hdrlen); +if (fd <0 ) { + err =-1 ; + goto cleanup; + } + +/* Then the data itself.. */ +do { +unsigned char *in0 = stream.next_in; + +if (!stream.avail_in && !in_stream->is_finished) { +const void *in = in_stream->read (in_stream, &stream.avail_in); + stream.next_in = (void *)in; + in0 = (unsigned char *)in; +/* All data has been read. */ +if (in_stream->is_finished) + flush =1 ; + } + ret =write_loose_object_common (&c, &stream, flush, in0, fd, + compressed,sizeof (compressed)); +/* +* Unlike write_loose_object(), we do not have the entire +* buffer. If we get Z_BUF_ERROR due to too few input bytes, +* then we'll replenish them in the next input_stream->read() +* call when we loop. +*/ + }while (ret ==Z_OK || ret ==Z_BUF_ERROR ); + +if (stream.total_in != len + hdrlen) +die (_ ("write stream object %ld != %" PRIuMAX), stream.total_in, + (uintmax_t )len + hdrlen); + +/* Common steps for write_loose_object and stream_loose_object to +* end writing loose oject: +* +* - End the compression of zlib stream. +* - Get the calculated oid. +*/ +if (ret !=Z_STREAM_END ) +die (_ ("unable to stream deflate new object (%d)" ), ret); + ret =end_loose_object_common (&c, &stream, oid); +if (ret !=Z_OK ) +die (_ ("deflateEnd on stream object failed (%d)" ), ret); +close_loose_object (fd, tmp_file.buf); + +if (freshen_packed_object (oid) ||freshen_loose_object (oid)) { +unlink_or_warn (tmp_file.buf); + goto cleanup; + } + +loose_object_path (the_repository, &filename, oid); + +/* We finally know the object path, and create the missing dir. */ + dirlen =directory_size (filename.buf); +if (dirlen) { +struct strbuf dir =STRBUF_INIT ; +strbuf_add (&dir, filename.buf, dirlen); + +if (mkdir_in_gitdir (dir.buf) && errno !=EEXIST ) { + err =error_errno (_ ("unable to create directory %s" ), dir.buf); +strbuf_release (&dir); + goto cleanup; + } +strbuf_release (&dir); + } + + err =finalize_object_file (tmp_file.buf, filename.buf); +cleanup: +strbuf_release (&tmp_file); +strbuf_release (&filename); +return err; +} +int write_object_file_flags (const void *buf,unsigned long len,enum object_type type,struct object_id *oid,unsigned flags)
object-store.h +8 -0
@@ -46,6 +46,12 @@ struct object_directory {char *path; }; +struct input_stream { +const void *(*read)(struct input_stream *,unsigned long *len); +void *data; +int is_finished; +}; + KHASH_INIT(odb_path_map, const char * /* key: odb_path */, struct object_directory *, 1, fspathhash, fspatheq)
@@ -269,6 +275,8 @@ static inline int write_object_file(const void *buf, unsigned long len,int write_object_file_literally (const void *buf,unsigned long len,const char *type,struct object_id *oid,unsigned flags); +int stream_loose_object (struct input_stream *in_stream,size_t len, +struct object_id *oid);/* * Add an object file to the in-memory object store, without writing it
Signature · 1 line
2.36.1.1124.g52838f02905
Re: [PATCH v13 5/7] object-file.c: add "stream_loose_object()" to handle large object
Quoted 10 lines · Ævar Arnfjörð Bjarmason
From: Han Xin <[email protected]>
If we want unpack and write a loose object using "
write_loose_object", we have to feed it with a buffer with the same size of the object, which will consume lots of memory and may cause OOM. This can be improved by feeding data to "stream_loose_object()" in a stream.Add a new function "
stream_loose_object()", which is a stream version of "write_loose_object()" but with a low memory footprint. We will use this function to unpack large blob object in later commit.
Yay.
Quoted 2 lines
Another difference with "
write_loose_object()" is that we have no chance to run "write_object_file_prepare()" to calculate the oid in advance.
That is somewhat curious. Is it fundamentally impossible, or is it just that this patch was written in such a way that conflates the two and it is cumbersome to split the "we repeat the sequence of reading and deflating just a bit until we process all" and the "we compute the hash over the data first and then we write out for real"?
Quoted 10 lines
In "
write_loose_object()", we know the oid and we can write the temporary file in the same directory as the final object, but for an object with an undetermined oid, we don't know the exact directory for the object.Still, we need to save the temporary file we're preparing somewhere. We'll do that in the top-level ".git/objects/" directory (or whatever "
GIT_OBJECT_DIRECTORY" is set to). Once we've streamed it we'll know the OID, and will move it to its canonical path.
This may have negative implications on some filesystems where cross directory links do not work atomically, but it is a small price to pay.
I am very tempted to ask why we do not do this to _all_ loose object files. Instead of running the machinery twice over the data (once to compute the object name, then to compute the contents and write out), if we can produce loose object files of any size with a single pass, wouldn't that be an overall win?
Is the fixed overhead, i.e. cost of setting up the streaming interface, reasonably large to make it not worth doing for smaller objects?
Quoted 2 lines
"
freshen_packed_object()" or "freshen_loose_object()" will be called inside "stream_loose_object()" after obtaining the "oid".
That much we can read from the patch text. Saying just "we do X" without explaining "why we do so" in the proposed log message leaves readers more confused than otherwise. Why is it worth pointing out in the proposed log message? Is the reason why we need to do so involve something tricky?
Quoted 50 lines
object-file.c +50
-0+int stream_loose_object (struct input_stream *in_stream,size_t len, +struct object_id *oid) +{ +int fd, ret, err =0 , flush =0 ; +unsigned char compressed[4096 ]; +git_zstream stream; +git_hash_ctx c; +struct strbuf tmp_file =STRBUF_INIT ; +struct strbuf filename =STRBUF_INIT ; +int dirlen; +char hdr[MAX_HEADER_LEN ]; +int hdrlen; + +/* Since oid is not determined, save tmp file to odb path. */ +strbuf_addf (&filename,"%s/" ,get_object_directory ()); + hdrlen =format_object_header (hdr,sizeof (hdr),OBJ_BLOB , len); + +/* +* Common steps for write_loose_object and stream_loose_object to +* start writing loose objects: +* +* - Create tmpfile for the loose object. +* - Setup zlib stream for compression. +* - Start to feed header to zlib stream. +*/ + fd =start_loose_object_common (&tmp_file, filename.buf,0 , + &stream, compressed,sizeof (compressed), + &c, hdr, hdrlen); +if (fd <0 ) { + err =-1 ; + goto cleanup; + } + +/* Then the data itself.. */ + do { + unsigned char *in0 = stream.next_in; + + if (!stream.avail_in && !in_stream->is_finished) { + const void *in = in_stream->read(in_stream, &stream.avail_in); + stream.next_in = (void *)in; + in0 = (unsigned char *)in; + /* All data has been read. */ + if (in_stream->is_finished) + flush = 1; + } + ret = write_loose_object_common(&c, &stream, flush, in0, fd, + compressed, sizeof(compressed)); + /* + * Unlike write_loose_object(), we do not have the entire + * buffer. If we get Z_BUF_ERROR due to too few input bytes,
object-file.c static int freshen_packed_object(const struct object_id *oid) · line 2181
Quoted 10 lines
object-file.c +8
-0+* then we'll replenish them in the next input_stream->read() +* call when we loop. +*/ + }while (ret ==Z_OK || ret ==Z_BUF_ERROR ); +if (stream.total_in != len + hdrlen) +die (_ ("write stream object %ld != %" PRIuMAX), stream.total_in, + (uintmax_t )len + hdrlen); +/* Common steps for write_loose_object and stream_loose_object to
Style.
object-file.c static int freshen_packed_object(const struct object_id *oid) · line 2188
Quoted 7 lines
object-file.c +7
-0+* end writing loose oject: +* +* - End the compression of zlib stream. +* - Get the calculated oid. +*/ +if (ret !=Z_STREAM_END ) +die (_ ("unable to stream deflate new object (%d)" ), ret);
Good to check this, after the loop exits above. I was expecting to see it immediately after the loop, but here is also OK.
object-file.c static int freshen_packed_object(const struct object_id *oid) · line 2196
Quoted 8 lines
object-file.c +8
-0+ ret =end_loose_object_common (&c, &stream, oid); +if (ret !=Z_OK ) +die (_ ("deflateEnd on stream object failed (%d)" ), ret); +close_loose_object (fd, tmp_file.buf); + +if (freshen_packed_object (oid) ||freshen_loose_object (oid)) { +unlink_or_warn (tmp_file.buf); + goto cleanup;
So, we were told to write an object, we wrote to a temporary file, and we wanted to mark the object to be recent and found that there indeed is already the object. We remove the temporary and do not leave the new copy of the object, and the value of err at this point is 0 (success) which is what is returned from cleanup: label.
Good.
Quoted 25 lines
object-file.c +25
-0+ } + +loose_object_path (the_repository, &filename, oid); + +/* We finally know the object path, and create the missing dir. */ + dirlen =directory_size (filename.buf); +if (dirlen) { +struct strbuf dir =STRBUF_INIT ; +strbuf_add (&dir, filename.buf, dirlen); + +if (mkdir_in_gitdir (dir.buf) && errno !=EEXIST ) { + err =error_errno (_ ("unable to create directory %s" ), dir.buf); +strbuf_release (&dir); + goto cleanup; + } +strbuf_release (&dir); + } + + err =finalize_object_file (tmp_file.buf, filename.buf); +cleanup: +strbuf_release (&tmp_file); +strbuf_release (&filename); +return err; +} +
Re: [PATCH v13 5/7] object-file.c: add "stream_loose_object()" to handle large object
Quoted 9 lines · Junio C Hamano
Quoted 2 lines
Another difference with "
write_loose_object()" is that we have no chance to run "write_object_file_prepare()" to calculate the oid in advance.That is somewhat curious. Is it fundamentally impossible, or is it just that this patch was written in such a way that conflates the two and it is cumbersome to split the "we repeat the sequence of reading and deflating just a bit until we process all" and the "we compute the hash over the data first and then we write out for real"?
OK, the answer lies somewhere in between.
The initial user of this streaming interface reads from an incoming packfile and feeds the inflated bytestream to the interface, which means we cannot seek. That meaks it "fundamentally impossible" for that codepath (i.e. unpack-objects to read from packstream and write to on-disk loose objects).
But if the input source is seekable (e.g. a file in the working tree), there is no fundamental reason why the new interface has "no chance to run prepare to calculate the oid in advance". It's just that the such a different caller is not added by the series and we chose not to allow the "prepare and then write" two-step process, because we currently do not need it when this series lands.
Quoted 5 lines
I am very tempted to ask why we do not do this to _all_ loose object files. Instead of running the machinery twice over the data (once to compute the object name, then to compute the contents and write out), if we can produce loose object files of any size with a single pass, wouldn't that be an overall win?
There is a patch later in the series whose proposed log message has benchmarks to show that it is slower in general. It still is curious where the slowness comes from and if it is something we can tune, though.
Thanks.
Re: [PATCH v13 5/7] object-file.c: add "stream_loose_object()" to handle large object
Quoted 9 lines · Junio C Hamano
Quoted 5 lines · Junio C Hamano
I am very tempted to ask why we do not do this to _all_ loose object files. Instead of running the machinery twice over the data (once to compute the object name, then to compute the contents and write out), if we can produce loose object files of any size with a single pass, wouldn't that be an overall win?
Quoted 5 lines
There is a patch later in the series whose proposed log message has benchmarks to show that it is slower in general. It still is curious where the slowness comes from and if it is something we can tune, though.
Compared with getting the whole object buffer, stream_loose_object() uses limited avail_in buffer and never fill new content until the whole avail_in has been
deflated. It will generate small avail_in fragments due to the limited avail_out,
and I think it is precisely because these avail_in fragments generate additional git_deflate() loops.
In "unpack-objects", we use a buffer size of 8192. Increasing the buffer can alleviate this problem, but maybe it's not worth it?
Quoted 1 line
Thanks.
Re: [PATCH v13 5/7] object-file.c: add "stream_loose_object()" to handle large object
In reply to Junio C HamanoQuoted 11 lines · Junio C Hamano
Quoted 4 lines
"
freshen_packed_object()" or "freshen_loose_object()" will be called inside "stream_loose_object()" after obtaining the "oid".That much we can read from the patch text. Saying just "we do X" without explaining "why we do so" in the proposed log message leaves readers more confused than otherwise. Why is it worth pointing out in the proposed log message? Is the reason why we need to do so involve something tricky?
Yes, it really should be made clear why this is done here.
Thanks.
-Han Xin
Quoted 17 lines
object-file.c static int freshen_packed_object(const struct object_id *oid) · line 2196
Quoted 8 lines
object-file.c +8
-0+ ret =end_loose_object_common (&c, &stream, oid); +if (ret !=Z_OK ) +die (_ ("deflateEnd on stream object failed (%d)" ), ret); +close_loose_object (fd, tmp_file.buf); + +if (freshen_packed_object (oid) ||freshen_loose_object (oid)) { +unlink_or_warn (tmp_file.buf); + goto cleanup;So, we were told to write an object, we wrote to a temporary file, and we wanted to mark the object to be recent and found that there indeed is already the object. We remove the temporary and do not leave the new copy of the object, and the value of err at this point is 0 (success) which is what is returned from cleanup: label.
Good.
Re: [PATCH v13 5/7] object-file.c: add "stream_loose_object()" to handle large object
In reply to Ævar Arnfjörð BjarmasonQuoted 18 lines · Ævar Arnfjörð Bjarmason
From: Han Xin <[email protected]>
If we want unpack and write a loose object using "
write_loose_object", we have to feed it with a buffer with the same size of the object, which will consume lots of memory and may cause OOM. This can be improved by feeding data to "stream_loose_object()" in a stream.Add a new function "
stream_loose_object()", which is a stream version of "write_loose_object()" but with a low memory footprint. We will use this function to unpack large blob object in later commit.Another difference with "
write_loose_object()" is that we have no chance to run "write_object_file_prepare()" to calculate the oid in advance. In "write_loose_object()", we know the oid and we can write the temporary file in the same directory as the final object, but for an object with an undetermined oid, we don't know the exact directory for the object.
Quoted 6 lines
Still, we need to save the temporary file we're preparing somewhere. We'll do that in the top-level ".git/objects/" directory (or whatever "
GIT_OBJECT_DIRECTORY" is set to). Once we've streamed it we'll know the OID, and will move it to its canonical path.
I think this new logic doesn't play well with batched-fsync. Even through we don't know the final OID, we should still call prepare_loose_object_bulk_checkin to potentially create the bulk checkin objdir.
Quoted 70 lines
object-file.c +62
-0@@ -2119,6 +2119,106 @@ static int freshen_packed_object(const struct object_id *oid)return 1 ; } +int stream_loose_object (struct input_stream *in_stream,size_t len, +struct object_id *oid) +{ +int fd, ret, err =0 , flush =0 ; +unsigned char compressed[4096 ]; +git_zstream stream; +git_hash_ctx c; +struct strbuf tmp_file =STRBUF_INIT ; +struct strbuf filename =STRBUF_INIT ; +int dirlen; +char hdr[MAX_HEADER_LEN ]; +int hdrlen; + +/* Since oid is not determined, save tmp file to odb path. */ +strbuf_addf (&filename,"%s/" ,get_object_directory ()); + hdrlen =format_object_header (hdr,sizeof (hdr),OBJ_BLOB , len); + +/* +* Common steps for write_loose_object and stream_loose_object to +* start writing loose objects: +* +* - Create tmpfile for the loose object. +* - Setup zlib stream for compression. +* - Start to feed header to zlib stream. +*/ + fd =start_loose_object_common (&tmp_file, filename.buf,0 , + &stream, compressed,sizeof (compressed), + &c, hdr, hdrlen); +if (fd <0 ) { + err =-1 ; + goto cleanup; + } + +/* Then the data itself.. */ +do { +unsigned char *in0 = stream.next_in; + +if (!stream.avail_in && !in_stream->is_finished) { +const void *in = in_stream->read (in_stream, &stream.avail_in); + stream.next_in = (void *)in; + in0 = (unsigned char *)in; +/* All data has been read. */ +if (in_stream->is_finished) + flush =1 ; + } + ret =write_loose_object_common (&c, &stream, flush, in0, fd, + compressed,sizeof (compressed)); +/* +* Unlike write_loose_object(), we do not have the entire +* buffer. If we get Z_BUF_ERROR due to too few input bytes, +* then we'll replenish them in the next input_stream->read() +* call when we loop. +*/ + }while (ret ==Z_OK || ret ==Z_BUF_ERROR ); + +if (stream.total_in != len + hdrlen) +die (_ ("write stream object %ld != %" PRIuMAX), stream.total_in, + (uintmax_t )len + hdrlen); + +/* Common steps for write_loose_object and stream_loose_object to +* end writing loose oject: +*
object-file.c static int freshen_packed_object(const struct object_id *oid) · line 2193
Quoted 10 lines
object-file.c +10
-0+* - End the compression of zlib stream. +* - Get the calculated oid. +*/ +if (ret !=Z_STREAM_END ) +die (_ ("unable to stream deflate new object (%d)" ), ret); + ret =end_loose_object_common (&c, &stream, oid); +if (ret !=Z_OK ) +die (_ ("deflateEnd on stream object failed (%d)" ), ret); +close_loose_object (fd, tmp_file.buf); +
If batch fsync is enabled, the close_loose_object call will refrain from syncing the tmp file.
object-file.c static int freshen_packed_object(const struct object_id *oid) · line 2200
Quoted 7 lines
object-file.c +7
-0+if (freshen_packed_object (oid) ||freshen_loose_object (oid)) { +unlink_or_warn (tmp_file.buf); + goto cleanup; + } + +loose_object_path (the_repository, &filename, oid); +
We expect this loose_object_path call to return a path in the bulk fsync object directory. It might not do so if we don't call prepare_loose_object_bulk_checkin.
In the new test case introduced in (7/7), we seem to be getting lucky in that there are some small objects (commits) earlier in the packfile, so we go through write_loose_object first.
Thanks for including me on the review!
-Neeraj
Re: [PATCH v13 5/7] object-file.c: add "stream_loose_object()" to handle large object
Quoted 11 lines · Neeraj Singh
Quoted 6 lines
Still, we need to save the temporary file we're preparing somewhere. We'll do that in the top-level ".git/objects/" directory (or whatever "
GIT_OBJECT_DIRECTORY" is set to). Once we've streamed it we'll know the OID, and will move it to its canonical path.I think this new logic doesn't play well with batched-fsync. Even through we don't know the final OID, we should still call
prepare_loose_object_bulk_checkinto potentially create the bulk checkin objdir.
Good point. Careful sanity checks like this are very much appreciated.
Quoted 1 line
Thanks for including me on the review!
Yes, indeed.
Thanks, both.
RFC PATCH 1/1 object-file.c: batched disk flushes for stream_loose_object()
In reply to Neeraj SinghNeeraj Singh[1] pointed out that if batch fsync is enabled, we should still call prepare_loose_object_bulk_checkin() to potentially create the bulk checkin objdir.
1. https://lore.kernel.org/git/[email protected]/
--- object-file.c | 3 +++ t/t5351-unpack-large-objects.sh | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-)
object-file.c +3 -0
@@ -2131,6 +2131,9 @@ int stream_loose_object(struct input_stream *in_stream, size_t len,char hdr[MAX_HEADER_LEN ];int hdrlen; +if (batch_fsync_enabled (FSYNC_COMPONENT_LOOSE_OBJECT )) +prepare_loose_object_bulk_checkin (); +/* Since oid is not determined, save tmp file to odb path. */ strbuf_addf (&filename,"%s/" ,get_object_directory ()); hdrlen =format_object_header (hdr,sizeof (hdr),OBJ_BLOB , len);
t/t5351-unpack-large-objects.sh +14 -1
@@ -18,7 +18,10 @@ test_expect_success "create large objects (1.5 MB) and PACK" ' test_commit --append foo big-blob && test-tool genrandom bar 1500000 >big-blob && test_commit --append bar big-blob && - PACK=$(echo HEAD | git pack-objects --revs pack) + PACK=$(echo HEAD | git pack-objects --revs pack) && + git verify-pack -v pack-$PACK.pack | + grep -E "commit|tree|blob" | + sed -n -e "s/^\([0-9a-f]*\).*/\1/p" >obj-list ' test_expect_success 'set memory limitation to 1MB' '
@@ -45,6 +48,16 @@ test_expect_success 'unpack big object in stream' ' test_dir_is_empty dest.git/objects/pack ' +BATCH_CONFIGURATION='-c core.fsync=loose-object -c core.fsyncmethod=batch' + +test_expect_success 'unpack big object in stream (core.fsyncmethod=batch)' ' + prepare_dest 1m && + git $BATCH_CONFIGURATION -C dest.git unpack-objects <pack-$PACK.pack && + test_dir_is_empty dest.git/objects/pack && + git -C dest.git cat-file --batch-check="%(objectname)" <obj-list >current && + cmp obj-list current +' + test_expect_success 'do not unpack existing large objects' ' prepare_dest 1m && git -C dest.git index-pack --stdin <pack-$PACK.pack &&
Signature · 1 line
2.36.1
Re: [RFC PATCH] object-file.c: batched disk flushes for stream_loose_object()
Quoted 49 lines · Han Xin
Neeraj Singh[1] pointed out that if batch fsync is enabled, we should still call
prepare_loose_object_bulk_checkin()to potentially create the bulk checkin objdir.1. https://lore.kernel.org/git/[email protected]/
Signed-off-by Han Xin <[email protected]>--- object-file.c | 3 +++ t/t5351-unpack-large-objects.sh | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-)object-file.c +3
-0@@ -2131,6 +2131,9 @@ int stream_loose_object(struct input_stream *in_stream, size_t len,char hdr[MAX_HEADER_LEN ];int hdrlen; +if (batch_fsync_enabled (FSYNC_COMPONENT_LOOSE_OBJECT )) +prepare_loose_object_bulk_checkin (); +/* Since oid is not determined, save tmp file to odb path. */ strbuf_addf (&filename,"%s/" ,get_object_directory ()); hdrlen =format_object_header (hdr,sizeof (hdr),OBJ_BLOB , len);t/t5351-unpack-large-objects.sh +7
-1@@ -18,7 +18,10 @@ test_expect_success "create large objects (1.5 MB) and PACK" ' test_commit --append foo big-blob && test-tool genrandom bar 1500000 >big-blob && test_commit --append bar big-blob && - PACK=$(echo HEAD | git pack-objects --revs pack) + PACK=$(echo HEAD | git pack-objects --revs pack) && + git verify-pack -v pack-$PACK.pack | + grep -E "commit|tree|blob" | + sed -n -e "s/^\([0-9a-f]*\).*/\1/p" >obj-list ' test_expect_success 'set memory limitation to 1MB' '@@ -45,6 +48,16 @@ test_expect_success 'unpack big object in stream' ' test_dir_is_empty dest.git/objects/pack ' +BATCH_CONFIGURATION='-c core.fsync=loose-object -c core.fsyncmethod=batch' + +test_expect_success 'unpack big object in stream (core.fsyncmethod=batch)' '
t/t5351-unpack-large-objects.sh test_expect_success 'unpack big object in stream' ' · line 63
Quoted 10 lines
t/t5351-unpack-large-objects.sh +7
-0+ prepare_dest 1m && + git $BATCH_CONFIGURATION -C dest.git unpack-objects <pack-$PACK.pack && + test_dir_is_empty dest.git/objects/pack && + git -C dest.git cat-file --batch-check="%(objectname)" <obj-list >current && + cmp obj-list current +' + test_expect_success 'do not unpack existing large objects' ' prepare_dest 1m && git -C dest.git index-pack --stdin <pack-$PACK.pack &&
This fix looks good to me.
Thanks.
-Neeraj