Prevent megablobs from gunking up git packs
PATCH v2 1/1 Prevent megablobs from gunking up git packs
git stores data in loose blobs or in packfiles. The former has essentially now become an exception mechanism, to store unusually *young* blobs. Here we re-use all this "exception" machinery to store unusually *large* blobs as well.
This patch implements the following:
1. git pack-objects takes a new --max-blob-size=N flag, with the effect that only blobs less than N KB are written to the packfiles(s). If an already packed blob violates this limit (perhaps these are fast-import packs or max-blob-size was reduced), a new loose object is made as needed so data is not lost.
2. git repack inspects repack.maxblobsize . If set, its value is passed to git pack-objects on the command line. --max-blob-size=N is also accepted by git repack.
3. No other git pack-objects caller uses this feature or sees any change.
During pack *creation* this avoids copying & deltifying megablobs. Therefore, for 100GB repositories, git repack can run in hours for the first time after git fast-import, and in minutes thereafter. You can still include megablobs in your packs thus:
% git-repack -a [-d] {with repack.maxblobsize = 256}
% git-repack --max-blob-size=0 [--max-pack-size=2047]
The first step creates a tight packfile and index with all metadata and reasonable blob data, and the second collects the megablobs, which could be enhanced with .keep file manipulation, etc.
During pack *use* this feature helps tighten access to heavily-used metadata since it can't be separated by large blobs, and allows all such metadata to be accessed through the smallest number of index files. Megablobs are accessible with some overhead as loose objects, which can be corrected with a second packing step as shown above at the cost of more index files and packing time.
Documentation has been updated and operation with pack-object's --stdout is prevented. This patch is based on "next".
--- Documentation/config.txt | 5 ++++ Documentation/git-pack-objects.txt | 6 +++++ Documentation/git-repack.txt | 10 ++++++++ builtin-pack-objects.c | 41 +++++++++++++++++++++++++++++++---- cache.h | 2 + git-repack.sh | 9 +++++++- sha1_file.c | 17 +++++++++++++- 7 files changed, 82 insertions(+), 8 deletions(-)
Documentation/config.txt +5 -0
@@ -599,6 +599,11 @@ remotes.<group>:: The list of remotes which are fetched by "git remote update <group>". See gitlink:git-remote[1]. +repack.maxblobsize:: + Prevent gitlink:git-repack[1] from including blobs larger than + the specified number in kB, unless overridden by --max-blob-size=N switch. + Defaults to zero which means no maximum size is in effect. + repack.usedeltabaseoffset:: Allow gitlink:git-repack[1] to create packs that uses delta-base offset. Defaults to false.
Documentation/git-pack-objects.txt +6 -0
@@ -85,6 +85,12 @@ base-name:: times to get to the necessary object. The default value for --window is 10 and --depth is 50. +--max-blob-size=<n>:: + Maximum size of included blobs, expressed in kB. + If specified, affected blobs only existing in packfiles + may be written out as new loose objects to prevent their loss. + The default is unlimited. + --max-pack-size=<n>:: Maximum size of each output packfile, expressed in MiB. If specified, multiple packfiles may be created.
Documentation/git-repack.txt +10 -0
@@ -65,6 +65,12 @@ OPTIONS to be applied that many times to get to the necessary object. The default value for --window is 10 and --depth is 50. +--max-blob-size=<n>:: + Maximum size of included blobs, expressed in kB. + If specified, affected blobs only existing in packfiles + may be written out as new loose objects to prevent their loss. + The default is unlimited. + --max-pack-size=<n>:: Maximum size of each output packfile, expressed in MiB. If specified, multiple packfiles may be created.
@@ -84,6 +90,10 @@ be able to read (this includes repositories from which packs can be copied out over http or rsync, and people who obtained packs that way can try to use older git with it). +The configuration variable `repack.MaxBlobSize` provides the +default for the --max-blob-size option if set. The latter +takes precedence. + Author ------
builtin-pack-objects.c +36 -5
95 lines not on this page · raw message
cache.h +2 -0
@@ -343,6 +343,8 @@ extern int sha1_object_info(const unsigned char *, unsigned long *);extern void *read_sha1_file (const unsigned char *sha1,enum object_type *type,unsigned long *size);extern int hash_sha1_file (const void *buf,unsigned long len,const char *type,unsigned char *sha1);extern int write_sha1_file (void *buf,unsigned long len,const char *type,unsigned char *return_sha1); +extern int write_sha1_file_ignore_packs (void *buf,unsigned long len,const char *type, +unsigned char *return_sha1);extern int pretend_sha1_file (void *,unsigned long ,enum object_type ,unsigned char *);extern int check_sha1_signature (const unsigned char *sha1,void *buf,unsigned long size,const char *type);
git-repack.sh +8 -1
@@ -8,7 +8,7 @@ SUBDIRECTORY_OK='Yes' . git-sh-setup no_update_info= all_into_one= remove_redundant= -local= quiet= no_reuse= extra= +local= quiet= no_reuse= extra= max_blob_size= while case "$#" in 0) break ;; esac do case "$1" in
@@ -18,6 +18,7 @@ do -q) quiet=-q ;; -f) no_reuse=--no-reuse-object ;; -l) local=--local ;; + --max-blob-size=*) extra="$extra $1" max_blob_size=t ;; --max-pack-size=*) extra="$extra $1" ;; --window=*) extra="$extra $1" ;; --depth=*) extra="$extra $1" ;;
@@ -35,6 +36,12 @@ true) extra="$extra --delta-base-offset" ;; esac +# handle blob limiting +if [ -z "$max_blob_size" ]; then + mbs="`git config --int repack.maxblobsize`" + [ -n "$mbs" ] && extra="$extra --max-blob-size=$mbs" +fi + PACKDIR="$GIT_OBJECT_DIRECTORY/pack" PACKTMP="$GIT_OBJECT_DIRECTORY/.tmp-$$-pack" rm -f "$PACKTMP"-*
sha1_file.c +15 -2
@@ -1979,7 +1979,8 @@ int hash_sha1_file(const void *buf, unsigned long len, const char *type,return 0 ; } -int write_sha1_file (void *buf,unsigned long len,const char *type,unsigned char *returnsha1) +static int write_sha1_file_core (void *buf,unsigned long len,const char *type, +int checkpacks,unsigned char *returnsha1) {int size, ret;unsigned char *compressed;
@@ -1997,7 +1998,7 @@ int write_sha1_file(void *buf, unsigned long len, const char *type, unsigned cha filename =sha1_file_name (sha1);if (returnsha1)hashcpy (returnsha1, sha1); -if (has_sha1_file (sha1)) +if (checkpacks &&has_sha1_file (sha1))return 0 ; fd =open (filename,O_RDONLY );if (fd >=0 ) {
@@ -2062,6 +2063,18 @@ int write_sha1_file(void *buf, unsigned long len, const char *type, unsigned chareturn move_temp_to_file (tmpfile, filename); } +int write_sha1_file (void *buf,unsigned long len,const char *type, +unsigned char *returnsha1) +{ +return write_sha1_file_core (buf, len, type,1 , returnsha1); +} + +int write_sha1_file_ignore_packs (void *buf,unsigned long len,const char *type, +unsigned char *returnsha1) +{ +return write_sha1_file_core (buf, len, type,0 , returnsha1); +} +/* * We need to unpack and recompress the object for writing * it out to a different file.
Signature · 1 line
1.5.2.762.gd8c6-dirty