delta compressed git
PATCH 1/1 delta compressed git
Hello everyone,
Here's an early form of some code for delta compression in git archives. It builds on top of my packed item patch from before. Using this patch will create repositories that can't be read by unpatched git, and it is only ready for light testing. The file format might change slightly in later revs.
deltas live as subfiles in packed files, and the packed item header has the sha1 of the file the delta is against. deltas are never taken against deltas, only whole files (so the chain length is only 1).
When importing all of Ingo's bk->cvs patches into git (28,000 changesets), delta git applies the patches faster (2hrs vs 2.5hrs), consumes less space (900MB vs 2.5GB), and checks out the resulting git tree faster in hot and cold caches.
Another 200MB or so would be saved by packing trees and commits into the same files as the blobs. This is easy to do, but makes the patch harder to maintain because I need to move code around in commit-tree.c and write-tree.c. So I've left those bits out for now.
Because the packed files are created per changeset, if a changeset only modifies one file the delta will still end up using a whole block. So, you could get much higher space savings with a tool to walk back over existing changesets and pack them together. This doesn't exist yet, but wouldn't be difficult, and I expect it to get close to the mercurial/bk repository sizes.
The patch uses zdelta for delta compression, which you can download here:
http://cis.poly.edu/zdelta/
I'm open to suggestions on better delta libs. I picked this one because it was easy to code. In order for things to work with git you need to apply the attached zdelta.diff to the zdelta-2.1 sources. It fixes a silly default in the Makefile and a symbol collision with zlib.
-chris
infcodes.c +1 -1
@@ -145,7 +145,7 @@if (m >=MAX_MATCH && n >=10 ) {UPDATE - r =inflate_fast (c->lbits, c->dbits, c->zdbits, + r =zd_inflate_fast (c->lbits, c->dbits, c->zdbits, c->ltree, c->dtree, c->zdtree, s, z);LOAD if (r != ZD_OK)
Only in zdelta-2.1: infcodes.o
inffast.c +2 -2
@@ -8,7 +8,7 @@/* zdelta: * * modified: -* inflate_fast +* zd_inflate_fast * added: * -- * removed:
@@ -41,7 +41,7 @@/* * zdelta: modified */ -int inflate_fast(bl, bd, bzd, tl, td, tzd, s, z) +int zd_inflate_fast(bl, bd, bzd, tl, td, tzd, s, z) uInt bl, bd, bzd; inflate_huft *tl; inflate_huft *td;
inffast.h +1 -1
@@ -22,7 +22,7 @@#ifndef ZD_INFFAST_H #define ZD_INFFAST_H -extern int inflate_fast OF(( +extern int zd_inflate_fast OF(( uInt, uInt, uInt,
Makefile +1 -1
@@ -35,7 +35,7 @@ CC=gcc -CFLAGS= -O2 -W -Wall -pedantic -ansi -g -DREFNUM=2 +CFLAGS= -O2 -W -Wall -pedantic -ansi -g -DREFNUM=1 LDSHARED=$(CC) CPP=$(CC) -E
Makefile (mode:100644) renamed from Makefile (mode:100644 sha1:2d2913b6b98ac836b43755b1304d2a838dad87dd) +1 -1
@@ -36,7 +36,7 @@ LIB_OBJS += diff.o LIBS = $(LIB_FILE) -LIBS += -lz +LIBS += -lzd -lz ifdef MOZILLA_SHA1 SHA1_HEADER="mozilla-sha1/sha1.h"
cache.h (mode:100644) renamed from cache.h (mode:100644 sha1:3277d48708f885fa1b7cc56c9d16061c65a2eeb9) +18 -1
@@ -16,6 +16,7 @@ #include SHA1_HEADER #include <zlib.h> +#include <zdlib.h> /* * Basic data structures for the directory cache
@@ -64,6 +65,18 @@ char name[0]; }; +struct packed_item { + /* length of compressed data */ + unsigned long len; + struct packed_item *next; + /* sha1 of uncompressed data */ + char sha1[20]; + char refsha1[20]; + char type[20]; + /* compressed data */ + char *data; +}; + #define CE_NAMEMASK (0x0fff) #define CE_STAGEMASK (0x3000) #define CE_STAGESHIFT 12
@@ -119,7 +132,7 @@ /* Read and unpack a sha1 file into memory, write memory to a sha1 file */ extern void * map_sha1_file(const unsigned char *sha1, unsigned long *size); -extern void * unpack_sha1_file(void *map, unsigned long mapsize, char *type, unsigned long *size); +extern void * unpack_sha1_file(const unsigned char *sha1, void *map, unsigned long mapsize, char *type, unsigned long *size); extern void * read_sha1_file(const unsigned char *sha1, char *type, unsigned long *size); extern int write_sha1_file(char *buf, unsigned long len, const char *type, unsigned char *return_sha1);
@@ -135,6 +148,10 @@ /* Convert to/from hex/sha1 representation */ extern int get_sha1_hex(const char *hex, unsigned char *sha1); extern char *sha1_to_hex(const unsigned char *sha1); /* static buffer result! */ +extern int pack_sha1_buffer(void *buf, unsigned long buf_len, char *type, + unsigned char *returnsha1, unsigned char *refsha1, + struct packed_item **); +int write_packed_buffer(struct packed_item *head); /* General helper functions */ extern void usage(const char *err);
fsck-cache.c (mode:100644) renamed from fsck-cache.c (mode:100644 sha1:f9b1431dd8f4f3b426a7e410de952277aaa11401) +1 -1
@@ -142,7 +142,7 @@ if (map) { char type[100]; unsigned long size; - void *buffer = unpack_sha1_file(map, mapsize, type, &size); + void *buffer = unpack_sha1_file(sha1, map, mapsize, type, &size); if (!buffer) return -1; if (check_sha1_signature(sha1, buffer, size, type) < 0)
git-mktag.c (mode:100644) renamed from git-mktag.c (mode:100644 sha1:5d2830dc2bdfa2e76afc3fd4687db8faffaefba2) +1 -1
@@ -31,7 +31,7 @@ if (map) { char type[100]; unsigned long size; - void *buffer = unpack_sha1_file(map, mapsize, type, &size); + void *buffer = unpack_sha1_file(sha1,map,mapsize,type,&size); if (buffer) { if (!strcmp(type, expected_type))
sha1_file.c (mode:100644) renamed from sha1_file.c (mode:100644 sha1:db2880e389e556dd3a5eef02aa8a3bb235528057) +497 -12
545 lines not on this page · raw message
update-cache.c (mode:100644) renamed from update-cache.c (mode:100644 sha1:16e1bb9aea6413db35039042289605124d759501) +42 -43
131 lines not on this page · raw message
write-tree.c (mode:100644) renamed from write-tree.c (mode:100644 sha1:168352853d37bdca71d68ad8312b87b84477dea1) +14 -19
@@ -5,24 +5,13 @@ */ #include "cache.h" -static int check_valid_sha1(unsigned char *sha1) -{ - char *filename = sha1_file_name(sha1); - int ret; - - /* If we were anal, we'd check that the sha1 of the contents actually matches */ - ret = access(filename, R_OK); - if (ret) - perror(filename); - return ret; -} - -static int write_tree(struct cache_entry **cachep, int maxentries, const char *base, int baselen, unsigned char *returnsha1) +static int write_tree(struct cache_entry **cachep, int maxentries, const char *base, int baselen, unsigned char *returnsha1, struct packed_item **head) { unsigned char subdir_sha1[20]; unsigned long size, offset; char *buffer; int nr; + struct packed_item *item; /* Guess at some random initial size */ size = 8192;
@@ -50,7 +39,7 @@ if (dirname) { int subdir_written; - subdir_written = write_tree(cachep + nr, maxentries - nr, pathname, dirname-pathname+1, subdir_sha1); + subdir_written = write_tree(cachep + nr, maxentries - nr, pathname, dirname-pathname+1, subdir_sha1, head); nr += subdir_written; /* Now we need to write out the directory entry into this tree.. */
@@ -62,9 +51,6 @@ sha1 = subdir_sha1; } - if (check_valid_sha1(sha1) < 0) - exit(1); - entrylen = pathlen - baselen; if (offset + entrylen + 100 > size) { size = alloc_nr(offset + entrylen + 100);
@@ -77,7 +63,11 @@ nr++; } while (nr < maxentries); - write_sha1_file(buffer, offset, "tree", returnsha1); + pack_sha1_buffer(buffer, offset, "tree", returnsha1, NULL, &item); + if (item) { + item->next = *head; + *head = item; + } free(buffer); return nr; }
@@ -87,6 +77,7 @@ int i, unmerged; int entries = read_cache(); unsigned char sha1[20]; + struct packed_item *head = NULL; if (entries <= 0) die("write-tree: no cache contents to write");
@@ -107,8 +98,12 @@ die("write-tree: not able to write tree"); /* Ok, write it out */ - if (write_tree(active_cache, entries, "", 0, sha1) != entries) + if (write_tree(active_cache, entries, "", 0, sha1, &head) != entries) die("write-tree: internal error"); + if (head) { + if (write_packed_buffer(head)) + die("write_packed_buffer error"); + } printf("%s\n", sha1_to_hex(sha1)); return 0; }
On Mon, 2 May 2005, Chris Mason wrote: