Reftable support git-core
Subsystems (2)
PATCH v6 5/5 Reftable support for git-core
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
For background, see the previous commit introducing the library.
TODO:
* Resolve spots marked with XXX
* Test strategy?
Example use:
$ ~/vc/git/git init --reftable
warning: templates not found in /usr/local/google/home/hanwen/share/git-core/templates
Initialized empty Git repository in /tmp/qz/.git/
$ echo q > a
$ ~/vc/git/git add a
$ ~/vc/git/git commit -mx
fatal: not a git repository (or any of the parent directories): .git
[master (root-commit) 373d969] x
1 file changed, 1 insertion(+)
create mode 100644 a
$ ~/vc/git/git show-ref
373d96972fca9b63595740bba3898a762778ba20 HEAD
373d96972fca9b63595740bba3898a762778ba20 refs/heads/master
$ ls -l .git/reftable/
total 12
-rw------- 1 hanwen primarygroup 126 Jan 23 20:08 000000000001-000000000001.ref
-rw------- 1 hanwen primarygroup 4277 Jan 23 20:08 000000000002-000000000002.ref
$ go run ~/vc/reftable/cmd/dump.go -table /tmp/qz/.git/reftable/000000000002-000000000002.ref
** DEBUG **
name /tmp/qz/.git/reftable/000000000002-000000000002.ref, sz 4209: 'r' reftable.readerOffsets{Present:true, Offset:0x0, IndexOffset:0x0}, 'o' reftable.readerOffsets{Present:false, Offset:0x0, IndexOffset:0x0} 'g' reftable.readerOffsets{Present:true, Offset:0x1000, IndexOffset:0x0}
** REFS **
reftable.RefRecord{RefName:"refs/heads/master", UpdateIndex:0x2, Value:[]uint8{0x37, 0x3d, 0x96, 0x97, 0x2f, 0xca, 0x9b, 0x63, 0x59, 0x57, 0x40, 0xbb, 0xa3, 0x89, 0x8a, 0x76, 0x27, 0x78, 0xba, 0x20}, TargetValue:[]uint8(nil), Target:""}
** LOGS **
reftable.LogRecord{RefName:"HEAD", UpdateIndex:0x2, New:[]uint8{0x37, 0x3d, 0x96, 0x97, 0x2f, 0xca, 0x9b, 0x63, 0x59, 0x57, 0x40, 0xbb, 0xa3, 0x89, 0x8a, 0x76, 0x27, 0x78, 0xba, 0x20}, Old:[]uint8{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, Name:"Han-Wen Nienhuys", Email:"[email protected]", Time:0x5e29ef27, TZOffset:100, Message:"commit (initial): x\n"}Signed-off-by Han-Wen Nienhuys <[email protected]>
Co-authored-by Jeff King <[email protected]>
--- .../technical/repository-version.txt | 7 + Makefile | 24 +- builtin/clone.c | 4 +- builtin/init-db.c | 55 +- cache.h | 4 +- refs.c | 20 +- refs.h | 3 + refs/refs-internal.h | 1 + refs/reftable-backend.c | 971 ++++++++++++++++++ repository.c | 2 + repository.h | 3 + setup.c | 12 +- 12 files changed, 1077 insertions(+), 29 deletions(-) create mode 100644 refs/reftable-backend.c
Documentation/technical/repository-version.txt +7 -0
@@ -100,3 +100,10 @@ If set, by default "git config" reads from both "config" and multiple working directory mode, "config" file is shared while "config.worktree" is per-working directory (i.e., it's in GIT_COMMON_DIR/worktrees/<id>/config.worktree) + +==== `refStorage` + +Specifies the file format for the ref database. Values are `files` +(for the traditional packed + loose ref format) and `reftable` for the +binary reftable format. See https://github.com/google/reftable for +more information.
Makefile +23 -1
@@ -814,6 +814,7 @@ TEST_SHELL_PATH = $(SHELL_PATH) LIB_FILE = libgit.a XDIFF_LIB = xdiff/lib.a VCSSVN_LIB = vcs-svn/lib.a +REFTABLE_LIB = reftable/libreftable.a GENERATED_H += command-list.h
@@ -959,6 +960,7 @@ LIB_OBJS += rebase-interactive.o LIB_OBJS += reflog-walk.o LIB_OBJS += refs.o LIB_OBJS += refs/files-backend.o +LIB_OBJS += refs/reftable-backend.o LIB_OBJS += refs/iterator.o LIB_OBJS += refs/packed-backend.o LIB_OBJS += refs/ref-cache.o
@@ -1163,7 +1165,7 @@ THIRD_PARTY_SOURCES += compat/regex/% THIRD_PARTY_SOURCES += sha1collisiondetection/% THIRD_PARTY_SOURCES += sha1dc/% -GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) +GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) EXTLIBS = GIT_USER_AGENT = git/$(GIT_VERSION)
@@ -2347,11 +2349,28 @@ VCSSVN_OBJS += vcs-svn/fast_export.o VCSSVN_OBJS += vcs-svn/svndiff.o VCSSVN_OBJS += vcs-svn/svndump.o +REFTABLE_OBJS += reftable/basics.o +REFTABLE_OBJS += reftable/block.o +REFTABLE_OBJS += reftable/bytes.o +REFTABLE_OBJS += reftable/file.o +REFTABLE_OBJS += reftable/iter.o +REFTABLE_OBJS += reftable/merged.o +REFTABLE_OBJS += reftable/pq.o +REFTABLE_OBJS += reftable/reader.o +REFTABLE_OBJS += reftable/record.o +REFTABLE_OBJS += reftable/slice.o +REFTABLE_OBJS += reftable/stack.o +REFTABLE_OBJS += reftable/tree.o +REFTABLE_OBJS += reftable/writer.o +REFTABLE_OBJS += reftable/zlib-compat.o + + TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) OBJECTS := $(LIB_OBJS) $(BUILTIN_OBJS) $(PROGRAM_OBJS) $(TEST_OBJS) \ $(XDIFF_OBJS) \ $(VCSSVN_OBJS) \ $(FUZZ_OBJS) \ + $(REFTABLE_OBJS) \ common-main.o \ git.o ifndef NO_CURL
@@ -2488,6 +2507,9 @@ $(XDIFF_LIB): $(XDIFF_OBJS) $(VCSSVN_LIB): $(VCSSVN_OBJS) $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ +$(REFTABLE_LIB): $(REFTABLE_OBJS) + $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ + export DEFAULT_EDITOR DEFAULT_PAGER Documentation/GIT-EXCLUDED-PROGRAMS: FORCE
builtin/clone.c +3 -1
@@ -1097,7 +1097,9 @@ int cmd_clone(int argc, const char **argv, const char *prefix) } } -init_db (git_dir, real_git_dir, option_template,INIT_DB_QUIET ); +init_db (git_dir, real_git_dir, option_template, +DEFAULT_REF_STORAGE ,/* XXX */ +INIT_DB_QUIET );if (real_git_dir) git_dir = real_git_dir;
builtin/init-db.c +40 -15
125 lines not on this page · raw message
cache.h +3 -1
@@ -627,7 +627,8 @@ int path_inside_repo(const char *prefix, const char *path);#define INIT_DB_EXIST_OK 0x0002int init_db (const char *git_dir,const char *real_git_dir, -const char *template_dir,unsigned int flags); +const char *template_dir,const char *ref_storage_format, +unsigned int flags);void sanitize_stdfds (void );int daemonize (void );
@@ -1041,6 +1042,7 @@ struct repository_format {int is_bare;int hash_algo;char *work_tree; +char *ref_storage;struct string_list unknown_extensions; };
refs.c +12 -8
@@ -20,7 +20,7 @@/* * List of all available backends */ -static struct ref_storage_be *refs_backends = &refs_be_files; +static struct ref_storage_be *refs_backends = &refs_be_reftable;static struct ref_storage_be *find_ref_storage_backend (const char *name) {
@@ -1836,13 +1836,13 @@ static struct ref_store *lookup_ref_store_map(struct hashmap *map,* Create, record, and return a ref_store instance for the specified * gitdir. */ -static struct ref_store *ref_store_init (const char *gitdir, +static struct ref_store *ref_store_init (const char *gitdir,const char *be_name,unsigned int flags) { -const char *be_name ="files" ; -struct ref_storage_be *be =find_ref_storage_backend (be_name); +struct ref_storage_be *be;struct ref_store *refs; + be =find_ref_storage_backend (be_name);if (!be)BUG ("reference backend %s is unknown" , be_name);
@@ -1858,7 +1858,10 @@ struct ref_store *get_main_ref_store(struct repository *r)if (!r->gitdir)BUG ("attempting to get main_ref_store outside of repository" ); - r->refs =ref_store_init (r->gitdir,REF_STORE_ALL_CAPS ); + r->refs =ref_store_init (r->gitdir, + r->ref_storage_format ? r->ref_storage_format : +DEFAULT_REF_STORAGE , +REF_STORE_ALL_CAPS );return r->refs; }
@@ -1913,7 +1916,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule) goto done;/* assume that add_submodule_odb() has been called */ - refs =ref_store_init (submodule_sb.buf, + refs =ref_store_init (submodule_sb.buf,DEFAULT_REF_STORAGE ,/* XXX */ REF_STORE_READ |REF_STORE_ODB );register_ref_store_map (&submodule_ref_stores,"submodule" , refs, submodule);
@@ -1927,6 +1930,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule)struct ref_store *get_worktree_ref_store (const struct worktree *wt) { +const char *format =DEFAULT_REF_STORAGE ;/* XXX */ struct ref_store *refs;const char *id;
@@ -1940,9 +1944,9 @@ struct ref_store *get_worktree_ref_store(const struct worktree *wt)if (wt->id) refs =ref_store_init (git_common_path ("worktrees/%s" , wt->id), -REF_STORE_ALL_CAPS ); + format,REF_STORE_ALL_CAPS );else - refs =ref_store_init (get_git_common_dir (), + refs =ref_store_init (get_git_common_dir (), format,REF_STORE_ALL_CAPS );if (refs)
refs.h +3 -0
@@ -9,6 +9,9 @@ struct string_list;struct string_list_item ;struct worktree ; +/* XXX where should this be? */ +#define DEFAULT_REF_STORAGE "files" +/* * Resolve a reference, recursively following symbolic refererences. *
refs/refs-internal.h +1 -0
@@ -661,6 +661,7 @@ struct ref_storage_be { };extern struct ref_storage_be refs_be_files; +extern struct ref_storage_be refs_be_reftable;extern struct ref_storage_be refs_be_packed;/*
refs/reftable-backend.c new file +971 -0
971 lines not on this page · raw message
repository.c +2 -0
@@ -174,6 +174,8 @@ int repo_init(struct repository *repo,if (worktree)repo_set_worktree (repo, worktree); + repo->ref_storage_format =xstrdup_or_null (format.ref_storage); +clear_repository_format (&format);return 0 ;
repository.h +3 -0
@@ -70,6 +70,9 @@ struct repository {/* The store in which the refs are held. */ struct ref_store *refs; +/* The format to use for the ref database. */ +char *ref_storage_format; +/* * Contains path to often used file names. */
setup.c +9 -3
@@ -457,9 +457,11 @@ static int check_repo_format(const char *var, const char *value, void *vdata)if (!value)return config_error_nonbool (var); data->partial_clone =xstrdup (value); - }else if (!strcmp (ext,"worktreeconfig" )) + }else if (!strcmp (ext,"worktreeconfig" )) { data->worktree_config =git_config_bool (var, value); -else + }else if (!strcmp (ext,"refstorage" )) { + data->ref_storage =xstrdup (value); + }else string_list_append (&data->unknown_extensions, ext); }
@@ -548,6 +550,7 @@ void clear_repository_format(struct repository_format *format)string_list_clear (&format->unknown_extensions,0 );free (format->work_tree);free (format->partial_clone); +free (format->ref_storage);init_repository_format (format); }
@@ -1190,8 +1193,11 @@ const char *setup_git_directory_gently(int *nongit_ok) gitdir =DEFAULT_GIT_DIR_ENVIRONMENT ;setup_git_env (gitdir); } -if (startup_info->have_repository) +if (startup_info->have_repository) {repo_set_hash_algo (the_repository, repo_fmt.hash_algo); + the_repository->ref_storage_format = +xstrdup_or_null (repo_fmt.ref_storage); + } }strbuf_release (&dir);
Signature · 1 line
gitgitgadget
Here is what I got from "git am --whitespace=fix" on these five patches. Does it match your test application?
Applying: refs.h: clarify reflog iteration order
Applying: create .git/refs in files-backend.c
Applying: refs: document how ref_iterator_advance_fn should handle symrefs .git/rebase-apply/patch:145: trailing whitespace.
.git/rebase-apply/patch:147: trailing whitespace.
.git/rebase-apply/patch:6544: indent with spaces.
left = *destLen;
.git/rebase-apply/patch:6545: indent with spaces.
*destLen = 0;
.git/rebase-apply/patch:6548: indent with spaces.
left = 1;warning: squelched 15 whitespace errors
warning: 20 lines applied after fixing whitespace errors.
Applying: Add reftable library
Applying: Reftable support for git-core
Quoted 11 lines · Junio C Hamano
Here is what I got from "git am --whitespace=fix" on these five patches. Does it match your test application?
Applying:
refs.h: clarify reflog iteration order
Applying: create .git/refs infiles-backend.c
Applying: refs: document howref_iterator_advance_fnshould handle symrefs .git/rebase-apply/patch:145: trailing whitespace..git/rebase-apply/patch:147: trailing whitespace.
Quoted 7 lines
.git/rebase-apply/patch:6544: indent with spaces.
left = *destLen; .git/rebase-apply/patch:6545: indent with spaces. *destLen = 0; .git/rebase-apply/patch:6548: indent with spaces. left = 1;warning: squelched 15 whitespace errors
your checker is tripping over code imported from zlib. I added a /* clang-format off */ comment to avoid reformatting this code. What do you suggest?
Signature · 1 line
Han-Wen Nienhuys - [email protected] - http://www.xs4all.nl/~hanwen
Quoted 21 lines · Han-Wen Nienhuys
Quoted 11 lines · Junio C Hamano
Here is what I got from "git am --whitespace=fix" on these five patches. Does it match your test application?
Applying:
refs.h: clarify reflog iteration order
Applying: create .git/refs infiles-backend.c
Applying: refs: document howref_iterator_advance_fnshould handle symrefs .git/rebase-apply/patch:145: trailing whitespace..git/rebase-apply/patch:147: trailing whitespace.
Quoted 7 lines
.git/rebase-apply/patch:6544: indent with spaces.
left = *destLen; .git/rebase-apply/patch:6545: indent with spaces. *destLen = 0; .git/rebase-apply/patch:6548: indent with spaces. left = 1;warning: squelched 15 whitespace errors
Quoted 3 lines
your checker is tripping over code imported from zlib. I added a /* clang-format off */ comment to avoid reformatting this code. What do you suggest?
Use zlib from the system instead?
Quoted 5 lines · Junio C Hamano
Quoted 3 lines
your checker is tripping over code imported from zlib. I added a /* clang-format off */ comment to avoid reformatting this code. What do you suggest?
Use zlib from the system instead?
uncompress2 is a 2016 addition to zlib. It doesn't pass on gitgitgadget's CI, because it is using an older version of zlib.
Signature · 1 line
Han-Wen Nienhuys - [email protected] - http://www.xs4all.nl/~hanwen
Quoted 10 lines · Han-Wen Nienhuys
Quoted 5 lines · Junio C Hamano
Quoted 3 lines
your checker is tripping over code imported from zlib. I added a /* clang-format off */ comment to avoid reformatting this code. What do you suggest?
Use zlib from the system instead?
uncompress2 is a 2016 addition to zlib. It doesn't pass on gitgitgadget's CI, because it is using an older version of zlib.
Ahh.
It is OK (and indeed you're right that you cannot avoid it) to ship a reasonably new snapshot as a fallback in such a case, but it still is far more preferrable to avoid linking with the fallback snapshot copy when a working one is available on the system, especially for a widely used and established library like zlib, because we have one less thing to keep up-to-date with the security patches made to the upstream.
In any case, if we cannot avoid shipping a borrowed code that we'd rather not touch or fix-up, covering them with local .gitattributes file, disabling selected whitespace checks, may be one way out. IIUC, our default settings for the whitespace check is in the top-level .gitattributes file
* whitespace=!indent,trail,space
*.[ch] whitespace=indent,trail,space diff=cpp
*.sh whitespace=indent,trail,space eol=lf
A new local .gitattrubutes file in retable/ directory to loosen the set of checks. The borrowed code triggers indent-with-non-tab and trailing-space at least, which needs to be disabled locally, I think.
Thanks.
reftable/.gitattributes | 1 + 1 file changed, 1 insertion(+)
reftable/.gitattributes new file +1 -0
@@ -0,0 +1 @@ +/zlib-compat.c whitespace=-indent-with-non-tab,-trailing-space
Quoted 7 lines · Junio C Hamano
It is OK (and indeed you're right that you cannot avoid it) to ship a reasonably new snapshot as a fallback in such a case, but it still is far more preferrable to avoid linking with the fallback snapshot copy when a working one is available on the system, especially for a widely used and established library like zlib, because we have one less thing to keep up-to-date with the security patches made to the upstream.
what is the procedure for doing this? Do I need to write autoconf macros (oh no!) ?
I've added a comment for now in system.h
Quoted 2 lines
reftable/.gitattributes | 1 + 1 file changed, 1 insertion(+)
thanks, I've put this in.
Signature · 1 line
Han-Wen Nienhuys - [email protected] - http://www.xs4all.nl/~hanwen
Quoted 11 lines · Han-Wen Nienhuys
Quoted 7 lines · Junio C Hamano
It is OK (and indeed you're right that you cannot avoid it) to ship a reasonably new snapshot as a fallback in such a case, but it still is far more preferrable to avoid linking with the fallback snapshot copy when a working one is available on the system, especially for a widely used and established library like zlib, because we have one less thing to keep up-to-date with the security patches made to the upstream.
what is the procedure for doing this? Do I need to write autoconf macros (oh no!) ?
We treat the Makefile macros listed at the top of the file as authoritative set of knobs, so
# Define ZLIB_LACKS_UNCOMPRESS2 if your zlib is so old to lack the # function uncompress2().
with conditional compilation/linkage based on the macro would be minimally sufficient. Setting it automatically in ./configure is nice-to-have but is optional.
Quoted 15 lines · Junio C Hamano
Quoted 10 lines · Han-Wen Nienhuys
Quoted 5 lines · Junio C Hamano
Quoted 3 lines
your checker is tripping over code imported from zlib. I added a /* clang-format off */ comment to avoid reformatting this code. What do you suggest?
Use zlib from the system instead?
uncompress2 is a 2016 addition to zlib. It doesn't pass on gitgitgadget's CI, because it is using an older version of zlib.
Ahh.
Quoted 7 lines
It is OK (and indeed you're right that you cannot avoid it) to ship a reasonably new snapshot as a fallback in such a case, but it still is far more preferrable to avoid linking with the fallback snapshot copy when a working one is available on the system, especially for a widely used and established library like zlib, because we have one less thing to keep up-to-date with the security patches made to the upstream.
If this were substantial code, I'd agree. But this is really just a thin wrapper around the usual loop zlib inflate() loop that we already do in a dozen places in our code (e.g., unpack_loose_rest(), which even does the "did we consume all the bytes" check that uncompress2 allows).
I'm not sure it is worth the mental energy of adding a Makefile knob to use the zlib version if we can just open-code it ourselves (or provide our own custom helper that we always use).
And if it _is_ worth taking the zlib version because we're concerned that it may contain or gain bugfixes[1], then possibly we should be using it in lots more places (though probably not everywhere, as we do sometimes need the streaming behavior of the loop).
[1] I'll admit we've hit some subtleties with that loop in the past, especially around corrupted or bogus inputs, but I think we've shaken them all out these days.
-Peff
Hi,
Quoted 1 line · Han-Wen Nienhuys
This adds the reftable library, and hooks it up as a ref backend.
As promised, here's a patch to include the reftable spec in git.git. Please include this in the next iteration of this patch series (or if you prefer for it to land separately, that's also fine with me).
[...]
Quoted 1 line
* support SHA256 as version 2 of the format.
I'd prefer if we error out for now when someone tries to use reftable when the_hash_algo != sha1. That would buy a bit more time to pin down the details of version 2 (e.g., should it only add an object format field, or object format and oid size?).
The current reftable spec says
Future
Longer hashes
Version will bump (e.g. 2) to indicate value uses a different object id length other than 20. The length could be stored in an expanded file header, or hardcoded as part of the version.
or in other words doesn't define that aspect of the format yet.
This is also a reason to get the reftable spec in git.git, since it would make it easier to have the discussion in one place. :) Thanks for your explanations so far at https://git.eclipse.org/r/c/157501/.
Sincerely,
Jonathan
-- >8 --
Subject: reftable: file format documentation
Shawn Pearce explains:
Some repositories contain a lot of references (e.g. android at 866k, rails at 31k). The reftable format provides:
- Near constant time lookup for any single reference, even when the repository is cold and not in process or kernel cache.
- Near constant time verification a SHA-1 is referred to by at least one reference (for allow-tip-sha1-in-want).
- Efficient lookup of an entire namespace, such as refs/tags/.
- Support atomic push O(size_of_update) operations.
- Combine reflog storage with ref storage.
This file format spec was originally written in July, 2017 by Shawn Pearce. Some refinements since then were made by Shawn and by Han-Wen Nienhuys based on experiences implementing and experimenting with the format. (All of this was in the context of our work at Google and Google is happy to contribute the result to the Git project.)
Imported from JGit[1]'s current version (c217d33ff, "Documentation/technical/reftable: improve repo layout", 2020-02-04) of Documentation/technical/reftable.md and converted to asciidoc by running
pandoc -t asciidoc -f markdown reftable.md >reftable.txt
using pandoc 2.2.1. The result required the following additional minor changes:
- removed the [TOC] directive to add a table of contents, since asciidoc does not support it
- replaced git-scm.com/docs links with linkgit: directives that link to other pages within Git's documentation
[1] https://eclipse.googlesource.com/jgit/jgit
Signed-off-by Jonathan Nieder <[email protected]>
--- Documentation/Makefile | 1 + Documentation/technical/reftable.txt | 1067 ++++++++++++++++++++++++++ 2 files changed, 1068 insertions(+) create mode 100644 Documentation/technical/reftable.txt
Documentation/Makefile +1 -0
@@ -92,6 +92,7 @@ TECH_DOCS += technical/protocol-capabilities TECH_DOCS += technical/protocol-common TECH_DOCS += technical/protocol-v2 TECH_DOCS += technical/racy-git +TECH_DOCS += technical/reftable TECH_DOCS += technical/send-pack-pipeline TECH_DOCS += technical/shallow TECH_DOCS += technical/signature-format
Documentation/technical/reftable.txt new file +1067 -0
1,067 lines not on this page · raw message
Signature · 1 line
2.25.1.481.gfbce0eb801
Quoted 5 lines · Jonathan Nieder
Quoted 1 line
This adds the reftable library, and hooks it up as a ref backend.
As promised, here's a patch to include the reftable spec in git.git. Please include this in the next iteration of this patch series (or if you prefer for it to land separately, that's also fine with me).
I did.
Quoted 7 lines
[...]
Quoted 1 line
* support SHA256 as version 2 of the format.
I'd prefer if we error out for now when someone tries to use reftable when
the_hash_algo!= sha1. That would buy a bit more time to pin down the details of version 2 (e.g., should it only add an object format field, or object format and oid size?).
You have convinced me that we should go with the 4-byte identifier.
How about setting version=2 and extending the header by 4 bytes (which hold the 4-byte identifier). The footer would also be increased in size equivalently. There are no further changes.
BTW, could we document that a reftable consisting of just a footer is a valid table too?
Signature · 7 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Quoted 8 lines · Han-Wen Nienhuys
Quoted 5 lines · Jonathan Nieder
Quoted 1 line
This adds the reftable library, and hooks it up as a ref backend.
As promised, here's a patch to include the reftable spec in git.git. Please include this in the next iteration of this patch series (or if you prefer for it to land separately, that's also fine with me).
I did.
Ahh, I didn't see that original request to make it part of the series. Let me drop the standalone patch and revert it out of 'next', so that we can polish it as a part of this topic, which makes a lot more sense.
Thanks.
Quoted 16 lines · Han-Wen Nienhuys
Quoted 5 lines · Jonathan Nieder
Quoted 1 line
This adds the reftable library, and hooks it up as a ref backend.
As promised, here's a patch to include the reftable spec in git.git. Please include this in the next iteration of this patch series (or if you prefer for it to land separately, that's also fine with me).
I did.
Quoted 7 lines
[...]
Quoted 1 line
* support SHA256 as version 2 of the format.
I'd prefer if we error out for now when someone tries to use reftable when
the_hash_algo!= sha1. That would buy a bit more time to pin down the details of version 2 (e.g., should it only add an object format field, or object format and oid size?).
If we do this, the testsuite will start to fail with SHA-256 unless you label those tests with the SHA1 prerequisite.
Quoted 5 lines
You have convinced me that we should go with the 4-byte identifier.
How about setting version=2 and extending the header by 4 bytes (which hold the 4-byte identifier). The footer would also be increased in size equivalently. There are no further changes.
Just so you're aware, the values we're going to be using in pack index version 3 are in sha1-file.c as the format_version values of struct hash_algo. They're 0x73686131 for SHA-1 ("sha1", big-endian) and 0x73323536 for SHA-256 ("s256", big-endian).
I know Jonathan is already familiar with those, but I thought I'd point them out since others may not be.
Signature · 2 lines
brian m. carlson: Houston, Texas, US OpenPGP: https://keybase.io/bk2204
A message that is not in the archive (sent off-list or not yet received): CAFQ2z_NQn9O3kFmHk8Cr31FY66ToU4bUdE=asHUfN++zBG+SPw@mail.gmail.com
(+Josh Steadmon, fuzz testing specialist)
Quoted 4 lines · Han-Wen Nienhuys
You have convinced me that we should go with the 4-byte identifier.
How about setting version=2 and extending the header by 4 bytes (which hold the 4-byte identifier). The footer would also be increased in size.
SGTM.
Quoted 2 lines
BTW, could we document that a reftable consisting of just a footer is a valid table too?
Yes, SGTM as well.
Do you have sample reftables collected somewhere that you use for testing? Asking since that would help us set up some fuzz tests. :)
Thanks,
Jonathan
Quoted 11 lines · Jonathan Nieder
(+Josh Steadmon, fuzz testing specialist)
Quoted 4 lines · Han-Wen Nienhuys
You have convinced me that we should go with the 4-byte identifier.
How about setting version=2 and extending the header by 4 bytes (which hold the 4-byte identifier). The footer would also be increased in size.
SGTM.
Quoted 7 lines
Quoted 2 lines
BTW, could we document that a reftable consisting of just a footer is a valid table too?
Yes, SGTM as well.
Do you have sample reftables collected somewhere that you use for testing? Asking since that would help us set up some fuzz tests. :)
No, but the library is standalone, so writing fuzz tests should be easy. The C code is essentially a copy of the Go code, so you could start fuzzing there. I believe it is really easy to setup fuzzing in Go, and we could generate a corpus starting there.
On a related note, I should write interoperation tests that pass tables back and forth between, JGit, C and Go, but I'm currently focusing on getting the code ready for git-core.
Signature · 11 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
v7 cover Reftable support git-core
In reply to Han-Wen Nienhuys via GitGitGadgetThis adds the reftable library, and hooks it up as a ref backend.
Feedback wanted:
* spots marked with XXX in the Git source code.
* what is a good test strategy? Could we have a CI flavor where we flip the default to reftable, and see how it fares?
v7
* support SHA256 as version 2 of the format.
v8
* propagate errors to git.
* discard empty tables in the stack.
* one very basic test (t0031.sh)
v9
* Added spec as technical doc.
* Use 4-byte hash ID as API. Storage format for SHA256 is still pending discussion.
Han-Wen Nienhuys (5):
refs.h: clarify reflog iteration order
create .git/refs in files-backend.c
refs: document how ref_iterator_advance_fn should handle symrefs Add reftable library
Reftable support for git-core
Jonathan Nieder (1):
reftable: file format documentation
Documentation/Makefile | 1 + Documentation/technical/reftable.txt | 1067 ++++++++++++++++ .../technical/repository-version.txt | 7 + Makefile | 24 +- builtin/clone.c | 4 +- builtin/init-db.c | 57 +- cache.h | 4 +- refs.c | 20 +- refs.h | 8 +- refs/files-backend.c | 6 + refs/refs-internal.h | 6 + refs/reftable-backend.c | 1015 +++++++++++++++ reftable/LICENSE | 31 + reftable/README.md | 11 + reftable/VERSION | 5 + reftable/basics.c | 160 +++ reftable/basics.h | 30 + reftable/block.c | 413 ++++++ reftable/block.h | 71 ++ reftable/blocksource.h | 20 + reftable/bytes.c | 0 reftable/config.h | 1 + reftable/constants.h | 25 + reftable/dump.c | 97 ++ reftable/file.c | 97 ++ reftable/iter.c | 229 ++++ reftable/iter.h | 56 + reftable/merged.c | 290 +++++ reftable/merged.h | 34 + reftable/pq.c | 114 ++ reftable/pq.h | 34 + reftable/reader.c | 720 +++++++++++ reftable/reader.h | 52 + reftable/record.c | 1119 +++++++++++++++++ reftable/record.h | 79 ++ reftable/reftable.h | 409 ++++++ reftable/slice.c | 199 +++ reftable/slice.h | 39 + reftable/stack.c | 1007 +++++++++++++++ reftable/stack.h | 40 + reftable/system.h | 53 + reftable/tree.c | 66 + reftable/tree.h | 24 + reftable/update.sh | 13 + reftable/writer.c | 637 ++++++++++ reftable/writer.h | 45 + reftable/zlib-compat.c | 92 ++ repository.c | 2 + repository.h | 3 + setup.c | 12 +- t/t0031-reftable.sh | 31 + 51 files changed, 8547 insertions(+), 32 deletions(-) create mode 100644 Documentation/technical/reftable.txt create mode 100644 refs/reftable-backend.c create mode 100644 reftable/LICENSE create mode 100644 reftable/README.md create mode 100644 reftable/VERSION create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/blocksource.h create mode 100644 reftable/bytes.c create mode 100644 reftable/config.h create mode 100644 reftable/constants.h create mode 100644 reftable/dump.c create mode 100644 reftable/file.c create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/reftable.h create mode 100644 reftable/slice.c create mode 100644 reftable/slice.h create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/system.h create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100644 reftable/update.sh create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h create mode 100644 reftable/zlib-compat.c create mode 100755 t/t0031-reftable.sh
base-commit: 51ebf55b9309824346a6589c9f3b130c6f371b8f
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-539%2Fhanwen%2Freftable-v7
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-539/hanwen/reftable-v7
Pull-Request: https://github.com/gitgitgadget/git/pull/539
Range-diff vs v6:
1: e4b0773becd = 1: b1e44bc431e refs.h: clarify reflog iteration order 2: c25b6c601dc = 2: b68488a095e create .git/refs in files-backend.c 3: e132e0f4e00 ! 3: da538ef7421 refs: document how ref_iterator_advance_fn should handle symrefs
@@ -13,8 +13,8 @@
+/*
+ * backend-specific implementation of ref_iterator_advance.
-+ * For symrefs, the function should set REF_ISSYMREF, and it should also dereference -+ * the symref to provide the OID referent.
++ * For symrefs, the function should set REF_ISSYMREF, and it should also ++ * dereference the symref to provide the OID referent.
+ */
typedef int ref_iterator_advance_fn(struct ref_iterator *ref_iterator);
-: ----------- > 4: aae26814983 reftable: file format documentation
4: fe29a9db399 ! 5: 30ed43a4fdb Add reftable library
@@ -2,19 +2,11 @@
Add reftable library
- Reftable is a new format for storing the ref database. It provides the
- following benefits:
-
- * Simple and fast atomic ref transactions, including multiple refs and reflogs.
- * Compact storage of ref data.
- * Fast look ups of ref data.
- * Case-sensitive ref names on Windows/OSX, regardless of file system
- * Eliminates file/directory conflicts in ref names
+ Reftable is a format for storing the ref database. Its rationale and
+ specification is in the preceding commit.
Further context and motivation can be found in background reading:
- * Spec: https://github.com/eclipse/jgit/blob/master/Documentation/technical/reftable.md
-
* Original discussion on JGit-dev: https://www.eclipse.org/lists/jgit-dev/msg03389.html
* First design discussion on git@vger: https://public-inbox.org/git/CAJo=hJtTp2eA3z9wW9cHo-nA7kK40vVThqh6inXpbCcqfdMP9g@mail.gmail.com/
@@ -80,15 +72,7 @@
+
+To update the library, do:
+
-+ ((cd reftable-repo && git fetch origin && git checkout origin/master ) ||
-+ git clone https://github.com/google/reftable reftable-repo) && \
-+ cp reftable-repo/c/*.[ch] reftable/ && \
-+ cp reftable-repo/LICENSE reftable/ &&
-+ git --git-dir reftable-repo/.git show --no-patch origin/master \
-+ > reftable/VERSION && \
-+ sed -i~ 's|if REFTABLE_IN_GITCORE|if 1 /* REFTABLE_IN_GITCORE */|' reftable/system.h
-+ rm reftable/*_test.c reftable/test_framework.*
-+ git add reftable/*.[ch]
++ sh reftable/update.sh
+
+Bugfixes should be accompanied by a test and applied to upstream project at
+https://github.com/google/reftable.
@@ -98,17 +82,11 @@
--- /dev/null
+++ b/reftable/VERSION
@@
-+commit 42aa04f301a682197f0698867b9771b78ce4fdaf
++commit ccce3b3eb763e79b23a3b4677d65ecceb4155ba3
+Author: Han-Wen Nienhuys <[email protected]>
-+Date: Mon Feb 17 16:05:22 2020 +0100
-+
-+ C: simplify system and config header handling
-+
-+ reftable.h only depends on <stdint.h>
-+
-+ system.h is only included from .c files. The git-core specific code is
-+ guarded by a #if REFTABLE_IN_GITCORE, which we'll edit with a sed
-+ script on the git-core side.
++Date: Tue Feb 25 20:42:29 2020 +0100
++
++ C: rephrase the hash ID API in terms of a uint32_t
diff --git a/reftable/basics.c b/reftable/basics.c
new file mode 100644
@@ -127,61 +105,23 @@
+
+#include "system.h"
+
-+void put_u24(byte *out, uint32_t i)
++void put_be24(byte *out, uint32_t i)
+{
+ out[0] = (byte)((i >> 16) & 0xff);
+ out[1] = (byte)((i >> 8) & 0xff);
+ out[2] = (byte)((i)&0xff);
+}
+
-+uint32_t get_u24(byte *in)
++uint32_t get_be24(byte *in)
+{
+ return (uint32_t)(in[0]) << 16 | (uint32_t)(in[1]) << 8 |
+ (uint32_t)(in[2]);
+}
+
-+void put_u32(byte *out, uint32_t i)
++void put_be16(uint8_t *out, uint16_t i)
+{
-+ out[0] = (byte)((i >> 24) & 0xff);
-+ out[1] = (byte)((i >> 16) & 0xff);
-+ out[2] = (byte)((i >> 8) & 0xff);
-+ out[3] = (byte)((i)&0xff);
-+}
-+
-+uint32_t get_u32(byte *in)
-+{
-+ return (uint32_t)(in[0]) << 24 | (uint32_t)(in[1]) << 16 |
-+ (uint32_t)(in[2]) << 8 | (uint32_t)(in[3]);
-+}
-+
-+void put_u64(byte *out, uint64_t v)
-+{
-+ int i = 0;
-+ for (i = sizeof(uint64_t); i--;) {
-+ out[i] = (byte)(v & 0xff);
-+ v >>= 8;
-+ }
-+}
-+
-+uint64_t get_u64(byte *out)
-+{
-+ uint64_t v = 0;
-+ int i = 0;
-+ for (i = 0; i < sizeof(uint64_t); i++) {
-+ v = (v << 8) | (byte)(out[i] & 0xff);
-+ }
-+ return v;
-+}
-+
-+void put_u16(byte *out, uint16_t i)
-+{
-+ out[0] = (byte)((i >> 8) & 0xff);
-+ out[1] = (byte)((i)&0xff);
-+}
-+
-+uint16_t get_u16(byte *in)
-+{
-+ return (uint32_t)(in[0]) << 8 | (uint32_t)(in[1]);
++ out[0] = (uint8_t)((i >> 8) & 0xff);
++ out[1] = (uint8_t)((i)&0xff);
+}
+
+/*
@@ -234,7 +174,9 @@
+int names_length(char **names)
+{
+ int len = 0;
-+ for (char **p = names; *p; p++) {
++ char **p = names;
++ while (*p) {
++ p++;
+ len++;
+ }
+ return len;
@@ -262,7 +204,7 @@
+ names = realloc(names,
+ names_cap * sizeof(char *));
+ }
-+ names[names_len++] = strdup(p);
++ names[names_len++] = xstrdup(p);
+ }
+ p = next + 1;
+ }
@@ -335,17 +277,10 @@
+#define true 1
+#define false 0
+
-+void put_u24(byte *out, uint32_t i);
-+uint32_t get_u24(byte *in);
-+
-+uint64_t get_u64(byte *in);
-+void put_u64(byte *out, uint64_t i);
++void put_be24(byte *out, uint32_t i);
++uint32_t get_be24(byte *in);
++void put_be16(uint8_t *out, uint16_t i);
+
-+void put_u32(byte *out, uint32_t i);
-+uint32_t get_u32(byte *in);
-+
-+void put_u16(byte *out, uint16_t i);
-+uint16_t get_u16(byte *in);
+int binsearch(int sz, int (*f)(int k, void *args), void *args);
+
+void free_names(char **a);
@@ -403,7 +338,7 @@
+ success */
+int block_writer_add(struct block_writer *w, struct record rec)
+{
-+ struct slice empty = {};
++ struct slice empty = { 0 };
+ struct slice last = w->entries % w->restart_interval == 0 ? empty :
+ w->last_key;
+ struct slice out = {
@@ -414,7 +349,7 @@
+ struct slice start = out;
+
+ bool restart = false;
-+ struct slice key = {};
++ struct slice key = { 0 };
+ int n = 0;
+
+ record_key(rec, &key);
@@ -480,17 +415,17 @@
+{
+ int i = 0;
+ for (i = 0; i < w->restart_len; i++) {
-+ put_u24(w->buf + w->next, w->restarts[i]);
++ put_be24(w->buf + w->next, w->restarts[i]);
+ w->next += 3;
+ }
+
-+ put_u16(w->buf + w->next, w->restart_len);
++ put_be16(w->buf + w->next, w->restart_len);
+ w->next += 2;
-+ put_u24(w->buf + 1 + w->header_off, w->next);
++ put_be24(w->buf + 1 + w->header_off, w->next);
+
+ if (block_writer_type(w) == BLOCK_TYPE_LOG) {
+ int block_header_skip = 4 + w->header_off;
-+ struct slice compressed = {};
++ struct slice compressed = { 0 };
+ int zresult = 0;
+ uLongf src_len = w->next - block_header_skip;
+ slice_resize(&compressed, src_len);
@@ -531,14 +466,14 @@
+{
+ uint32_t full_block_size = table_block_size;
+ byte typ = block->data[header_off];
-+ uint32_t sz = get_u24(block->data + header_off + 1);
++ uint32_t sz = get_be24(block->data + header_off + 1);
+
+ if (!is_block_type(typ)) {
+ return FORMAT_ERROR;
+ }
+
+ if (typ == BLOCK_TYPE_LOG) {
-+ struct slice uncompressed = {};
++ struct slice uncompressed = { 0 };
+ int block_header_skip = 4 + header_off;
+ uLongf dst_len = sz - block_header_skip;
+ uLongf src_len = block->len - block_header_skip;
@@ -570,7 +505,7 @@
+ }
+
+ {
-+ uint16_t restart_count = get_u16(block->data + sz - 2);
++ uint16_t restart_count = get_be16(block->data + sz - 2);
+ uint32_t restart_start = sz - 2 - 3 * restart_count;
+
+ byte *restart_bytes = block->data + restart_start;
@@ -593,7 +528,7 @@
+
+static uint32_t block_reader_restart_offset(struct block_reader *br, int i)
+{
-+ return get_u24(br->restart_bytes + 3 * i);
++ return get_be24(br->restart_bytes + 3 * i);
+}
+
+void block_reader_start(struct block_reader *br, struct block_iter *it)
@@ -604,9 +539,9 @@
+}
+
+struct restart_find_args {
++ int error;
+ struct slice key;
+ struct block_reader *r;
-+ int error;
+};
+
+static int restart_key_less(int idx, void *args)
@@ -620,8 +555,8 @@
+
+ /* the restart key is verbatim in the block, so this could avoid the
+ alloc for decoding the key */
-+ struct slice rkey = {};
-+ struct slice last_key = {};
++ struct slice rkey = { 0 };
++ struct slice last_key = { 0 };
+ byte unused_extra;
+ int n = decode_key(&rkey, &unused_extra, last_key, in);
+ if (n < 0) {
@@ -656,7 +591,7 @@
+ .len = it->br->block_len - it->next_off,
+ };
+ struct slice start = in;
-+ struct slice key = {};
++ struct slice key = { 0 };
+ byte extra;
+ int n = decode_key(&key, &extra, it->last_key, in);
+ if (n < 0) {
@@ -681,7 +616,7 @@
+
+int block_reader_first_key(struct block_reader *br, struct slice *key)
+{
-+ struct slice empty = {};
++ struct slice empty = { 0 };
+ int off = br->header_off + 4;
+ struct slice in = {
+ .buf = br->block.data + off,
@@ -729,10 +664,10 @@
+
+ {
+ struct record rec = new_record(block_reader_type(br));
-+ struct slice key = {};
++ struct slice key = { 0 };
+ int result = 0;
+ int err = 0;
-+ struct block_iter next = {};
++ struct block_iter next = { 0 };
+ while (true) {
+ block_iter_copy_from(&next, it);
+
@@ -830,9 +765,9 @@
+};
+
+struct block_iter {
++ uint32_t next_off;
+ struct block_reader *br;
+ struct slice last_key;
-+ uint32_t next_off;
+};
+
+int block_reader_init(struct block_reader *br, struct block *bl,
@@ -937,7 +872,7 @@
+
+static int dump_table(const char *tablename)
+{
-+ struct block_source src = {};
++ struct block_source src = { 0 };
+ int err = block_source_from_file(&src, tablename);
+ if (err < 0) {
+ return err;
@@ -950,13 +885,13 @@
+ }
+
+ {
-+ struct iterator it = {};
++ struct iterator it = { 0 };
+ err = reader_seek_ref(r, &it, "");
+ if (err < 0) {
+ return err;
+ }
+
-+ struct ref_record ref = {};
++ struct ref_record ref = { 0 };
+ while (1) {
+ err = iterator_next_ref(it, &ref);
+ if (err > 0) {
@@ -972,12 +907,12 @@
+ }
+
+ {
-+ struct iterator it = {};
++ struct iterator it = { 0 };
+ err = reader_seek_log(r, &it, "");
+ if (err < 0) {
+ return err;
+ }
-+ struct log_record log = {};
++ struct log_record log = { 0 };
+ while (1) {
+ err = iterator_next_log(it, &log);
+ if (err > 0) {
@@ -1001,7 +936,7 @@
+ while ((opt = getopt(argc, argv, "t:")) != -1) {
+ switch (opt) {
+ case 't':
-+ table = strdup(optarg);
++ table = xstrdup(optarg);
+ break;
+ case '?':
+ printf("usage: %s [-table tablefile]\n", argv[0]);
@@ -1091,7 +1026,7 @@
+
+int block_source_from_file(struct block_source *bs, const char *name)
+{
-+ struct stat st = {};
++ struct stat st = { 0 };
+ int err = 0;
+ int fd = open(name, O_RDONLY);
+ if (fd < 0) {
@@ -1188,14 +1123,14 @@
+
+int iterator_next_ref(struct iterator it, struct ref_record *ref)
+{
-+ struct record rec = {};
++ struct record rec = { 0 };
+ record_from_ref(&rec, ref);
+ return iterator_next(it, rec);
+}
+
+int iterator_next_log(struct iterator it, struct log_record *log)
+{
-+ struct record rec = {};
++ struct record rec = { 0 };
+ record_from_log(&rec, log);
+ return iterator_next(it, rec);
+}
@@ -1221,7 +1156,7 @@
+ }
+
+ if (fri->double_check) {
-+ struct iterator it = {};
++ struct iterator it = { 0 };
+
+ int err = reader_seek_ref(fri->r, &it, ref->ref_name);
+ if (err == 0) {
@@ -1389,9 +1324,9 @@
+bool iterator_is_null(struct iterator it);
+
+struct filtering_ref_iterator {
++ bool double_check;
+ struct reader *r;
+ struct slice oid;
-+ bool double_check;
+ struct iterator it;
+};
+
@@ -1511,7 +1446,7 @@
+
+static int merged_iter_next(struct merged_iter *mi, struct record rec)
+{
-+ struct slice entry_key = {};
++ struct slice entry_key = { 0 };
+ struct pq_entry entry = merged_iter_pqueue_remove(&mi->pq);
+ int err = merged_iter_advance_subiter(mi, entry.index);
+ if (err < 0) {
@@ -1521,7 +1456,7 @@
+ record_key(entry.rec, &entry_key);
+ while (!merged_iter_pqueue_is_empty(mi->pq)) {
+ struct pq_entry top = merged_iter_pqueue_top(mi->pq);
-+ struct slice k = {};
++ struct slice k = { 0 };
+ int err = 0, cmp = 0;
+
+ record_key(top.rec, &k);
@@ -1542,7 +1477,7 @@
+ free(record_yield(&top.rec));
+ }
+
-+ record_copy_from(rec, entry.rec, mi->hash_size);
++ record_copy_from(rec, entry.rec, hash_size(mi->hash_id));
+ record_clear(entry.rec);
+ free(record_yield(&entry.rec));
+ free(slice_yield(&entry_key));
@@ -1572,14 +1507,14 @@
+}
+
+int new_merged_table(struct merged_table **dest, struct reader **stack, int n,
-+ int hash_size)
++ uint32_t hash_id)
+{
+ uint64_t last_max = 0;
+ uint64_t first_min = 0;
+ int i = 0;
+ for (i = 0; i < n; i++) {
+ struct reader *r = stack[i];
-+ if (reader_hash_size(r) != hash_size) {
++ if (r->hash_id != hash_id) {
+ return FORMAT_ERROR;
+ }
+ if (i > 0 && last_max >= reader_min_update_index(r)) {
@@ -1598,7 +1533,7 @@
+ .stack_len = n,
+ .min = first_min,
+ .max = last_max,
-+ .hash_size = hash_size,
++ .hash_id = hash_id,
+ };
+
+ *dest = calloc(sizeof(struct merged_table), 1);
@@ -1650,7 +1585,7 @@
+ struct merged_iter merged = {
+ .stack = iters,
+ .typ = record_type(rec),
-+ .hash_size = mt->hash_size,
++ .hash_id = mt->hash_id,
+ };
+ int n = 0;
+ int err = 0;
@@ -1693,7 +1628,7 @@
+ struct ref_record ref = {
+ .ref_name = (char *)name,
+ };
-+ struct record rec = {};
++ struct record rec = { 0 };
+ record_from_ref(&rec, &ref);
+ return merged_table_seek_record(mt, it, rec);
+}
@@ -1705,7 +1640,7 @@
+ .ref_name = (char *)name,
+ .update_index = update_index,
+ };
-+ struct record rec = {};
++ struct record rec = { 0 };
+ record_from_log(&rec, &log);
+ return merged_table_seek_record(mt, it, rec);
+}
@@ -1739,7 +1674,7 @@
+struct merged_table {
+ struct reader **stack;
+ int stack_len;
-+ int hash_size;
++ uint32_t hash_id;
+
+ uint64_t min;
+ uint64_t max;
@@ -1747,7 +1682,7 @@
+
+struct merged_iter {
+ struct iterator *stack;
-+ int hash_size;
++ uint32_t hash_id;
+ int stack_len;
+ byte typ;
+ struct merged_iter_pqueue pq;
@@ -1776,8 +1711,8 @@
+
+int pq_less(struct pq_entry a, struct pq_entry b)
+{
-+ struct slice ak = {};
-+ struct slice bk = {};
++ struct slice ak = { 0 };
++ struct slice bk = { 0 };
+ int cmp = 0;
+ record_key(a.rec, &ak);
+ record_key(b.rec, &bk);
@@ -1896,8 +1831,8 @@
+#include "record.h"
+
+struct pq_entry {
-+ struct record rec;
+ int index;
++ struct record rec;
+};
+
+int pq_less(struct pq_entry a, struct pq_entry b);
@@ -2005,9 +1940,9 @@
+ block_source_return_block(r->source, p);
+}
+
-+int reader_hash_size(struct reader *r)
++uint32_t reader_hash_id(struct reader *r)
+{
-+ return r->hash_size;
++ return r->hash_id;
+}
+
+const char *reader_name(struct reader *r)
@@ -2030,11 +1965,12 @@
+ goto exit;
+ }
+
-+ r->hash_size = SHA1_SIZE;
++ r->hash_id = SHA1_ID;
+ {
+ byte version = *f++;
+ if (version == 2) {
-+ r->hash_size = SHA256_SIZE;
++ /* DO NOT SUBMIT. Not yet in the standard. */
++ r->hash_id = SHA256_ID;
+ version = 1;
+ }
+ if (version != 1) {
@@ -2043,33 +1979,33 @@
+ }
+ }
+
-+ r->block_size = get_u24(f);
++ r->block_size = get_be24(f);
+
+ f += 3;
-+ r->min_update_index = get_u64(f);
++ r->min_update_index = get_be64(f);
+ f += 8;
-+ r->max_update_index = get_u64(f);
++ r->max_update_index = get_be64(f);
+ f += 8;
+
-+ r->ref_offsets.index_offset = get_u64(f);
++ r->ref_offsets.index_offset = get_be64(f);
+ f += 8;
+
-+ r->obj_offsets.offset = get_u64(f);
++ r->obj_offsets.offset = get_be64(f);
+ f += 8;
+
+ r->object_id_len = r->obj_offsets.offset & ((1 << 5) - 1);
+ r->obj_offsets.offset >>= 5;
+
-+ r->obj_offsets.index_offset = get_u64(f);
++ r->obj_offsets.index_offset = get_be64(f);
+ f += 8;
-+ r->log_offsets.offset = get_u64(f);
++ r->log_offsets.offset = get_be64(f);
+ f += 8;
-+ r->log_offsets.index_offset = get_u64(f);
++ r->log_offsets.index_offset = get_be64(f);
+ f += 8;
+
+ {
+ uint32_t computed_crc = crc32(0, footer, f - footer);
-+ uint32_t file_crc = get_u32(f);
++ uint32_t file_crc = get_be32(f);
+ f += 4;
+ if (computed_crc != file_crc) {
+ err = FORMAT_ERROR;
@@ -2092,15 +2028,15 @@
+
+int init_reader(struct reader *r, struct block_source source, const char *name)
+{
-+ struct block footer = {};
-+ struct block header = {};
++ struct block footer = { 0 };
++ struct block header = { 0 };
+ int err = 0;
+
+ memset(r, 0, sizeof(struct reader));
+ r->size = block_source_size(source) - FOOTER_SIZE;
+ r->source = source;
-+ r->name = strdup(name);
-+ r->hash_size = 0;
++ r->name = xstrdup(name);
++ r->hash_id = 0;
+
+ err = block_source_read_block(source, &footer, r->size, FOOTER_SIZE);
+ if (err != FOOTER_SIZE) {
@@ -2173,7 +2109,7 @@
+
+ *typ = data[0];
+ if (is_block_type(*typ)) {
-+ result = get_u24(data + 1);
++ result = get_be24(data + 1);
+ }
+ return result;
+}
@@ -2183,7 +2119,7 @@
+{
+ int32_t guess_block_size = r->block_size ? r->block_size :
+ DEFAULT_BLOCK_SIZE;
-+ struct block block = {};
++ struct block block = { 0 };
+ byte block_typ = 0;
+ int err = 0;
+ uint32_t header_off = next_off ? 0 : HEADER_SIZE;
@@ -2217,14 +2153,14 @@
+ }
+
+ return block_reader_init(br, &block, header_off, r->block_size,
-+ r->hash_size);
++ hash_size(r->hash_id));
+}
+
+static int table_iter_next_block(struct table_iter *dest,
+ struct table_iter *src)
+{
+ uint64_t next_block_off = src->block_off + src->bi.br->full_block_size;
-+ struct block_reader br = {};
++ struct block_reader br = { 0 };
+ int err = 0;
+
+ dest->r = src->r;
@@ -2257,7 +2193,7 @@
+ }
+
+ while (true) {
-+ struct table_iter next = {};
++ struct table_iter next = { 0 };
+ int err = 0;
+ if (ti->finished) {
+ return 1;
@@ -2307,7 +2243,7 @@
+static int reader_table_iter_at(struct reader *r, struct table_iter *ti,
+ uint64_t off, byte typ)
+{
-+ struct block_reader br = {};
++ struct block_reader br = { 0 };
+ struct block_reader *brp = NULL;
+
+ int err = reader_init_block_reader(r, &br, off, typ);
@@ -2344,9 +2280,9 @@
+ struct record want)
+{
+ struct record rec = new_record(record_type(want));
-+ struct slice want_key = {};
-+ struct slice got_key = {};
-+ struct table_iter next = {};
++ struct slice want_key = { 0 };
++ struct slice got_key = { 0 };
++ struct table_iter next = { 0 };
+ int err = -1;
+ record_key(want, &want_key);
+
@@ -2394,12 +2330,12 @@
+static int reader_seek_indexed(struct reader *r, struct iterator *it,
+ struct record rec)
+{
-+ struct index_record want_index = {};
-+ struct record want_index_rec = {};
-+ struct index_record index_result = {};
-+ struct record index_result_rec = {};
-+ struct table_iter index_iter = {};
-+ struct table_iter next = {};
++ struct index_record want_index = { 0 };
++ struct record want_index_rec = { 0 };
++ struct index_record index_result = { 0 };
++ struct record index_result_rec = { 0 };
++ struct table_iter index_iter = { 0 };
++ struct table_iter next = { 0 };
+ int err = 0;
+
+ record_key(rec, &want_index.last_key);
@@ -2461,7 +2397,7 @@
+{
+ struct reader_offsets *offs = reader_offsets_for(r, record_type(rec));
+ uint64_t idx = offs->index_offset;
-+ struct table_iter ti = {};
++ struct table_iter ti = { 0 };
+ int err = 0;
+ if (idx > 0) {
+ return reader_seek_indexed(r, it, rec);
@@ -2503,7 +2439,7 @@
+ struct ref_record ref = {
+ .ref_name = (char *)name,
+ };
-+ struct record rec = {};
++ struct record rec = { 0 };
+ record_from_ref(&rec, &ref);
+ return reader_seek(r, it, rec);
+}
@@ -2515,7 +2451,7 @@
+ .ref_name = (char *)name,
+ .update_index = update_index,
+ };
-+ struct record rec = {};
++ struct record rec = { 0 };
+ record_from_log(&rec, &log);
+ return reader_seek(r, it, rec);
+}
@@ -2557,10 +2493,10 @@
+ .hash_prefix = oid,
+ .hash_prefix_len = r->object_id_len,
+ };
-+ struct record want_rec = {};
-+ struct iterator oit = {};
-+ struct obj_record got = {};
-+ struct record got_rec = {};
++ struct record want_rec = { 0 };
++ struct iterator oit = { 0 };
++ struct obj_record got = { 0 };
++ struct record got_rec = { 0 };
+ int err = 0;
+
+ record_from_obj(&want_rec, &want);
@@ -2585,7 +2521,8 @@
+
+ {
+ struct indexed_table_ref_iter *itr = NULL;
-+ err = new_indexed_table_ref_iter(&itr, r, oid, r->hash_size,
++ err = new_indexed_table_ref_iter(&itr, r, oid,
++ hash_size(r->hash_id),
+ got.offsets, got.offset_len);
+ if (err < 0) {
+ record_clear(got_rec);
@@ -2675,9 +2612,9 @@
+};
+
+struct reader {
-+ struct block_source source;
+ char *name;
-+ int hash_size;
++ struct block_source source;
++ uint32_t hash_id;
+ uint64_t size;
+ uint32_t block_size;
+ uint64_t min_update_index;
@@ -2755,7 +2692,7 @@
+
+int put_var_int(struct slice dest, uint64_t val)
+{
-+ byte buf[10] = {};
++ byte buf[10] = { 0 };
+ int i = 9;
+ buf[i] = (byte)(val & 0x7f);
+ i--;
@@ -2868,11 +2805,11 @@
+ fields. */
+ ref_record_clear(ref);
+ if (src->ref_name != NULL) {
-+ ref->ref_name = strdup(src->ref_name);
++ ref->ref_name = xstrdup(src->ref_name);
+ }
+
+ if (src->target != NULL) {
-+ ref->target = strdup(src->target);
++ ref->target = xstrdup(src->target);
+ }
+
+ if (src->target_value != NULL) {
@@ -2910,7 +2847,7 @@
+
+void ref_record_print(struct ref_record *ref, int hash_size)
+{
-+ char hex[SHA256_SIZE + 1] = {};
++ char hex[SHA256_SIZE + 1] = { 0 };
+
+ printf("ref{%s(%" PRIu64 ") ", ref->ref_name, ref->update_index);
+ if (ref->value != NULL) {
@@ -3066,7 +3003,7 @@
+ in.len -= hash_size;
+ break;
+ case 3: {
-+ struct slice dest = {};
++ struct slice dest = { 0 };
+ int n = decode_string(&dest, in);
+ if (n < 0) {
+ return -1;
@@ -3302,7 +3239,7 @@
+
+void log_record_print(struct log_record *log, int hash_size)
+{
-+ char hex[SHA256_SIZE + 1] = {};
++ char hex[SHA256_SIZE + 1] = { 0 };
+
+ printf("log{%s(%" PRIu64 ") %s <%s> %" PRIu64 " %04d\n", log->ref_name,
+ log->update_index, log->name, log->email, log->time,
@@ -3326,7 +3263,7 @@
+ slice_resize(dest, len + 9);
+ memcpy(dest->buf, rec->ref_name, len + 1);
+ ts = (~ts) - rec->update_index;
-+ put_u64(dest->buf + 1 + len, ts);
++ put_be64(dest->buf + 1 + len, ts);
+}
+
+static void log_record_copy_from(void *rec, const void *src_rec, int hash_size)
@@ -3335,10 +3272,10 @@
+ const struct log_record *src = (const struct log_record *)src_rec;
+
+ *dst = *src;
-+ dst->ref_name = strdup(dst->ref_name);
-+ dst->email = strdup(dst->email);
-+ dst->name = strdup(dst->name);
-+ dst->message = strdup(dst->message);
++ dst->ref_name = xstrdup(dst->ref_name);
++ dst->email = xstrdup(dst->email);
++ dst->name = xstrdup(dst->name);
++ dst->message = xstrdup(dst->message);
+ if (dst->new_hash != NULL) {
+ dst->new_hash = malloc(hash_size);
+ memcpy(dst->new_hash, src->new_hash, hash_size);
@@ -3371,7 +3308,7 @@
+ return 1;
+}
+
-+static byte zero[SHA256_SIZE] = {};
++static byte zero[SHA256_SIZE] = { 0 };
+
+static int log_record_encode(const void *rec, struct slice s, int hash_size)
+{
@@ -3421,7 +3358,7 @@
+ return -1;
+ }
+
-+ put_u16(s.buf, r->tz_offset);
++ put_be16(s.buf, r->tz_offset);
+ s.buf += 2;
+ s.len -= 2;
+
@@ -3442,7 +3379,7 @@
+ struct log_record *r = (struct log_record *)rec;
+ uint64_t max = 0;
+ uint64_t ts = 0;
-+ struct slice dest = {};
++ struct slice dest = { 0 };
+ int n;
+
+ if (key.len <= 9 || key.buf[key.len - 9] != 0) {
@@ -3451,7 +3388,7 @@
+
+ r->ref_name = realloc(r->ref_name, key.len - 8);
+ memcpy(r->ref_name, key.buf, key.len - 8);
-+ ts = get_u64(key.buf + key.len - 8);
++ ts = get_be64(key.buf + key.len - 8);
+
+ r->update_index = (~max) - ts;
+
@@ -3503,7 +3440,7 @@
+ goto error;
+ }
+
-+ r->tz_offset = get_u16(in.buf);
++ r->tz_offset = get_be16(in.buf);
+ in.buf += 2;
+ in.len -= 2;
+
@@ -3810,6 +3747,18 @@
+{
+ /* XXX */
+ return false;
++}
++
++int hash_size(uint32_t id)
++{
++ switch (id) {
++ case 0:
++ case SHA1_ID:
++ return SHA1_SIZE;
++ case SHA256_ID:
++ return SHA256_SIZE;
++ }
++ abort();
+}
diff --git a/reftable/record.h b/reftable/record.h
@@ -3842,7 +3791,7 @@
+ void (*clear)(void *rec);
+};
+
-+/* record is a generic wrapper for differnt types of records. */
++/* record is a generic wrapper for different types of records. */
+struct record {
+ void *data;
+ struct record_vtable *ops;
@@ -3863,8 +3812,8 @@
+ struct slice in);
+
+struct index_record {
-+ struct slice last_key;
+ uint64_t offset;
++ struct slice last_key;
+};
+
+struct obj_record {
@@ -3965,9 +3914,10 @@
+ /* how often to write complete keys in each block. */
+ int restart_interval;
+
-+ /* width of the hash. Should be 20 for SHA1 or 32 for SHA256. Defaults
-+ * to SHA1 if unset */
-+ int hash_size;
++ /* 4-byte identifier ("sha1", "s256") of the hash.
++ * Defaults to SHA1 if unset
++ */
++ uint32_t hash_id;
+};
+
+/* ref_record holds a ref database entry target_value */
@@ -4102,6 +4052,9 @@
+/* Decompression error */
+#define ZLIB_ERROR -7
+
++/* Wrote a table without blocks. */
++#define EMPTY_TABLE_ERROR -8
++
+const char *error_str(int err);
+
+/* new_writer creates a new writer */
@@ -4162,10 +4115,10 @@
+ struct reader *r = NULL;
+ int err = new_reader(&r, src, "filename");
+ if (err < 0) { ... }
-+ struct iterator it = {};
++ struct iterator it = {0};
+ err = reader_seek_ref(r, &it, "refs/heads/master");
+ if (err < 0) { ... }
-+ struct ref_record ref = {};
++ struct ref_record ref = {0};
+ while (1) {
+ err = iterator_next_ref(it, &ref);
+ if (err > 0) {
@@ -4181,8 +4134,8 @@
+ */
+int reader_seek_ref(struct reader *r, struct iterator *it, const char *name);
+
-+/* returns the hash size used in this table. */
-+int reader_hash_size(struct reader *r);
++/* returns the hash ID used in this table. */
++uint32_t reader_hash_id(struct reader *r);
+
+/* seek to logs for the given name, older than update_index. */
+int reader_seek_log_at(struct reader *r, struct iterator *it, const char *name,
@@ -4211,10 +4164,10 @@
+ array.
+*/
+int new_merged_table(struct merged_table **dest, struct reader **stack, int n,
-+ int hash_size);
++ uint32_t hash_id);
+
-+/* returns the hash size used in this merged table. */
-+int merged_hash_size(struct merged_table *mt);
++/* returns the hash id used in this merged table. */
++uint32_t merged_hash_id(struct merged_table *mt);
+
+/* returns an iterator positioned just before 'name' */
+int merged_table_seek_ref(struct merged_table *mt, struct iterator *it,
@@ -4403,7 +4356,7 @@
+/* return a newly malloced string for this slice */
+char *slice_to_string(struct slice in)
+{
-+ struct slice s = {};
++ struct slice s = { 0 };
+ slice_resize(&s, in.len + 1);
+ s.buf[in.len] = 0;
+ memcpy(s.buf, in.buf, in.len);
@@ -4585,8 +4538,8 @@
+ struct stack *p = calloc(sizeof(struct stack), 1);
+ int err = 0;
+ *dest = NULL;
-+ p->list_file = strdup(list_file);
-+ p->reftable_dir = strdup(dir);
++ p->list_file = xstrdup(list_file);
++ p->reftable_dir = xstrdup(dir);
+ p->config = config;
+
+ err = stack_reload(p);
@@ -4690,7 +4643,7 @@
+ int new_tables_len = 0;
+ struct merged_table *new_merged = NULL;
+
-+ struct slice table_path = {};
++ struct slice table_path = { 0 };
+
+ while (*names) {
+ struct reader *rd = NULL;
@@ -4708,7 +4661,7 @@
+ }
+
+ if (rd == NULL) {
-+ struct block_source src = {};
++ struct block_source src = { 0 };
+ slice_set_string(&table_path, st->reftable_dir);
+ slice_append_string(&table_path, "/");
+ slice_append_string(&table_path, name);
@@ -4730,7 +4683,7 @@
+
+ /* success! */
+ err = new_merged_table(&new_merged, new_tables, new_tables_len,
-+ st->config.hash_size);
++ st->config.hash_id);
+ if (err < 0) {
+ goto exit;
+ }
@@ -4780,7 +4733,7 @@
+
+static int stack_reload_maybe_reuse(struct stack *st, bool reuse_open)
+{
-+ struct timeval deadline = {};
++ struct timeval deadline = { 0 };
+ int err = gettimeofday(&deadline, NULL);
+ int64_t delay = 0;
+ int tries = 0;
@@ -4792,7 +4745,7 @@
+ while (true) {
+ char **names = NULL;
+ char **names_after = NULL;
-+ struct timeval now = {};
++ struct timeval now = { 0 };
+ int err = gettimeofday(&now, NULL);
+ if (err < 0) {
+ return err;
@@ -4904,11 +4857,11 @@
+int stack_try_add(struct stack *st,
+ int (*write_table)(struct writer *wr, void *arg), void *arg)
+{
-+ struct slice lock_name = {};
-+ struct slice temp_tab_name = {};
-+ struct slice tab_name = {};
-+ struct slice next_name = {};
-+ struct slice table_list = {};
++ struct slice lock_name = { 0 };
++ struct slice temp_tab_name = { 0 };
++ struct slice tab_name = { 0 };
++ struct slice next_name = { 0 };
++ struct slice table_list = { 0 };
+ struct writer *wr = NULL;
+ int err = 0;
+ int tab_fd = 0;
@@ -4962,6 +4915,10 @@
+ }
+
+ err = writer_close(wr);
++ if (err == EMPTY_TABLE_ERROR) {
++ err = 0;
++ goto exit;
++ }
+ if (err < 0) {
+ goto exit;
+ }
@@ -5062,7 +5019,7 @@
+ struct slice *temp_tab,
+ struct log_expiry_config *config)
+{
-+ struct slice next_name = {};
++ struct slice next_name = { 0 };
+ int tab_fd = -1;
+ struct writer *wr = NULL;
+ int err = 0;
@@ -5113,9 +5070,9 @@
+ calloc(sizeof(struct reader *), last - first + 1);
+ struct merged_table *mt = NULL;
+ int err = 0;
-+ struct iterator it = {};
-+ struct ref_record ref = {};
-+ struct log_record log = {};
++ struct iterator it = { 0 };
++ struct ref_record ref = { 0 };
++ struct log_record log = { 0 };
+
+ int i = 0, j = 0;
+ for (i = first, j = 0; i <= last; i++) {
@@ -5126,7 +5083,7 @@
+ writer_set_limits(wr, st->merged->stack[first]->min_update_index,
+ st->merged->stack[last]->max_update_index);
+
-+ err = new_merged_table(&mt, subtabs, subtabs_len, st->config.hash_size);
++ err = new_merged_table(&mt, subtabs, subtabs_len, st->config.hash_id);
+ if (err < 0) {
+ free(subtabs);
+ goto exit;
@@ -5207,11 +5164,11 @@
+static int stack_compact_range(struct stack *st, int first, int last,
+ struct log_expiry_config *expiry)
+{
-+ struct slice temp_tab_name = {};
-+ struct slice new_table_name = {};
-+ struct slice lock_file_name = {};
-+ struct slice ref_list_contents = {};
-+ struct slice new_table_path = {};
++ struct slice temp_tab_name = { 0 };
++ struct slice new_table_name = { 0 };
++ struct slice lock_file_name = { 0 };
++ struct slice ref_list_contents = { 0 };
++ struct slice new_table_path = { 0 };
+ int err = 0;
+ bool have_lock = false;
+ int lock_file_fd = 0;
@@ -5220,6 +5177,7 @@
+ char **subtable_locks = calloc(sizeof(char *), compact_count + 1);
+ int i = 0;
+ int j = 0;
++ bool is_empty_table = false;
+
+ if (first > last || (expiry == NULL && first == last)) {
+ err = 0;
@@ -5248,8 +5206,8 @@
+ }
+
+ for (i = first, j = 0; i <= last; i++) {
-+ struct slice subtab_name = {};
-+ struct slice subtab_lock = {};
++ struct slice subtab_name = { 0 };
++ struct slice subtab_lock = { 0 };
+ slice_set_string(&subtab_name, st->reftable_dir);
+ slice_append_string(&subtab_name, "/");
+ slice_append_string(&subtab_name,
@@ -5288,6 +5246,12 @@
+ have_lock = false;
+
+ err = stack_compact_locked(st, first, last, &temp_tab_name, expiry);
++ /* Compaction + tombstones can create an empty table out of non-empty
++ * tables. */
++ is_empty_table = (err == EMPTY_TABLE_ERROR);
++ if (is_empty_table) {
++ err = 0;
++ }
+ if (err < 0) {
+ goto exit;
+ }
@@ -5313,10 +5277,12 @@
+
+ slice_append(&new_table_path, new_table_name);
+
-+ err = rename(slice_as_string(&temp_tab_name),
-+ slice_as_string(&new_table_path));
-+ if (err < 0) {
-+ goto exit;
++ if (!is_empty_table) {
++ err = rename(slice_as_string(&temp_tab_name),
++ slice_as_string(&new_table_path));
++ if (err < 0) {
++ goto exit;
++ }
+ }
+
+ for (i = 0; i < first; i++) {
@@ -5324,8 +5290,10 @@
+ st->merged->stack[i]->name);
+ slice_append_string(&ref_list_contents, "\n");
+ }
-+ slice_append(&ref_list_contents, new_table_name);
-+ slice_append_string(&ref_list_contents, "\n");
++ if (!is_empty_table) {
++ slice_append(&ref_list_contents, new_table_name);
++ slice_append_string(&ref_list_contents, "\n");
++ }
+ for (i = last + 1; i < st->merged->stack_len; i++) {
+ slice_append_string(&ref_list_contents,
+ st->merged->stack[i]->name);
@@ -5351,18 +5319,26 @@
+ }
+ have_lock = false;
+
-+ for (char **p = delete_on_success; *p; p++) {
-+ if (strcmp(*p, slice_as_string(&new_table_path))) {
-+ unlink(*p);
++ {
++ char **p = delete_on_success;
++ while (*p) {
++ if (strcmp(*p, slice_as_string(&new_table_path))) {
++ unlink(*p);
++ }
++ p++;
+ }
+ }
+
+ err = stack_reload_maybe_reuse(st, first < last);
+exit:
-+ for (char **p = subtable_locks; *p; p++) {
-+ unlink(*p);
-+ }
+ free_names(delete_on_success);
++ {
++ char **p = subtable_locks;
++ while (*p) {
++ unlink(*p);
++ p++;
++ }
++ }
+ free_names(subtable_locks);
+ if (lock_file_fd > 0) {
+ close(lock_file_fd);
@@ -5413,7 +5389,7 @@
+{
+ struct segment *segs = calloc(sizeof(struct segment), n);
+ int next = 0;
-+ struct segment cur = {};
++ struct segment cur = { 0 };
+ int i = 0;
+ for (i = 0; i < n; i++) {
+ int log = fastlog2(sizes[i]);
@@ -5501,7 +5477,7 @@
+int stack_read_ref(struct stack *st, const char *refname,
+ struct ref_record *ref)
+{
-+ struct iterator it = {};
++ struct iterator it = { 0 };
+ struct merged_table *mt = stack_merged_table(st);
+ int err = merged_table_seek_ref(mt, &it, refname);
+ if (err) {
@@ -5526,7 +5502,7 @@
+int stack_read_log(struct stack *st, const char *refname,
+ struct log_record *log)
+{
-+ struct iterator it = {};
++ struct iterator it = { 0 };
+ struct merged_table *mt = stack_merged_table(st);
+ int err = merged_table_seek_log(mt, &it, refname);
+ if (err) {
@@ -5631,31 +5607,25 @@
+#include <unistd.h>
+#include <zlib.h>
+
-+#define ARRAY_SIZE(a) sizeof((a)) / sizeof((a)[0])
-+#define FREE_AND_NULL(x) \
-+ do { \
-+ free(x); \
-+ (x) = NULL; \
-+ } while (0)
-+#define QSORT(arr, n, cmp) qsort(arr, n, sizeof(arr[0]), cmp)
-+#define SWAP(a, b) \
-+ { \
-+ char tmp[sizeof(a)]; \
-+ assert(sizeof(a) == sizeof(b)); \
-+ memcpy(&tmp[0], &a, sizeof(a)); \
-+ memcpy(&a, &b, sizeof(a)); \
-+ memcpy(&b, &tmp[0], sizeof(a)); \
-+ }
++#include "compat.h"
++
+#endif /* REFTABLE_IN_GITCORE */
+
++#define SHA1_ID 0x73686131
++#define SHA256_ID 0x73323536
++#define SHA1_SIZE 20
++#define SHA256_SIZE 32
++
+typedef uint8_t byte;
+typedef int bool;
+
++/* This is uncompress2, which is only available in zlib as of 2017.
++ *
++ * TODO: in git-core, this should fallback to uncompress2 if it is available.
++ */
+int uncompress_return_consumed(Bytef *dest, uLongf *destLen,
+ const Bytef *source, uLong *sourceLen);
-+
-+#define SHA1_SIZE 20
-+#define SHA256_SIZE 32
++int hash_size(uint32_t id);
+
+#endif
@@ -5761,6 +5731,25 @@
+
+#endif
+ diff --git a/reftable/update.sh b/reftable/update.sh
+ new file mode 100644
+ --- /dev/null
+ +++ b/reftable/update.sh
+@@
++#!/bin/sh
++
++set -eux
++
++((cd reftable-repo && git fetch origin && git checkout origin/master ) ||
++git clone https://github.com/google/reftable reftable-repo) && \
++cp reftable-repo/c/*.[ch] reftable/ && \
++cp reftable-repo/LICENSE reftable/ &&
++git --git-dir reftable-repo/.git show --no-patch origin/master \
++> reftable/VERSION && \
++sed -i~ 's|if REFTABLE_IN_GITCORE|if 1 /* REFTABLE_IN_GITCORE */|' reftable/system.h
++rm reftable/*_test.c reftable/test_framework.* reftable/compat.*
++git add reftable/*.[ch]
+
diff --git a/reftable/writer.c b/reftable/writer.c
new file mode 100644
--- /dev/null
@@ -5831,8 +5820,8 @@
+ opts->restart_interval = 16;
+ }
+
-+ if (opts->hash_size == 0) {
-+ opts->hash_size = SHA1_SIZE;
++ if (opts->hash_id == 0) {
++ opts->hash_id = SHA1_ID;
+ }
+ if (opts->block_size == 0) {
+ opts->block_size = DEFAULT_BLOCK_SIZE;
@@ -5842,11 +5831,14 @@
+static int writer_write_header(struct writer *w, byte *dest)
+{
+ memcpy((char *)dest, "REFT", 4);
-+ dest[4] = (w->hash_size == SHA1_SIZE) ? 1 : 2; /* version */
+
-+ put_u24(dest + 5, w->opts.block_size);
-+ put_u64(dest + 8, w->min_update_index);
-+ put_u64(dest + 16, w->max_update_index);
++ /* DO NOT SUBMIT. This has not been encoded in the standard yet. */
++ dest[4] = (hash_size(w->opts.hash_id) == SHA1_SIZE) ? 1 : 2; /* version
++ */
++
++ put_be24(dest + 5, w->opts.block_size);
++ put_be64(dest + 8, w->min_update_index);
++ put_be64(dest + 16, w->max_update_index);
+ return 24;
+}
+
@@ -5858,7 +5850,8 @@
+ }
+
+ block_writer_init(&w->block_writer_data, typ, w->block,
-+ w->opts.block_size, block_start, w->hash_size);
++ w->opts.block_size, block_start,
++ hash_size(w->opts.hash_id));
+ w->block_writer = &w->block_writer_data;
+ w->block_writer->restart_interval = w->opts.restart_interval;
+}
@@ -5872,7 +5865,6 @@
+ /* TODO - error return? */
+ abort();
+ }
-+ wp->hash_size = opts->hash_size;
+ wp->block = calloc(opts->block_size, 1);
+ wp->write = writer_func;
+ wp->write_arg = writer_arg;
@@ -5941,7 +5933,7 @@
+static int writer_add_record(struct writer *w, struct record rec)
+{
+ int result = -1;
-+ struct slice key = {};
++ struct slice key = { 0 };
+ int err = 0;
+ record_key(rec, &key);
+ if (slice_compare(w->last_key, key) >= 0) {
@@ -5981,7 +5973,7 @@
+
+int writer_add_ref(struct writer *w, struct ref_record *ref)
+{
-+ struct record rec = {};
++ struct record rec = { 0 };
+ struct ref_record copy = *ref;
+ int err = 0;
+
@@ -6003,7 +5995,7 @@
+ if (!w->opts.skip_index_objects && ref->value != NULL) {
+ struct slice h = {
+ .buf = ref->value,
-+ .len = w->hash_size,
++ .len = hash_size(w->opts.hash_id),
+ };
+
+ writer_index_hash(w, h);
@@ -6011,7 +6003,7 @@
+ if (!w->opts.skip_index_objects && ref->target_value != NULL) {
+ struct slice h = {
+ .buf = ref->target_value,
-+ .len = w->hash_size,
++ .len = hash_size(w->opts.hash_id),
+ };
+ writer_index_hash(w, h);
+ }
@@ -6047,7 +6039,7 @@
+ w->pending_padding = 0;
+
+ {
-+ struct record rec = {};
++ struct record rec = { 0 };
+ int err;
+ record_from_log(&rec, log);
+ err = writer_add_record(w, rec);
@@ -6094,7 +6086,7 @@
+ w->index_len = 0;
+ w->index_cap = 0;
+ for (i = 0; i < idx_len; i++) {
-+ struct record rec = {};
++ struct record rec = { 0 };
+ record_from_index(&rec, idx + i);
+ if (block_writer_add(w->block_writer, rec) == 0) {
+ continue;
@@ -6172,7 +6164,7 @@
+ .offsets = entry->offsets,
+ .offset_len = entry->offset_len,
+ };
-+ struct record rec = {};
++ struct record rec = { 0 };
+ if (arg->err < 0) {
+ goto exit;
+ }
@@ -6214,7 +6206,7 @@
+static int writer_dump_object_index(struct writer *w)
+{
+ struct write_record_arg closure = { .w = w };
-+ struct common_prefix_arg common = {};
++ struct common_prefix_arg common = { 0 };
+ if (w->obj_index_tree != NULL) {
+ infix_walk(w->obj_index_tree, &update_common, &common);
+ }
@@ -6271,39 +6263,43 @@
+
+ int err = writer_finish_public_section(w);
+ if (err != 0) {
-+ return err;
++ goto exit;
+ }
+
+ writer_write_header(w, footer);
+ p += 24;
-+ put_u64(p, w->stats.ref_stats.index_offset);
++ put_be64(p, w->stats.ref_stats.index_offset);
+ p += 8;
-+ put_u64(p, (w->stats.obj_stats.offset) << 5 | w->stats.object_id_len);
++ put_be64(p, (w->stats.obj_stats.offset) << 5 | w->stats.object_id_len);
+ p += 8;
-+ put_u64(p, w->stats.obj_stats.index_offset);
++ put_be64(p, w->stats.obj_stats.index_offset);
+ p += 8;
+
-+ put_u64(p, w->stats.log_stats.offset);
++ put_be64(p, w->stats.log_stats.offset);
+ p += 8;
-+ put_u64(p, w->stats.log_stats.index_offset);
++ put_be64(p, w->stats.log_stats.index_offset);
+ p += 8;
+
-+ put_u32(p, crc32(0, footer, p - footer));
++ put_be32(p, crc32(0, footer, p - footer));
+ p += 4;
+ w->pending_padding = 0;
+
-+ {
-+ int n = padded_write(w, footer, sizeof(footer), 0);
-+ if (n < 0) {
-+ return n;
-+ }
++ err = padded_write(w, footer, sizeof(footer), 0);
++ if (err < 0) {
++ goto exit;
++ }
++
++ if (w->stats.log_stats.entries + w->stats.ref_stats.entries == 0) {
++ err = EMPTY_TABLE_ERROR;
++ goto exit;
+ }
+
++exit:
+ /* free up memory. */
+ block_writer_clear(&w->block_writer_data);
+ writer_clear_index(w);
+ free(slice_yield(&w->last_key));
-+ return 0;
++ return err;
+}
+
+void writer_clear_index(struct writer *w)
@@ -6348,7 +6344,7 @@
+ if (debug) {
+ fprintf(stderr, "block %c off %" PRIu64 " sz %d (%d)\n", typ,
+ w->next, raw_bytes,
-+ get_u24(w->block + w->block_writer->header_off + 1));
++ get_be24(w->block + w->block_writer->header_off + 1));
+ }
+
+ if (w->next == 0) {
@@ -6423,7 +6419,6 @@
+ int (*write)(void *, byte *, int);
+ void *write_arg;
+ int pending_padding;
-+ int hash_size;
+ struct slice last_key;
+
+ uint64_t next;
5: 8d95ab52f75 ! 6: a622d8066c7 Reftable support for git-core
@@ -9,32 +9,7 @@
* Resolve spots marked with XXX
* Test strategy?
- Example use:
-
- $ ~/vc/git/git init --reftable
- warning: templates not found in /usr/local/google/home/hanwen/share/git-core/templates
- Initialized empty Git repository in /tmp/qz/.git/
- $ echo q > a
- $ ~/vc/git/git add a
- $ ~/vc/git/git commit -mx
- fatal: not a git repository (or any of the parent directories): .git
- [master (root-commit) 373d969] x
- 1 file changed, 1 insertion(+)
- create mode 100644 a
- $ ~/vc/git/git show-ref
- 373d96972fca9b63595740bba3898a762778ba20 HEAD
- 373d96972fca9b63595740bba3898a762778ba20 refs/heads/master
- $ ls -l .git/reftable/
- total 12
- -rw------- 1 hanwen primarygroup 126 Jan 23 20:08 000000000001-000000000001.ref
- -rw------- 1 hanwen primarygroup 4277 Jan 23 20:08 000000000002-000000000002.ref
- $ go run ~/vc/reftable/cmd/dump.go -table /tmp/qz/.git/reftable/000000000002-000000000002.ref
- ** DEBUG **
- name /tmp/qz/.git/reftable/000000000002-000000000002.ref, sz 4209: 'r' reftable.readerOffsets{Present:true, Offset:0x0, IndexOffset:0x0}, 'o' reftable.readerOffsets{Present:false, Offset:0x0, IndexOffset:0x0} 'g' reftable.readerOffsets{Present:true, Offset:0x1000, IndexOffset:0x0}
- ** REFS **
- reftable.RefRecord{RefName:"refs/heads/master", UpdateIndex:0x2, Value:[]uint8{0x37, 0x3d, 0x96, 0x97, 0x2f, 0xca, 0x9b, 0x63, 0x59, 0x57, 0x40, 0xbb, 0xa3, 0x89, 0x8a, 0x76, 0x27, 0x78, 0xba, 0x20}, TargetValue:[]uint8(nil), Target:""}
- ** LOGS **
- reftable.LogRecord{RefName:"HEAD", UpdateIndex:0x2, New:[]uint8{0x37, 0x3d, 0x96, 0x97, 0x2f, 0xca, 0x9b, 0x63, 0x59, 0x57, 0x40, 0xbb, 0xa3, 0x89, 0x8a, 0x76, 0x27, 0x78, 0xba, 0x20}, Old:[]uint8{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, Name:"Han-Wen Nienhuys", Email:"[email protected]", Time:0x5e29ef27, TZOffset:100, Message:"commit (initial): x\n"}
+ Example use: see t/t0031-reftable.sh
Signed-off-by: Han-Wen Nienhuys <[email protected]>
Co-authored-by: Jeff King <[email protected]>
@@ -463,7 +438,7 @@ $^
+ struct ref_store *ref_store = (struct ref_store *)refs;
+ struct write_options cfg = {
+ .block_size = 4096,
-+ .hash_size = the_hash_algo->rawsz,
++ .hash_id = the_hash_algo->format_id,
+ };
+ struct strbuf sb = STRBUF_INIT;
+
@@ -610,11 +585,13 @@ $^
+ struct reftable_iterator *ri = xcalloc(1, sizeof(*ri));
+ struct merged_table *mt = NULL;
+
-+ mt = stack_merged_table(refs->stack);
-+ ri->err = refs->err;
-+ if (ri->err == 0) {
++ if (refs->err < 0) {
++ ri->err = refs->err;
++ } else {
++ mt = stack_merged_table(refs->stack);
+ ri->err = merged_table_seek_ref(mt, &ri->iter, prefix);
+ }
++
+ base_ref_iterator_init(&ri->base, &reftable_ref_iterator_vtable, 1);
+ ri->base.oid = &ri->oid;
+ ri->flags = flags;
@@ -739,7 +716,12 @@ $^
+{
+ struct reftable_ref_store *refs =
+ (struct reftable_ref_store *)ref_store;
-+ int err = stack_add(refs->stack, &write_transaction_table, transaction);
++ int err = 0;
++ if (refs->err < 0) {
++ return refs->err;
++ }
++
++ err = stack_add(refs->stack, &write_transaction_table, transaction);
+ if (err < 0) {
+ strbuf_addf(errmsg, "reftable: transaction failure %s",
+ error_str(err));
@@ -819,6 +801,10 @@ $^
+ .logmsg = msg,
+ .flags = flags,
+ };
++ if (refs->err < 0) {
++ return refs->err;
++ }
++
+ return stack_add(refs->stack, &write_delete_refs_table, &arg);
+}
+
@@ -826,6 +812,9 @@ $^
+{
+ struct reftable_ref_store *refs =
+ (struct reftable_ref_store *)ref_store;
++ if (refs->err < 0) {
++ return refs->err;
++ }
+ return stack_compact_all(refs->stack, NULL);
+}
+
@@ -896,6 +885,9 @@ $^
+ .refname = refname,
+ .target = target,
+ .logmsg = logmsg };
++ if (refs->err < 0) {
++ return refs->err;
++ }
+ return stack_add(refs->stack, &write_create_symref_table, &arg);
+}
+
@@ -912,6 +904,7 @@ $^
+ uint64_t ts = stack_next_update_index(arg->stack);
+ struct ref_record ref = {};
+ int err = stack_read_ref(arg->stack, arg->oldname, &ref);
++
+ if (err) {
+ goto exit;
+ }
@@ -987,6 +980,10 @@ $^
+ .newname = newrefname,
+ .logmsg = logmsg,
+ };
++ if (refs->err < 0) {
++ return refs->err;
++ }
++
+ return stack_add(refs->stack, &write_rename_table, &arg);
+}
+
@@ -1083,10 +1080,16 @@ $^
+ struct iterator it = {};
+ struct reftable_ref_store *refs =
+ (struct reftable_ref_store *)ref_store;
-+ struct merged_table *mt = stack_merged_table(refs->stack);
-+ int err = merged_table_seek_log(mt, &it, refname);
++ struct merged_table *mt = NULL;
++ int err = 0;
+ struct log_record log = {};
+
++ if (refs->err < 0) {
++ return refs->err;
++ }
++
++ mt = stack_merged_table(refs->stack);
++ err = merged_table_seek_log(mt, &it, refname);
+ while (err == 0) {
+ err = iterator_next_log(it, &log);
+ if (err != 0) {
@@ -1133,12 +1136,17 @@ $^
+ struct iterator it = {};
+ struct reftable_ref_store *refs =
+ (struct reftable_ref_store *)ref_store;
-+ struct merged_table *mt = stack_merged_table(refs->stack);
-+ int err = merged_table_seek_log(mt, &it, refname);
-+
++ struct merged_table *mt = NULL;
+ struct log_record *logs = NULL;
+ int cap = 0;
+ int len = 0;
++ int err = 0;
++
++ if (refs->err < 0) {
++ return refs->err;
++ }
++ mt = stack_merged_table(refs->stack);
++ err = merged_table_seek_log(mt, &it, refname);
+
+ while (err == 0) {
+ struct log_record log = {};
@@ -1280,13 +1288,19 @@ $^
+ */
+ struct reftable_ref_store *refs =
+ (struct reftable_ref_store *)ref_store;
-+ struct merged_table *mt = stack_merged_table(refs->stack);
++ struct merged_table *mt = NULL;
+ struct reflog_expiry_arg arg = {
+ .refs = refs,
+ };
+ struct log_record log = {};
+ struct iterator it = {};
-+ int err = merged_table_seek_log(mt, &it, refname);
++ int err = 0;
++ if (refs->err < 0) {
++ return refs->err;
++ }
++
++ mt = stack_merged_table(refs->stack);
++ err = merged_table_seek_log(mt, &it, refname);
+ if (err < 0) {
+ return err;
+ }
@@ -1326,7 +1340,12 @@ $^
+ struct reftable_ref_store *refs =
+ (struct reftable_ref_store *)ref_store;
+ struct ref_record ref = {};
-+ int err = stack_read_ref(refs->stack, refname, &ref);
++ int err = 0;
++ if (refs->err < 0) {
++ return refs->err;
++ }
++
++ err = stack_read_ref(refs->stack, refname, &ref);
+ if (err) {
+ goto exit;
+ }
@@ -1436,3 +1455,40 @@ $^
}
strbuf_release(&dir);
+
+ diff --git a/t/t0031-reftable.sh b/t/t0031-reftable.sh
+ new file mode 100755
+ --- /dev/null
+ +++ b/t/t0031-reftable.sh
+@@
++#!/bin/sh
++#
++# Copyright (c) 2020 Google LLC
++#
++
++test_description='reftable basics'
++
++. ./test-lib.sh
++
++test_expect_success 'basic operation of reftable storage' '
++ git init --ref-storage=reftable repo &&
++ cd repo &&
++ echo "hello" > world.txt &&
++ git add world.txt &&
++ git commit -m "first post" &&
++ printf "HEAD\nrefs/heads/master\n" > want &&
++ git show-ref | cut -f2 -d" " > got &&
++ test_cmp got want &&
++ for count in $(test_seq 1 10); do
++ echo "hello" >> world.txt
++ git commit -m "number ${count}" world.txt
++ done &&
++ git gc &&
++ nfiles=$(ls -1 .git/reftable | wc -l | tr -d "[ \t]" ) &&
++ test "${nfiles}" = "2" &&
++ git reflog refs/heads/master > output &&
++ grep "commit (initial): first post" output &&
++ grep "commit: number 10" output
++'
++
++test_doneSignature · 1 line
gitgitgadget
PATCH v7 1/6 refs.h: clarify reflog iteration order
From: Han-Wen Nienhuys <[email protected]>
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- refs.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
refs.h +4 -1
@@ -444,18 +444,21 @@ int delete_refs(const char *msg, struct string_list *refnames,int refs_delete_reflog (struct ref_store *refs,const char *refname);int delete_reflog (const char *refname); -/* iterate over reflog entries */ +/* Iterate over reflog entries. */ typedef int each_reflog_ent_fn (struct object_id *old_oid,struct object_id *new_oid,const char *committer,timestamp_t timestamp,int tz,const char *msg,void *cb_data); +/* Iterate in over reflog entries, oldest entry first. */ int refs_for_each_reflog_ent (struct ref_store *refs,const char *refname,each_reflog_ent_fn fn,void *cb_data);int refs_for_each_reflog_ent_reverse (struct ref_store *refs,const char *refname,each_reflog_ent_fn fn,void *cb_data); + +/* Call a function for each reflog entry, oldest entry first. */ int for_each_reflog_ent (const char *refname,each_reflog_ent_fn fn,void *cb_data);int for_each_reflog_ent_reverse (const char *refname,each_reflog_ent_fn fn,void *cb_data);
Signature · 1 line
gitgitgadget
PATCH v7 2/6 create .git/refs in files-backend.c
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This prepares for supporting the reftable format, which will want create its own file system layout in .git
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- builtin/init-db.c | 2 -- refs/files-backend.c | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-)
builtin/init-db.c +0 -2
@@ -226,8 +226,6 @@ static int create_default_files(const char *template_path,* We need to create a "refs" dir in any case so that older * versions of git can tell that this is a repository. */ -safe_create_dir (git_path ("refs" ),1 ); -adjust_shared_perm (git_path ("refs" ));if (refs_init_db (&err))die ("failed to set up refs db: %s" , err.buf);
refs/files-backend.c +6 -0
@@ -3157,9 +3157,15 @@ static int files_init_db(struct ref_store *ref_store, struct strbuf *err)files_downcast (ref_store,REF_STORE_WRITE ,"init_db" );struct strbuf sb =STRBUF_INIT ; +files_ref_path (refs, &sb,"refs" ); +safe_create_dir (sb.buf,1 ); +/* adjust permissions even if directory already exists. */ +adjust_shared_perm (sb.buf); +/* * Create .git/refs/{heads,tags} */ +strbuf_reset (&sb);files_ref_path (refs, &sb,"refs/heads" );safe_create_dir (sb.buf,1 );
Signature · 1 line
gitgitgadget
PATCH v7 3/6 refs: document how ref_iterator_advance_fn should handle symrefs
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- refs/refs-internal.h | 5 +++++ 1 file changed, 5 insertions(+)
refs/refs-internal.h +5 -0
@@ -438,6 +438,11 @@ void base_ref_iterator_free(struct ref_iterator *iter);/* Virtual function declarations for ref_iterators: */ +/* +* backend-specific implementation of ref_iterator_advance. +* For symrefs, the function should set REF_ISSYMREF, and it should also +* dereference the symref to provide the OID referent. +*/ typedef int ref_iterator_advance_fn (struct ref_iterator *ref_iterator); typedef int ref_iterator_peel_fn(struct ref_iterator *ref_iterator,
Signature · 1 line
gitgitgadget
PATCH v7 4/6 reftable: file format documentation
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Jonathan Nieder <[email protected]>
Shawn Pearce explains:
Some repositories contain a lot of references (e.g. android at 866k, rails at 31k). The reftable format provides:
- Near constant time lookup for any single reference, even when the repository is cold and not in process or kernel cache.
- Near constant time verification a SHA-1 is referred to by at least one reference (for allow-tip-sha1-in-want).
- Efficient lookup of an entire namespace, such as refs/tags/.
- Support atomic push O(size_of_update) operations.
- Combine reflog storage with ref storage.
This file format spec was originally written in July, 2017 by Shawn Pearce. Some refinements since then were made by Shawn and by Han-Wen Nienhuys based on experiences implementing and experimenting with the format. (All of this was in the context of our work at Google and Google is happy to contribute the result to the Git project.)
Imported from JGit[1]'s current version (c217d33ff, "Documentation/technical/reftable: improve repo layout", 2020-02-04) of Documentation/technical/reftable.md and converted to asciidoc by running
pandoc -t asciidoc -f markdown reftable.md >reftable.txt
using pandoc 2.2.1. The result required the following additional minor changes:
- removed the [TOC] directive to add a table of contents, since asciidoc does not support it
- replaced git-scm.com/docs links with linkgit: directives that link to other pages within Git's documentation
[1] https://eclipse.googlesource.com/jgit/jgit
Signed-off-by Jonathan Nieder <[email protected]>
--- Documentation/Makefile | 1 + Documentation/technical/reftable.txt | 1067 ++++++++++++++++++++++++++ 2 files changed, 1068 insertions(+) create mode 100644 Documentation/technical/reftable.txt
Documentation/Makefile +1 -0
@@ -92,6 +92,7 @@ TECH_DOCS += technical/protocol-capabilities TECH_DOCS += technical/protocol-common TECH_DOCS += technical/protocol-v2 TECH_DOCS += technical/racy-git +TECH_DOCS += technical/reftable TECH_DOCS += technical/send-pack-pipeline TECH_DOCS += technical/shallow TECH_DOCS += technical/signature-format
Documentation/technical/reftable.txt new file +1067 -0
1,067 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v7 5/6 Add reftable library
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Reftable is a format for storing the ref database. Its rationale and specification is in the preceding commit.
Further context and motivation can be found in background reading:
* Original discussion on JGit-dev: https://www.eclipse.org/lists/jgit-dev/msg03389.html
* First design discussion on git@vger: https://public-inbox.org/git/CAJo=hJtTp2eA3z9wW9cHo-nA7kK40vVThqh6inXpbCcqfdMP9g@mail.gmail.com/
* Last design discussion on git@vger: https://public-inbox.org/git/CAJo=hJsZcAM9sipdVr7TMD-FD2V2W6_pvMQ791EGCDsDkQ033w@mail.gmail.com/
* First attempt at implementation: https://public-inbox.org/git/CAP8UFD0PPZSjBnxCA7ez91vBuatcHKQ+JUWvTD1iHcXzPBjPBg@mail.gmail.com/
* libgit2 support issue: https://github.com/libgit2/libgit2/issues
* GitLab support issue: https://gitlab.com/gitlab-org/git/issues/6
* go-git support issue: https://github.com/src-d/go-git/issues/1059
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- reftable/LICENSE | 31 ++ reftable/README.md | 11 + reftable/VERSION | 5 + reftable/basics.c | 160 ++++++ reftable/basics.h | 30 ++ reftable/block.c | 413 +++++++++++++++ reftable/block.h | 71 +++ reftable/blocksource.h | 20 + reftable/bytes.c | 0 reftable/config.h | 1 + reftable/constants.h | 25 + reftable/dump.c | 97 ++++ reftable/file.c | 97 ++++ reftable/iter.c | 229 ++++++++ reftable/iter.h | 56 ++ reftable/merged.c | 290 +++++++++++ reftable/merged.h | 34 ++ reftable/pq.c | 114 ++++ reftable/pq.h | 34 ++ reftable/reader.c | 720 ++++++++++++++++++++++++++ reftable/reader.h | 52 ++ reftable/record.c | 1119 ++++++++++++++++++++++++++++++++++++++++ reftable/record.h | 79 +++ reftable/reftable.h | 409 +++++++++++++++ reftable/slice.c | 199 +++++++ reftable/slice.h | 39 ++ reftable/stack.c | 1007 ++++++++++++++++++++++++++++++++++++ reftable/stack.h | 40 ++ reftable/system.h | 53 ++ reftable/tree.c | 66 +++ reftable/tree.h | 24 + reftable/update.sh | 13 + reftable/writer.c | 637 +++++++++++++++++++++++ reftable/writer.h | 45 ++ reftable/zlib-compat.c | 92 ++++ 35 files changed, 6312 insertions(+) create mode 100644 reftable/LICENSE create mode 100644 reftable/README.md create mode 100644 reftable/VERSION create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/blocksource.h create mode 100644 reftable/bytes.c create mode 100644 reftable/config.h create mode 100644 reftable/constants.h create mode 100644 reftable/dump.c create mode 100644 reftable/file.c create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/reftable.h create mode 100644 reftable/slice.c create mode 100644 reftable/slice.h create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/system.h create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100644 reftable/update.sh create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h create mode 100644 reftable/zlib-compat.c
reftable/LICENSE new file +31 -0
@@ -0,0 +1,31 @@ +BSD License + +Copyright (c) 2020, Google LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of Google LLC nor the names of its contributors may +be used to endorse or promote products derived from this software +without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
reftable/README.md new file +11 -0
@@ -0,0 +1,11 @@ + +The source code in this directory comes from https://github.com/google/reftable. + +The VERSION file keeps track of the current version of the reftable library. + +To update the library, do: + + sh reftable/update.sh + +Bugfixes should be accompanied by a test and applied to upstream project at +https://github.com/google/reftable.
reftable/VERSION new file +5 -0
@@ -0,0 +1,5 @@ +commit ccce3b3eb763e79b23a3b4677d65ecceb4155ba3 +Author: Han-Wen Nienhuys <[email protected]> +Date: Tue Feb 25 20:42:29 2020 +0100 + + C: rephrase the hash ID API in terms of a uint32_t
reftable/basics.c new file +160 -0
160 lines not on this page · raw message
reftable/basics.h new file +30 -0
@@ -0,0 +1,30 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef BASICS_H +#define BASICS_H + +#include "system.h" + +#include "reftable.h" + +#define true 1 +#define false 0 + +void put_be24 (byte *out,uint32_t i); +uint32_t get_be24 (byte *in); +void put_be16 (uint8_t *out,uint16_t i); + +int binsearch (int sz,int (*f)(int k,void *args),void *args); + +void free_names (char **a); +void parse_names (char *buf,int size,char ***namesp); +int names_equal (char **a,char **b); +int names_length (char **names); + +#endif
reftable/block.c new file +413 -0
413 lines not on this page · raw message
reftable/block.h new file +71 -0
71 lines not on this page · raw message
reftable/blocksource.h new file +20 -0
@@ -0,0 +1,20 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef BLOCKSOURCE_H +#define BLOCKSOURCE_H + +#include "reftable.h" + +uint64_t block_source_size (struct block_source source); +int block_source_read_block (struct block_source source,struct block *dest, +uint64_t off,uint32_t size); +void block_source_return_block (struct block_source source,struct block *ret); +void block_source_close (struct block_source source); + +#endif
reftable/bytes.c new file +0 -0
reftable/config.h new file +1 -0
@@ -0,0 +1 @@ +/* empty */
reftable/constants.h new file +25 -0
@@ -0,0 +1,25 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef CONSTANTS_H +#define CONSTANTS_H + +#define VERSION 1 +#define HEADER_SIZE 24 +#define FOOTER_SIZE 68 + +#define BLOCK_TYPE_LOG 'g' +#define BLOCK_TYPE_INDEX 'i' +#define BLOCK_TYPE_REF 'r' +#define BLOCK_TYPE_OBJ 'o' +#define BLOCK_TYPE_ANY 0 + +#define MAX_RESTARTS ((1 << 16) - 1) +#define DEFAULT_BLOCK_SIZE 4096 + +#endif
reftable/dump.c new file +97 -0
97 lines not on this page · raw message
reftable/file.c new file +97 -0
97 lines not on this page · raw message
reftable/iter.c new file +229 -0
229 lines not on this page · raw message
reftable/iter.h new file +56 -0
56 lines not on this page · raw message
reftable/merged.c new file +290 -0
290 lines not on this page · raw message
reftable/merged.h new file +34 -0
@@ -0,0 +1,34 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef MERGED_H +#define MERGED_H + +#include "pq.h" +#include "reftable.h" + +struct merged_table { +struct reader **stack; +int stack_len; +uint32_t hash_id; + +uint64_t min; +uint64_t max; +}; + +struct merged_iter { +struct iterator *stack; +uint32_t hash_id; +int stack_len; +byte typ; +struct merged_iter_pqueue pq; +} merged_iter; + +void merged_table_clear (struct merged_table *mt); + +#endif
reftable/pq.c new file +114 -0
114 lines not on this page · raw message
reftable/pq.h new file +34 -0
@@ -0,0 +1,34 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef PQ_H +#define PQ_H + +#include "record.h" + +struct pq_entry { +int index; +struct record rec; +}; + +int pq_less (struct pq_entry a,struct pq_entry b); + +struct merged_iter_pqueue { +struct pq_entry *heap; +int len; +int cap; +}; + +struct pq_entry merged_iter_pqueue_top (struct merged_iter_pqueue pq); +bool merged_iter_pqueue_is_empty (struct merged_iter_pqueue pq); +void merged_iter_pqueue_check (struct merged_iter_pqueue pq); +struct pq_entry merged_iter_pqueue_remove (struct merged_iter_pqueue *pq); +void merged_iter_pqueue_add (struct merged_iter_pqueue *pq,struct pq_entry e); +void merged_iter_pqueue_clear (struct merged_iter_pqueue *pq); + +#endif
reftable/reader.c new file +720 -0
720 lines not on this page · raw message
reftable/reader.h new file +52 -0
52 lines not on this page · raw message
reftable/record.c new file +1119 -0
510 lines not on this page · raw message
reftable/record.h new file +79 -0
79 lines not on this page · raw message
reftable/reftable.h new file +409 -0
409 lines not on this page · raw message
reftable/slice.c new file +199 -0
199 lines not on this page · raw message
reftable/stack.c new file +1007 -0
1,007 lines not on this page · raw message
reftable/system.h new file +53 -0
53 lines not on this page · raw message
reftable/tree.c new file +66 -0
66 lines not on this page · raw message
reftable/writer.c new file +637 -0
637 lines not on this page · raw message
reftable/writer.h new file +45 -0
45 lines not on this page · raw message
reftable/zlib-compat.c new file +92 -0
92 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
gitgitgadget
Part of the diff is not shown here. View the raw message.
PATCH v7 6/6 Reftable support for git-core
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
For background, see the previous commit introducing the library.
TODO:
* Resolve spots marked with XXX
* Test strategy?
Example use: see t/t0031-reftable.sh
Signed-off-by Han-Wen Nienhuys <[email protected]>
Co-authored-by Jeff King <[email protected]>
--- .../technical/repository-version.txt | 7 + Makefile | 24 +- builtin/clone.c | 4 +- builtin/init-db.c | 55 +- cache.h | 4 +- refs.c | 20 +- refs.h | 3 + refs/refs-internal.h | 1 + refs/reftable-backend.c | 1015 +++++++++++++++++ repository.c | 2 + repository.h | 3 + setup.c | 12 +- t/t0031-reftable.sh | 31 + 13 files changed, 1152 insertions(+), 29 deletions(-) create mode 100644 refs/reftable-backend.c create mode 100755 t/t0031-reftable.sh
Documentation/technical/repository-version.txt +7 -0
@@ -100,3 +100,10 @@ If set, by default "git config" reads from both "config" and multiple working directory mode, "config" file is shared while "config.worktree" is per-working directory (i.e., it's in GIT_COMMON_DIR/worktrees/<id>/config.worktree) + +==== `refStorage` + +Specifies the file format for the ref database. Values are `files` +(for the traditional packed + loose ref format) and `reftable` for the +binary reftable format. See https://github.com/google/reftable for +more information.
Makefile +23 -1
@@ -814,6 +814,7 @@ TEST_SHELL_PATH = $(SHELL_PATH) LIB_FILE = libgit.a XDIFF_LIB = xdiff/lib.a VCSSVN_LIB = vcs-svn/lib.a +REFTABLE_LIB = reftable/libreftable.a GENERATED_H += command-list.h
@@ -959,6 +960,7 @@ LIB_OBJS += rebase-interactive.o LIB_OBJS += reflog-walk.o LIB_OBJS += refs.o LIB_OBJS += refs/files-backend.o +LIB_OBJS += refs/reftable-backend.o LIB_OBJS += refs/iterator.o LIB_OBJS += refs/packed-backend.o LIB_OBJS += refs/ref-cache.o
@@ -1163,7 +1165,7 @@ THIRD_PARTY_SOURCES += compat/regex/% THIRD_PARTY_SOURCES += sha1collisiondetection/% THIRD_PARTY_SOURCES += sha1dc/% -GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) +GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) EXTLIBS = GIT_USER_AGENT = git/$(GIT_VERSION)
@@ -2347,11 +2349,28 @@ VCSSVN_OBJS += vcs-svn/fast_export.o VCSSVN_OBJS += vcs-svn/svndiff.o VCSSVN_OBJS += vcs-svn/svndump.o +REFTABLE_OBJS += reftable/basics.o +REFTABLE_OBJS += reftable/block.o +REFTABLE_OBJS += reftable/bytes.o +REFTABLE_OBJS += reftable/file.o +REFTABLE_OBJS += reftable/iter.o +REFTABLE_OBJS += reftable/merged.o +REFTABLE_OBJS += reftable/pq.o +REFTABLE_OBJS += reftable/reader.o +REFTABLE_OBJS += reftable/record.o +REFTABLE_OBJS += reftable/slice.o +REFTABLE_OBJS += reftable/stack.o +REFTABLE_OBJS += reftable/tree.o +REFTABLE_OBJS += reftable/writer.o +REFTABLE_OBJS += reftable/zlib-compat.o + + TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) OBJECTS := $(LIB_OBJS) $(BUILTIN_OBJS) $(PROGRAM_OBJS) $(TEST_OBJS) \ $(XDIFF_OBJS) \ $(VCSSVN_OBJS) \ $(FUZZ_OBJS) \ + $(REFTABLE_OBJS) \ common-main.o \ git.o ifndef NO_CURL
@@ -2488,6 +2507,9 @@ $(XDIFF_LIB): $(XDIFF_OBJS) $(VCSSVN_LIB): $(VCSSVN_OBJS) $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ +$(REFTABLE_LIB): $(REFTABLE_OBJS) + $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ + export DEFAULT_EDITOR DEFAULT_PAGER Documentation/GIT-EXCLUDED-PROGRAMS: FORCE
builtin/clone.c +3 -1
@@ -1097,7 +1097,9 @@ int cmd_clone(int argc, const char **argv, const char *prefix) } } -init_db (git_dir, real_git_dir, option_template,INIT_DB_QUIET ); +init_db (git_dir, real_git_dir, option_template, +DEFAULT_REF_STORAGE ,/* XXX */ +INIT_DB_QUIET );if (real_git_dir) git_dir = real_git_dir;
builtin/init-db.c +40 -15
125 lines not on this page · raw message
cache.h +3 -1
@@ -627,7 +627,8 @@ int path_inside_repo(const char *prefix, const char *path);#define INIT_DB_EXIST_OK 0x0002int init_db (const char *git_dir,const char *real_git_dir, -const char *template_dir,unsigned int flags); +const char *template_dir,const char *ref_storage_format, +unsigned int flags);void sanitize_stdfds (void );int daemonize (void );
@@ -1041,6 +1042,7 @@ struct repository_format {int is_bare;int hash_algo;char *work_tree; +char *ref_storage;struct string_list unknown_extensions; };
refs.c +12 -8
@@ -20,7 +20,7 @@/* * List of all available backends */ -static struct ref_storage_be *refs_backends = &refs_be_files; +static struct ref_storage_be *refs_backends = &refs_be_reftable;static struct ref_storage_be *find_ref_storage_backend (const char *name) {
@@ -1836,13 +1836,13 @@ static struct ref_store *lookup_ref_store_map(struct hashmap *map,* Create, record, and return a ref_store instance for the specified * gitdir. */ -static struct ref_store *ref_store_init (const char *gitdir, +static struct ref_store *ref_store_init (const char *gitdir,const char *be_name,unsigned int flags) { -const char *be_name ="files" ; -struct ref_storage_be *be =find_ref_storage_backend (be_name); +struct ref_storage_be *be;struct ref_store *refs; + be =find_ref_storage_backend (be_name);if (!be)BUG ("reference backend %s is unknown" , be_name);
@@ -1858,7 +1858,10 @@ struct ref_store *get_main_ref_store(struct repository *r)if (!r->gitdir)BUG ("attempting to get main_ref_store outside of repository" ); - r->refs =ref_store_init (r->gitdir,REF_STORE_ALL_CAPS ); + r->refs =ref_store_init (r->gitdir, + r->ref_storage_format ? r->ref_storage_format : +DEFAULT_REF_STORAGE , +REF_STORE_ALL_CAPS );return r->refs; }
@@ -1913,7 +1916,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule) goto done;/* assume that add_submodule_odb() has been called */ - refs =ref_store_init (submodule_sb.buf, + refs =ref_store_init (submodule_sb.buf,DEFAULT_REF_STORAGE ,/* XXX */ REF_STORE_READ |REF_STORE_ODB );register_ref_store_map (&submodule_ref_stores,"submodule" , refs, submodule);
@@ -1927,6 +1930,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule)struct ref_store *get_worktree_ref_store (const struct worktree *wt) { +const char *format =DEFAULT_REF_STORAGE ;/* XXX */ struct ref_store *refs;const char *id;
@@ -1940,9 +1944,9 @@ struct ref_store *get_worktree_ref_store(const struct worktree *wt)if (wt->id) refs =ref_store_init (git_common_path ("worktrees/%s" , wt->id), -REF_STORE_ALL_CAPS ); + format,REF_STORE_ALL_CAPS );else - refs =ref_store_init (get_git_common_dir (), + refs =ref_store_init (get_git_common_dir (), format,REF_STORE_ALL_CAPS );if (refs)
refs.h +3 -0
@@ -9,6 +9,9 @@ struct string_list;struct string_list_item ;struct worktree ; +/* XXX where should this be? */ +#define DEFAULT_REF_STORAGE "files" +/* * Resolve a reference, recursively following symbolic refererences. *
refs/refs-internal.h +1 -0
@@ -661,6 +661,7 @@ struct ref_storage_be { };extern struct ref_storage_be refs_be_files; +extern struct ref_storage_be refs_be_reftable;extern struct ref_storage_be refs_be_packed;/*
refs/reftable-backend.c new file +1015 -0
1,015 lines not on this page · raw message
repository.c +2 -0
@@ -174,6 +174,8 @@ int repo_init(struct repository *repo,if (worktree)repo_set_worktree (repo, worktree); + repo->ref_storage_format =xstrdup_or_null (format.ref_storage); +clear_repository_format (&format);return 0 ;
repository.h +3 -0
@@ -70,6 +70,9 @@ struct repository {/* The store in which the refs are held. */ struct ref_store *refs; +/* The format to use for the ref database. */ +char *ref_storage_format; +/* * Contains path to often used file names. */
setup.c +9 -3
@@ -458,9 +458,11 @@ static int check_repo_format(const char *var, const char *value, void *vdata)if (!value)return config_error_nonbool (var); data->partial_clone =xstrdup (value); - }else if (!strcmp (ext,"worktreeconfig" )) + }else if (!strcmp (ext,"worktreeconfig" )) { data->worktree_config =git_config_bool (var, value); -else + }else if (!strcmp (ext,"refstorage" )) { + data->ref_storage =xstrdup (value); + }else string_list_append (&data->unknown_extensions, ext); }
@@ -549,6 +551,7 @@ void clear_repository_format(struct repository_format *format)string_list_clear (&format->unknown_extensions,0 );free (format->work_tree);free (format->partial_clone); +free (format->ref_storage);init_repository_format (format); }
@@ -1191,8 +1194,11 @@ const char *setup_git_directory_gently(int *nongit_ok) gitdir =DEFAULT_GIT_DIR_ENVIRONMENT ;setup_git_env (gitdir); } -if (startup_info->have_repository) +if (startup_info->have_repository) {repo_set_hash_algo (the_repository, repo_fmt.hash_algo); + the_repository->ref_storage_format = +xstrdup_or_null (repo_fmt.ref_storage); + } }strbuf_release (&dir);
t/t0031-reftable.sh new file +31 -0
@@ -0,0 +1,31 @@ +#!/bin/sh +# +# Copyright (c) 2020 Google LLC +# + +test_description='reftable basics' + +. ./test-lib.sh + +test_expect_success 'basic operation of reftable storage' ' + git init --ref-storage=reftable repo && + cd repo && + echo "hello" > world.txt && + git add world.txt && + git commit -m "first post" && + printf "HEAD\nrefs/heads/master\n" > want && + git show-ref | cut -f2 -d" " > got && + test_cmp got want && + for count in $(test_seq 1 10); do + echo "hello" >> world.txt + git commit -m "number ${count}" world.txt + done && + git gc && + nfiles=$(ls -1 .git/reftable | wc -l | tr -d "[ \t]" ) && + test "${nfiles}" = "2" && + git reflog refs/heads/master > output && + grep "commit (initial): first post" output && + grep "commit: number 10" output +' + +test_done
Signature · 1 line
gitgitgadget
Re: [PATCH v7 6/6] Reftable support for git-core
Quoted 7 lines · Han-Wen Nienhuys via GitGitGadget
t/t0031-reftable.sh new file +1
-0@@ -0,0 +1,31 @@ +#!/bin/sh
t/t0031-reftable.sh line 11
Quoted 10 lines
t/t0031-reftable.sh new file +10
-0+# +# Copyright (c) 2020 Google LLC +# + +test_description='reftable basics' + +. ./test-lib.sh + +test_expect_success 'basic operation of reftable storage' ' + git init --ref-storage=reftable repo &&
Test script bodies are supposed to be indented with HT, not 8 SPs.
t/t0031-reftable.sh line 12
Quoted 1 line
t/t0031-reftable.sh new file +1
-0+ cd repo &&
You are not supposed to chdir around inside tests, unless you do so in a subshell. i.e.
test_expect_success 'basic operation' ' git init --ref-storage=reftable repo && ( cd repo && ... ) '
This is because the next test somebody else will add after this one *will* start from inside that repo (but if and only if this step gets run---and there are ways to skip steps).
t/t0031-reftable.sh line 13
Quoted 1 line
t/t0031-reftable.sh new file +1
-0+ echo "hello" > world.txt &&
A shell redirection operator should immediately be followed by the filename without SP in between, i.e.
echo hello >world.txt &&
t/t0031-reftable.sh line 16
Quoted 3 lines
t/t0031-reftable.sh new file +3
-0+ git add world.txt && + git commit -m "first post" && + printf "HEAD\nrefs/heads/master\n" > want &&
There is an easier-to-read helper. Also, by convention, the file that holds the expected state is called "expect", while the file that records the actual state observed is called "actual".
test_write_lines HEAD refs/heads/master >expect &&
t/t0031-reftable.sh line 17
Quoted 1 line
t/t0031-reftable.sh new file +1
-0+ git show-ref | cut -f2 -d" " > got &&
Is the order of "show-ref" output guaranteed in some way? Otherwise we may need to make sure that we sort it to keep it stable.
Without --show-head, HEAD should not appear in "git show-ref" output, but the expected output has HEAD, which is puzzling.
We will not notice if a "git" command that placed on the left hand side of a pipeline segfaults after showing its output.
People often split a pipeline like this into separate command for that reason (but there is probably a better way in this case, as we see soon).
t/t0031-reftable.sh line 18
Quoted 1 line
t/t0031-reftable.sh new file +1
-0+ test_cmp got want &&
Taking the above comments together, perhaps
test_write_lines refs/heads/master >expect && git for-each-ref --sort=refname >actual && test_cmp expect actual &&
t/t0031-reftable.sh line 22
Quoted 4 lines
t/t0031-reftable.sh new file +4
-0+ for count in $(test_seq 1 10); do + echo "hello" >> world.txt + git commit -m "number ${count}" world.txt + done &&
Style: write "do" on its own line, aligned with "for" on the previous line. Same for "if/then/else/fi" "while/do/done" etc.
An easy way to remember is that you never use any semicolon in your shell script, except for the double-semicolon you use to delimit your case arms.
When any of these steps fail, you do not notice it, unless the failure happens to occur to "git commit" during the last round. I think you can use "return 1" to signal the failure to the test framework, like so.
for count in $(test_seq 1 10) do echo hello >>world.txt && git commit -m "number $count} world.txt || return 1 done &&
t/t0031-reftable.sh line 25
Quoted 3 lines
t/t0031-reftable.sh new file +3
-0+ git gc && + nfiles=$(ls -1 .git/reftable | wc -l | tr -d "[ \t]" ) && + test "${nfiles}" = "2" &&
No need to pipe "wc -l" output to "tr" to strip spaces. Just stop double-quoting it on the using side, and even better is to use test_line_count. Perhaps
ls -1 .git/reftable >files && test_line_count = 2 files
t/t0031-reftable.sh line 28
Quoted 3 lines
t/t0031-reftable.sh new file +3
-0+ git reflog refs/heads/master > output && + grep "commit (initial): first post" output && + grep "commit: number 10" output
Do we give guarantees that we keep all the reflog entries? Would it help to count the number of lines in the output file here?
Quoted 3 lines
t/t0031-reftable.sh new file +3
-0+' + +test_done
Re: [PATCH v7 6/6] Reftable support for git-core
Quoted 5 lines · Junio C Hamano
t/t0031-reftable.sh
Quoted 5 lines
t/t0031-reftable.sh new file +0
-0
Thanks for your careful review. Should we not start with the C code, though? I've written the reftable library to my best ability, but it hasn't been peer reviewed yet.
Quoted 4 lines
t/t0031-reftable.sh line 11
Quoted 2 lines
t/t0031-reftable.sh new file +2
-0+test_expect_success 'basic operation of reftable storage' ' + git init --ref-storage=reftable repo &&Test script bodies are supposed to be indented with HT, not 8 SPs.
Done
Quoted 9 lines
Quoted 1 line
+ cd repo &&
You are not supposed to chdir around inside tests, unless you do so in a subshell. i.e.
test_expect_success 'basic operation' ' git init --ref-storage=reftable repo && ( cd repo &&
Done.
Quoted 4 lines
Quoted 1 line
+ echo "hello" > world.txt &&
A shell redirection operator should immediately be followed by the filename without SP in between, i.e.
Done.
Quoted 9 lines
echo hello >world.txt &&t/t0031-reftable.sh line 16
Quoted 3 lines
t/t0031-reftable.sh new file +3
-0+ git add world.txt && + git commit -m "first post" && + printf "HEAD\nrefs/heads/master\n" > want &&There is an easier-to-read helper. Also, by convention, the file that holds the expected state is called "expect", while the file that records the actual state observed is called "actual".
Done
Quoted 6 lines
test_write_lines HEAD refs/heads/master >expect &&Quoted 1 line
+ git show-ref | cut -f2 -d" " > got &&
Is the order of "show-ref" output guaranteed in some way? Otherwise we may need to make sure that we sort it to keep it stable.
The ref iteration result is lexicographically ordered.
Quoted 2 lines
Without --show-head, HEAD should not appear in "git show-ref" output, but the expected output has HEAD, which is puzzling.
The HEAD symref is stored in reftable too, and to the reftable code it's just another ref. How should we refactor this? Shouldn't the files ref backend produce "HEAD" and then it's up to the show-ref builtin to filter it out?
Quoted 15 lines
We will not notice if a "git" command that placed on the left hand side of a pipeline segfaults after showing its output.
People often split a pipeline like this into separate command for that reason (but there is probably a better way in this case, as we see soon).
Quoted 1 line
+
test_cmpgot want &&Taking the above comments together, perhaps
test_write_lines refs/heads/master >expect && git for-each-ref --sort=refname >actual && test_cmp expect actual &&
Quoted 7 lines
t/t0031-reftable.sh line 22
Quoted 4 lines
t/t0031-reftable.sh new file +4
-0+ for count in $(test_seq 1 10); do + echo "hello" >> world.txt + git commit -m "number ${count}" world.txt + done &&Style: write "do" on its own line, aligned with "for" on the previous line. Same for "if/then/else/fi" "while/do/done" etc.
Done.
Quoted 9 lines
An easy way to remember is that you never use any semicolon in your shell script, except for the double-semicolon you use to delimit your case arms.
When any of these steps fail, you do not notice it, unless the failure happens to occur to "git commit" during the last round. I think you can use "return 1" to signal the failure to the test framework, like so.
Quoted 6 lines
for count in $(test_seq 1 10)
do
echo hello >>world.txt &&
git commit -m "number $count} world.txt ||
return 1
done &&Done.
Quoted 3 lines
ls -1 .git/reftable >files &&
test_line_count= 2 files
Done.
Quoted 6 lines
t/t0031-reftable.sh line 28
Quoted 3 lines
t/t0031-reftable.sh new file +3
-0+ git reflog refs/heads/master > output && + grep "commit (initial): first post" output && + grep "commit: number 10" outputDo we give guarantees that we keep all the reflog entries? Would it help to count the number of lines in the output file here?
The reflog entries are only removed if someone calls into the ref backend to expire them. Does that happen automatically?
Signature · 11 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Re: [PATCH v7 6/6] Reftable support for git-core
Quoted 10 lines · Han-Wen Nienhuys
Quoted 5 lines · Junio C Hamano
t/t0031-reftable.sh
Quoted 5 lines
t/t0031-reftable.sh new file +0
-0Thanks for your careful review. Should we not start with the C code, though? I've written the reftable library to my best ability, but it hasn't been peer reviewed yet.
My intention was to let others, especially those from outside Google, to comment before I do ;-)
Quoted 9 lines
Quoted 6 lines
test_write_lines HEAD refs/heads/master >expect &&Quoted 1 line
+ git show-ref | cut -f2 -d" " > got &&
Is the order of "show-ref" output guaranteed in some way? Otherwise we may need to make sure that we sort it to keep it stable.
The ref iteration result is lexicographically ordered.
Quoted 7 lines
Quoted 2 lines
Without --show-head, HEAD should not appear in "git show-ref" output, but the expected output has HEAD, which is puzzling.
The HEAD symref is stored in reftable too, and to the reftable code it's just another ref. How should we refactor this? Shouldn't the files ref backend produce "HEAD" and then it's up to the show-ref builtin to filter it out?
After looking at show-ref.c, the answer is obvious, I would think. for_each_ref() MUST NOT include HEAD in its enumeration, and that is why the caller of for_each_ref() makes a separate call to head_ref() when it wants to talk about HEAD. As a backend-agnostic abstraction layer, behaviour of functions like for_each_ref() should not change depending on what ref backend is in use.
Quoted 2 lines
The reflog entries are only removed if someone calls into the ref backend to expire them. Does that happen automatically?
"gc auto" may kick in in theory but not likely in these tests, so it would be safe and sensible to know and check the expected number of reflog entries in this test, I guess.
Thanks.
Re: [PATCH v7 6/6] Reftable support for git-core
Quoted 8 lines · Junio C Hamano
Quoted 7 lines
Quoted 2 lines
Without --show-head, HEAD should not appear in "git show-ref" output, but the expected output has HEAD, which is puzzling.
The HEAD symref is stored in reftable too, and to the reftable code it's just another ref. How should we refactor this? Shouldn't the files ref backend produce "HEAD" and then it's up to the show-ref builtin to filter it out?
Quoted 6 lines
After looking at
show-ref.c, the answer is obvious, I would think.for_each_ref()MUST NOT include HEAD in its enumeration, and that is why the caller offor_each_ref()makes a separate call tohead_ref()when it wants to talk about HEAD. As a backend-agnostic abstraction layer, behaviour of functions likefor_each_ref()should not change depending on what ref backend is in use.
So, the ref backend should manage the HEAD ref, but iteration should not produce the HEAD ref.
Are there other refs that behave like this, or is this the only special case?
Signature · 11 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Re: [PATCH v7 6/6] Reftable support for git-core
Quoted 2 lines · Han-Wen Nienhuys
So, the ref backend should manage the HEAD ref, but iteration should not produce the HEAD ref.
Yeah, as there is a dedicated API function head_ref().
Things like ORIG_HEAD and MERGE_HEAD have always been curiosity outside the official API, but IIRC read_ref() and friends are prepared to read them [*1*], so the vocabulary may be unbounded [*2*].
These won't be listed in for_each_ref() iteration, either (think of for_each_ref() as a filtered "ls -R .git/refs/" output).
Thanks.
[Footnotes]
*1* I do not offhand know if these *_HEAD pseudo-refs are written via the refs API---it probably is the safest to arrange the ref backends in such a way that they always go to the files backend, even if HEAD and refs/* are managed by a backend different from the files one.
*2* I vaguely recall that back when ref-backend infrastructure was introduced, we had discussion on declaring that "^[A-Z_]*HEAD$" directly under $GIT_DIR/ are these special refs (and nothing else directly under $GIT_DIR/ is). I do not recall what happend to the discussion, though other folks may.
Re: [PATCH v7 6/6] Reftable support for git-core
Quoted 6 lines · Junio C Hamano
Quoted 2 lines · Han-Wen Nienhuys
So, the ref backend should manage the HEAD ref, but iteration should not produce the HEAD ref.
Quoted 8 lines
Yeah, as there is a dedicated API function
head_ref().Things like
ORIG_HEADandMERGE_HEADhave always been curiosity outside the official API, but IIRCread_ref()and friends are prepared to read them [*1*], so the vocabulary may be unbounded [*2*].These won't be listed in
for_each_ref()iteration, either (think offor_each_ref()as a filtered "ls -R .git/refs/" output).
currently the code says
int refs_for_each_ref(struct ref_store *refs, each_ref_fn fn, void *cb_data) {
return do_for_each_ref(refs, "", fn, 0, 0, cb_data);
}
but it looks like this should do
return do_for_each_ref(refs, "refs/", fn, 0, 0, cb_data);
instead.
Signature · 7 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Re: [PATCH v7 6/6] Reftable support for git-core
In reply to Han-Wen Nienhuys via GitGitGadgetbuiltin/init-db.c static void separate_git_dir(const char *git_dir, const char *git_link) · line 359
Quoted 5 lines · Han-Wen Nienhuys via GitGitGadget
int init_db(const char
*git_dir, const char*real_git_dir,builtin/init-db.c +2
-1- const char *template_dir, unsigned int flags) + const char *template_dir, const char *ref_storage_format, + unsigned int flags) {
This change, and Brian's SHA-256 work, obviously will introduce a conflict as the other side wants to add an extra parameter to choose from different hash algorhtims.
I wonder if we should rather add a single pointer to a struct that can hold various initialization options as an extra parameter. That way, each topic can add and manage its own new member in the struct.
builtin/init-db.c static int needs_work_tree_config(const char *git_dir, const char *work_tree) · line 181
Quoted 4 lines
static int create_default_files(const char
*template_path,builtin/init-db.c +2
-1- const char *original_git_dir) +const char *original_git_dir, + const char *ref_storage_format, int flags)
Pretty much the same story here, too. The other side aims to be more generic and passes a "struct repository_format *" as an extra parameter. I think the repository-format structure needs to have what ref backend is in use, so instead of passing only the string like this patch, we can use the appropriate member from the struct?
Please try to apply this series to 'master', and try to merge the result with the tip of 'pu', to see if there are other areas that a better coordination between you and Brian would help moving these two topics together and work well with each other.
I am not sure how quickly Brian's SHA-256 work solidifies enough to build on top of, but if it is a good option to build on top of the topic, that may save some work from this topic, too, as the mechanism to choose something (i.e. hash algorithm for Brian's topic, ref backend for this topic) fundamental to the repository at runtime and at initialization time needs similar supporting infrastructure, such as changes in "[12/24] setup: allow check_repository_format to read repository format", and "[13/24] builtin/init-db: allow specifying hash algorithm on command line" of that series.
Thanks.
Re: [PATCH v7 6/6] Reftable support for git-core
Quoted 8 lines · Junio C Hamano
This change, and Brian's SHA-256 work, obviously will introduce a conflict as the other side wants to add an extra parameter to choose from different hash algorhtims.
I wonder if we should rather add a single pointer to a struct that can hold various initialization options as an extra parameter. That way, each topic can add and manage its own new member in the struct.
Quoted 8 lines
builtin/init-db.c static int needs_work_tree_config(const char *git_dir, const char *work_tree) · line 181
Quoted 4 lines
static int create_default_files(const char
*template_path,builtin/init-db.c +2
-1- const char *original_git_dir) +const char *original_git_dir, + const char *ref_storage_format, int flags)Pretty much the same story here, too. The other side aims to be more generic and passes a "struct
repository_format*" as an extra parameter.
Sounds great to me. Could we submit a patch that changes this into master already so Brian and can work on the same basis?
Quoted 10 lines
I am not sure how quickly Brian's SHA-256 work solidifies enough to build on top of, but if it is a good option to build on top of the topic, that may save some work from this topic, too, as the mechanism to choose something (i.e. hash algorithm for Brian's topic, ref backend for this topic) fundamental to the repository at runtime and at initialization time needs similar supporting infrastructure, such as changes in "[12/24] setup: allow
check_repository_formatto read repository format", and "[13/24] builtin/init-db: allow specifying hash algorithm on command line" of that series.
Can I pull this series as a git branch somewhere?
Signature · 11 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Re: [PATCH v7 6/6] Reftable support for git-core
Quoted 2 lines · Han-Wen Nienhuys
Sounds great to me. Could we submit a patch that changes this into master already so Brian and can work on the same basis?
Heh, "into master already" is never be the way things work around here. But if these two topics can agree the best way forward, it can be proposed, polished and then merged to 'master', just like any other topic. And the two topics both can build on top of that topic.
Quoted 11 lines
Quoted 10 lines
I am not sure how quickly Brian's SHA-256 work solidifies enough to build on top of, but if it is a good option to build on top of the topic, that may save some work from this topic, too, as the mechanism to choose something (i.e. hash algorithm for Brian's topic, ref backend for this topic) fundamental to the repository at runtime and at initialization time needs similar supporting infrastructure, such as changes in "[12/24] setup: allow
check_repository_formatto read repository format", and "[13/24] builtin/init-db: allow specifying hash algorithm on command line" of that series.
Quoted 1 line
Can I pull this series as a git branch somewhere?
There is a single merge somewhere on "git log --first-parent master..pu" that merges bc/sha-256- something (I assume you know how to obtain 'pu').
Thanks.
Quoted 1 line · Han-Wen Nienhuys via GitGitGadget
This adds the reftable library, and hooks it up as a ref backend.
I do not see the promised .gitattributes addition; please add one early in the series, before the step in which you start adding files that want different whitespace rules, so that I do not have to apply with "git am --whitespace=nowarn".
It is rather unfortunate that you included Jonathan's reftable doc patch in the series---it has separately been queued and merged already in 'next'. I'll fork hn/reftable on top of the reftable doc topic of Jonathan's, while dropping the extra copy [4/6] in this series, to cope with this (an alternative would be to revert the one in 'next' and take this series as-is).
Quoted 1 line
* spots marked with XXX in the Git source code.
What follows is my notes while "git diff master...hn/reftable" and looking for those lines, except for the ones in the reftable implementation proper (for which you would be a better judge).
The init-db call in clone hardcodes DEFAULT_REF_STORAGE and that should be OK at least for now. Eventually the code would want to read from the system-wide and the per-user configuration files to figure it out, but I do not think the config subsystem is prepared to read the configuration before making this call to init_db() right now---let's leave such a complication until after the basics is done. After all, "git init && git add remote && git pull" can be used as a rough approximate for "git clone"; as long as "git init" can be told what backend to use, that would be sufficient to get us going.
It sounds a bit too magical to convert the repository by running "git init" again on an already existing repository. But a user who runs "git init" in an existing repository and asks to use settings that are different from what the repository uses from the command line cannot be expecting the command to error out when the settings do not match (i.e. as a mechanism to check the setting)---the only sensible way to interpret such an invocation is as a request to convert the existing repository. Again, I do not think it is a huge loss to do without such an automatic conversion in the initial deployment.
In refs.c, I am puzzled by the fact that a call to ref_store_init() in the get_submodule_ref_store() needs to pass DEFAULT_REF_STORAGE, or any format for that matter, at all. Wouldn't the gitdir argument sufficient for ref_store_init() to learn what backend the repository uses? IOW, I am not sure why the caller needs to pass the backend format.
The way r->ref_storage_format is used by get_main_ref_store(), at least when the field is set, looks a lot more sensible. Would it be possible that get_submodule_ref_store() function would also want to have, not just a submodule as a mere "string", but an instance of a repository for that submodule?
Speaking of the use of ref_store_init() and r->ref_storage_format in get_main_ref_store(), when is the storage_format field unspecified? Is there a chicken-and-egg problem around here?
The same comments apply to the use of the default format in get_worktree_ref_store(), but do we foresee to "add" a worktree that uses one ref backend off of a repository that uses a different ref backend? Even if we do, it probably makes sense to default to the same format as the primary worktree uses, I would think. It is still unclear to me why ref_store_init() wants the format in the first place, though, given that it takes a path to the .git/ directory (or its equivalent), and ref_store_init() should be able to figuire it out without being told.
I think it is fine to define the textual name of the default ref storage in refs.h
Thanks.
Hi!
Quoted 8 lines · Han-Wen Nienhuys
This adds the reftable library, and hooks it up as a ref backend.
Feedback wanted:
* spots marked with XXX in the Git source code.
* what is a good test strategy? Could we have a CI flavor where we flip the default to reftable, and see how it fares?
[...]
Quoted 9 lines
Han-Wen Nienhuys (5):
refs.h: clarify reflog iteration order
create .git/refs infiles-backend.c
refs: document howref_iterator_advance_fnshould handle symrefs Add reftable library
Reftable support for git-coreJonathan Nieder (1):
reftable: file format documentation
[...]
Quoted 1 line
51 files changed, 8547 insertions(+), 32 deletions(-)
This series was discussed at the Git Contributor Summit, and I promised to provide a summary here. Sorry for the delay.
Contribution format
~~~~~~~~~~~~~~~~~~~
This is a bit of an unusual contribution to Git:
- the heart of the series is in a single commit
- it is under a different license than most of Git
- the code doesn't use the same dialect as and "look like" the rest of Git
All three of these are because the code is meant to be a standalone library that can be used by Git, libgit2, and any other project that wants to understand reftables. But they create obstacles to reviewing (one reviewer said he had spent a two-hour block looking at the patch and not made much headway).
On the other hand, people mostly agreed that having the ability to share this code with libgit2 is beneficial. So how can we get a substantial review without losing that benefit?
1. Most importantly, people would like a series of multiple patches that tell a story. If review of an early patch in the series reveals significant changes that should happen in reftable upstream, that's fine --- upstream, that can happen in a patch on top, whereas in the series being contributed, the patch would be amended.
In the end, there would be two different series of commits: the contributed series, and the upstream history. The upstream history might be messier, and that's fine. The contributed series would be applied to git.git.
If I understood correctly, this is more important for the initial contribution than for later changes. Git includes some code (e.g. in compat/) with outside upstreams and gets all-in-one-commit updates. As long as upstream has a functional review process, this is fine --- here, reftable upstream (you :)) wants review on the initial contribution and splitting into a series that tells a story is the cost of making that review feasible to provide.
2. Both Git and libgit2 have abstractions like strbuf. We'd like reftable to make use of similar abstractions where they make the code cleaner.
3. libgit2 has a git_malloc allocator. reftable doesn't necessarily have to use it or make it pluggable --- at worst, we can #define malloc to use it. But it's something to be aware of.
Maintenance
~~~~~~~~~~~
There's some interest in the maintenance story: if we run into an issue with the reftable library, do we report and fix it on the git mailing list or does the reftable library have its own upstream forums for that?
Portability
~~~~~~~~~~~
There was some confusion about the scope of what the reftable library provides. It provides a reader and a writer and the caller is responsible for connecting those to files.
There were some questions about mmap usage here (there is none) and whether the library needs some kind of seeking reader interface for abstracting the OS interface to files.
I think the upshot is
4. Some summary documentation would be helpful e.g. in the README.md, to point people to what header file a person should start with to understand the library
5. People are also interested in the file-oriented part of reftable, even though this library doesn't implement it (that's patch 6, which is Git-specific).
Testing
~~~~~~~
Git's testsuite has various GIT_TEST_* knobs. A GIT_TEST_REF_BACKEND knob would be helpful, to allow running the full testsuite with reftable. That's a good way to suss out edge cases.
The testsuite should *also* include some specific tests designed for reftable. They can demonstrate edge cases and demonstrate some of the benefits
- case sensitivity
- handling of directory/file conflicts
That provides another entry point for people to understand reftable.
We also discussed table-driven tests that can be shared between implementations but didn't end up saying anything too concrete about that.
Summary
~~~~~~~
Breaking down the library into multiple patches is likely to be a signficant amount of work, but it's a good way to break the review into manageable pieces that get the project engaged.
Integration testing is also likely to be helpful.
People were very excited about the benefits reftable brings and it feels a little strange to ask so much when you've already written this code, but this seemed like the best way to make it into something the project understands well and to get any kinks ironed out. I'm happy to help in any way I can.
Thoughts?
Thanks,
Jonathan
v8 cover Reftable support git-core
In reply to Han-Wen Nienhuys via GitGitGadgetThis adds the reftable library, and hooks it up as a ref backend.
Feedback wanted:
* spots marked with XXX in the Git source code.
* what is a good test strategy? Could we have a CI flavor where we flip the default to reftable, and see how it fares?
v8
* propagate errors to git.
* discard empty tables in the stack.
* one very basic test (t0031.sh)
v9
* Added spec as technical doc.
* Use 4-byte hash ID as API. Storage format for SHA256 is still pending discussion.
v10
* Serialize 4 byte hash ID to the version-2 format
v11
* use reftable_ namespace prefix.
* transaction API to provide locking.
* reorganized record.{h,c}. More doc in reftable.h
* support log record deletion.
* pluggable malloc/free
Han-Wen Nienhuys (8):
refs.h: clarify reflog iteration order
create .git/refs in files-backend.c
refs: document how ref_iterator_advance_fn should handle symrefs Add .gitattributes for the reftable/ directory
reftable: define version 2 of the spec to accomodate SHA256
reftable: clarify how empty tables should be written Add reftable library
Reftable support for git-core
Jonathan Nieder (1):
reftable: file format documentation
Documentation/Makefile | 1 + Documentation/technical/reftable.txt | 1080 ++++++++++++++++ .../technical/repository-version.txt | 7 + Makefile | 24 +- builtin/clone.c | 5 +- builtin/init-db.c | 51 +- cache.h | 4 +- refs.c | 20 +- refs.h | 7 +- refs/files-backend.c | 6 + refs/refs-internal.h | 6 + refs/reftable-backend.c | 1002 +++++++++++++++ reftable/.gitattributes | 1 + reftable/LICENSE | 31 + reftable/README.md | 11 + reftable/VERSION | 5 + reftable/basics.c | 194 +++ reftable/basics.h | 35 + reftable/block.c | 433 +++++++ reftable/block.h | 76 ++ reftable/blocksource.h | 22 + reftable/bytes.c | 0 reftable/config.h | 1 + reftable/constants.h | 21 + reftable/dump.c | 97 ++ reftable/file.c | 98 ++ reftable/iter.c | 234 ++++ reftable/iter.h | 56 + reftable/merged.c | 299 +++++ reftable/merged.h | 34 + reftable/pq.c | 115 ++ reftable/pq.h | 34 + reftable/reader.c | 756 +++++++++++ reftable/reader.h | 60 + reftable/record.c | 1098 ++++++++++++++++ reftable/record.h | 106 ++ reftable/reftable.h | 520 ++++++++ reftable/slice.c | 219 ++++ reftable/slice.h | 42 + reftable/stack.c | 1132 +++++++++++++++++ reftable/stack.h | 42 + reftable/system.h | 53 + reftable/tree.c | 67 + reftable/tree.h | 24 + reftable/update.sh | 14 + reftable/writer.c | 653 ++++++++++ reftable/writer.h | 45 + reftable/zlib-compat.c | 92 ++ repository.c | 2 + repository.h | 3 + setup.c | 12 +- t/t0031-reftable.sh | 35 + 52 files changed, 8954 insertions(+), 31 deletions(-) create mode 100644 Documentation/technical/reftable.txt create mode 100644 refs/reftable-backend.c create mode 100644 reftable/.gitattributes create mode 100644 reftable/LICENSE create mode 100644 reftable/README.md create mode 100644 reftable/VERSION create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/blocksource.h create mode 100644 reftable/bytes.c create mode 100644 reftable/config.h create mode 100644 reftable/constants.h create mode 100644 reftable/dump.c create mode 100644 reftable/file.c create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/reftable.h create mode 100644 reftable/slice.c create mode 100644 reftable/slice.h create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/system.h create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100644 reftable/update.sh create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h create mode 100644 reftable/zlib-compat.c create mode 100755 t/t0031-reftable.sh
base-commit: 9fadedd637b312089337d73c3ed8447e9f0aa775
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-539%2Fhanwen%2Freftable-v8
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-539/hanwen/reftable-v8
Pull-Request: https://github.com/gitgitgadget/git/pull/539
Range-diff vs v7:
1: b1e44bc431e = 1: 05634d26dd5 refs.h: clarify reflog iteration order
2: b68488a095e = 2: 8d34e2c5c8b create .git/refs in files-backend.c
3: da538ef7421 = 3: d08f823844d refs: document how ref_iterator_advance_fn should handle symrefs
-: ----------- > 4: ca95b3a371e Add .gitattributes for the reftable/ directory
4: aae26814983 = 5: dfc8b131294 reftable: file format documentation
-: ----------- > 6: 45ab5361750 reftable: define version 2 of the spec to accomodate SHA256
-: ----------- > 7: 67ee5962e85 reftable: clarify how empty tables should be written
5: 30ed43a4fdb ! 8: 4f9bdd7312b Add reftable library
@@ -5,21 +5,21 @@
Reftable is a format for storing the ref database. Its rationale and
specification is in the preceding commit.
- Further context and motivation can be found in background reading:
+ This imports the upstream library as one big commit. For understanding
+ the code, it is suggested to read the code in the following order:
- * Original discussion on JGit-dev: https://www.eclipse.org/lists/jgit-dev/msg03389.html
+ * The specification under Documentation/technical/reftable.txt
- * First design discussion on git@vger: https://public-inbox.org/git/CAJo=hJtTp2eA3z9wW9cHo-nA7kK40vVThqh6inXpbCcqfdMP9g@mail.gmail.com/
+ * reftable.h - the public API
- * Last design discussion on git@vger: https://public-inbox.org/git/CAJo=hJsZcAM9sipdVr7TMD-FD2V2W6_pvMQ791EGCDsDkQ033w@mail.gmail.com/
+ * block.{c,h} - reading and writing blocks.
- * First attempt at implementation: https://public-inbox.org/git/CAP8UFD0PPZSjBnxCA7ez91vBuatcHKQ+JUWvTD1iHcXzPBjPBg@mail.gmail.com/
+ * writer.{c,h} - writing a complete reftable file.
- * libgit2 support issue: https://github.com/libgit2/libgit2/issues
+ * merged.{c,h} and pq.{c,h} - reading a stack of reftables
- * GitLab support issue: https://gitlab.com/gitlab-org/git/issues/6
-
- * go-git support issue: https://github.com/src-d/go-git/issues/1059
+ * stack.{c,h} - writing and compacting stacks of reftable on the
+ filesystem.
Signed-off-by: Han-Wen Nienhuys <[email protected]>
@@ -82,11 +82,11 @@
--- /dev/null
+++ b/reftable/VERSION
@@
-+commit ccce3b3eb763e79b23a3b4677d65ecceb4155ba3
++commit 920f7413f7a25f308d9a203f584873f6753192ec
+Author: Han-Wen Nienhuys <[email protected]>
-+Date: Tue Feb 25 20:42:29 2020 +0100
++Date: Wed Apr 1 12:41:41 2020 +0200
+
-+ C: rephrase the hash ID API in terms of a uint32_t
++ C: remove stray debug printfs
diff --git a/reftable/basics.c b/reftable/basics.c
new file mode 100644
@@ -165,10 +165,10 @@
+ return;
+ }
+ while (*p) {
-+ free(*p);
++ reftable_free(*p);
+ p++;
+ }
-+ free(a);
++ reftable_free(a);
+}
+
+int names_length(char **names)
@@ -201,8 +201,8 @@
+ if (p < next) {
+ if (names_len == names_cap) {
+ names_cap = 2 * names_cap + 1;
-+ names = realloc(names,
-+ names_cap * sizeof(char *));
++ names = reftable_realloc(
++ names, names_cap * sizeof(char *));
+ }
+ names[names_len++] = xstrdup(p);
+ }
@@ -211,7 +211,7 @@
+
+ if (names_len == names_cap) {
+ names_cap = 2 * names_cap + 1;
-+ names = realloc(names, names_cap * sizeof(char *));
++ names = reftable_realloc(names, names_cap * sizeof(char *));
+ }
+
+ names[names_len] = NULL;
@@ -232,7 +232,7 @@
+ return *a == *b;
+}
+
-+const char *error_str(int err)
++const char *reftable_error_str(int err)
+{
+ switch (err) {
+ case IO_ERROR:
@@ -252,6 +252,40 @@
+ default:
+ return "unknown error code";
+ }
++}
++
++void *(*reftable_malloc_ptr)(size_t sz) = &malloc;
++void *(*reftable_realloc_ptr)(void *, size_t) = &realloc;
++void (*reftable_free_ptr)(void *) = &free;
++
++void *reftable_malloc(size_t sz)
++{
++ return (*reftable_malloc_ptr)(sz);
++}
++
++void *reftable_realloc(void *p, size_t sz)
++{
++ return (*reftable_realloc_ptr)(p, sz);
++}
++
++void reftable_free(void *p)
++{
++ reftable_free_ptr(p);
++}
++
++void *reftable_calloc(size_t sz)
++{
++ void *p = reftable_malloc(sz);
++ memset(p, 0, sz);
++ return p;
++}
++
++void reftable_set_alloc(void *(*malloc)(size_t),
++ void *(*realloc)(void *, size_t), void (*free)(void *))
++{
++ reftable_malloc_ptr = malloc;
++ reftable_realloc_ptr = realloc;
++ reftable_free_ptr = free;
+}
diff --git a/reftable/basics.h b/reftable/basics.h
@@ -288,6 +322,11 @@
+int names_equal(char **a, char **b);
+int names_length(char **names);
+
++void *reftable_malloc(size_t sz);
++void *reftable_realloc(void *p, size_t sz);
++void reftable_free(void *p);
++void *reftable_calloc(size_t sz);
++
+#endif
diff --git a/reftable/block.c b/reftable/block.c
@@ -313,10 +352,32 @@
+#include "reftable.h"
+#include "zlib.h"
+
-+int block_writer_register_restart(struct block_writer *w, int n, bool restart,
-+ struct slice key);
++int header_size(int version)
++{
++ switch (version) {
++ case 1:
++ return 24;
++ case 2:
++ return 28;
++ }
++ abort();
++}
++
++int footer_size(int version)
++{
++ switch (version) {
++ case 1:
++ return 68;
++ case 2:
++ return 72;
++ }
++ abort();
++}
++
++int block_writer_register_restart(struct reftable_block_writer *w, int n,
++ bool restart, struct slice key);
+
-+void block_writer_init(struct block_writer *bw, byte typ, byte *buf,
++void block_writer_init(struct reftable_block_writer *bw, byte typ, byte *buf,
+ uint32_t block_size, uint32_t header_off, int hash_size)
+{
+ bw->buf = buf;
@@ -329,14 +390,14 @@
+ bw->entries = 0;
+}
+
-+byte block_writer_type(struct block_writer *bw)
++byte block_writer_type(struct reftable_block_writer *bw)
+{
+ return bw->buf[bw->header_off];
+}
+
+/* adds the record to the block. Returns -1 if it does not fit, 0 on
+ success */
-+int block_writer_add(struct block_writer *w, struct record rec)
++int block_writer_add(struct reftable_block_writer *w, struct record rec)
+{
+ struct slice empty = { 0 };
+ struct slice last = w->entries % w->restart_interval == 0 ? empty :
@@ -357,32 +418,29 @@
+ if (n < 0) {
+ goto err;
+ }
-+ out.buf += n;
-+ out.len -= n;
++ slice_consume(&out, n);
+
+ n = record_encode(rec, out, w->hash_size);
+ if (n < 0) {
+ goto err;
+ }
-+
-+ out.buf += n;
-+ out.len -= n;
++ slice_consume(&out, n);
+
+ if (block_writer_register_restart(w, start.len - out.len, restart,
+ key) < 0) {
+ goto err;
+ }
+
-+ free(slice_yield(&key));
++ reftable_free(slice_yield(&key));
+ return 0;
+
+err:
-+ free(slice_yield(&key));
++ reftable_free(slice_yield(&key));
+ return -1;
+}
+
-+int block_writer_register_restart(struct block_writer *w, int n, bool restart,
-+ struct slice key)
++int block_writer_register_restart(struct reftable_block_writer *w, int n,
++ bool restart, struct slice key)
+{
+ int rlen = w->restart_len;
+ if (rlen >= MAX_RESTARTS) {
@@ -398,7 +456,7 @@
+ if (restart) {
+ if (w->restart_len == w->restart_cap) {
+ w->restart_cap = w->restart_cap * 2 + 1;
-+ w->restarts = realloc(
++ w->restarts = reftable_realloc(
+ w->restarts, sizeof(uint32_t) * w->restart_cap);
+ }
+
@@ -411,7 +469,7 @@
+ return 0;
+}
+
-+int block_writer_finish(struct block_writer *w)
++int block_writer_finish(struct reftable_block_writer *w)
+{
+ int i = 0;
+ for (i = 0; i < w->restart_len; i++) {
@@ -442,7 +500,7 @@
+ }
+
+ if (Z_OK != zresult) {
-+ free(slice_yield(&compressed));
++ reftable_free(slice_yield(&compressed));
+ return ZLIB_ERROR;
+ }
+
@@ -455,14 +513,14 @@
+ return w->next;
+}
+
-+byte block_reader_type(struct block_reader *r)
++byte block_reader_type(struct reftable_block_reader *r)
+{
+ return r->block.data[r->header_off];
+}
+
-+int block_reader_init(struct block_reader *br, struct block *block,
-+ uint32_t header_off, uint32_t table_block_size,
-+ int hash_size)
++int block_reader_init(struct reftable_block_reader *br,
++ struct reftable_block *block, uint32_t header_off,
++ uint32_t table_block_size, int hash_size)
+{
+ uint32_t full_block_size = table_block_size;
+ byte typ = block->data[header_off];
@@ -485,7 +543,7 @@
+ uncompressed.buf + block_header_skip,
+ &dst_len, block->data + block_header_skip,
+ &src_len)) {
-+ free(slice_yield(&uncompressed));
++ reftable_free(slice_yield(&uncompressed));
+ return ZLIB_ERROR;
+ }
+
@@ -526,12 +584,14 @@
+ return 0;
+}
+
-+static uint32_t block_reader_restart_offset(struct block_reader *br, int i)
++static uint32_t block_reader_restart_offset(struct reftable_block_reader *br,
++ int i)
+{
+ return get_be24(br->restart_bytes + 3 * i);
+}
+
-+void block_reader_start(struct block_reader *br, struct block_iter *it)
++void block_reader_start(struct reftable_block_reader *br,
++ struct reftable_block_iter *it)
+{
+ it->br = br;
+ slice_resize(&it->last_key, 0);
@@ -541,7 +601,7 @@
+struct restart_find_args {
+ int error;
+ struct slice key;
-+ struct block_reader *r;
++ struct reftable_block_reader *r;
+};
+
+static int restart_key_less(int idx, void *args)
@@ -566,12 +626,13 @@
+
+ {
+ int result = slice_compare(a->key, rkey);
-+ free(slice_yield(&rkey));
++ reftable_free(slice_yield(&rkey));
+ return result;
+ }
+}
+
-+void block_iter_copy_from(struct block_iter *dest, struct block_iter *src)
++void block_iter_copy_from(struct reftable_block_iter *dest,
++ struct reftable_block_iter *src)
+{
+ dest->br = src->br;
+ dest->next_off = src->next_off;
@@ -579,7 +640,7 @@
+}
+
+/* return < 0 for error, 0 for OK, > 0 for EOF. */
-+int block_iter_next(struct block_iter *it, struct record rec)
++int block_iter_next(struct reftable_block_iter *it, struct record rec)
+{
+ if (it->next_off >= it->br->block_len) {
+ return 1;
@@ -598,23 +659,21 @@
+ return -1;
+ }
+
-+ in.buf += n;
-+ in.len -= n;
++ slice_consume(&in, n);
+ n = record_decode(rec, key, extra, in, it->br->hash_size);
+ if (n < 0) {
+ return -1;
+ }
-+ in.buf += n;
-+ in.len -= n;
++ slice_consume(&in, n);
+
+ slice_copy(&it->last_key, key);
+ it->next_off += start.len - in.len;
-+ free(slice_yield(&key));
++ reftable_free(slice_yield(&key));
+ return 0;
+ }
+}
+
-+int block_reader_first_key(struct block_reader *br, struct slice *key)
++int block_reader_first_key(struct reftable_block_reader *br, struct slice *key)
+{
+ struct slice empty = { 0 };
+ int off = br->header_off + 4;
@@ -631,18 +690,18 @@
+ return 0;
+}
+
-+int block_iter_seek(struct block_iter *it, struct slice want)
++int block_iter_seek(struct reftable_block_iter *it, struct slice want)
+{
+ return block_reader_seek(it->br, it, want);
+}
+
-+void block_iter_close(struct block_iter *it)
++void block_iter_close(struct reftable_block_iter *it)
+{
-+ free(slice_yield(&it->last_key));
++ reftable_free(slice_yield(&it->last_key));
+}
+
-+int block_reader_seek(struct block_reader *br, struct block_iter *it,
-+ struct slice want)
++int block_reader_seek(struct reftable_block_reader *br,
++ struct reftable_block_iter *it, struct slice want)
+{
+ struct restart_find_args args = {
+ .key = want,
@@ -667,7 +726,7 @@
+ struct slice key = { 0 };
+ int result = 0;
+ int err = 0;
-+ struct block_iter next = { 0 };
++ struct reftable_block_iter next = { 0 };
+ while (true) {
+ block_iter_copy_from(&next, it);
+
@@ -687,25 +746,25 @@
+ }
+
+ exit:
-+ free(slice_yield(&key));
-+ free(slice_yield(&next.last_key));
++ reftable_free(slice_yield(&key));
++ reftable_free(slice_yield(&next.last_key));
+ record_clear(rec);
-+ free(record_yield(&rec));
++ reftable_free(record_yield(&rec));
+
+ return result;
+ }
+}
+
-+void block_writer_reset(struct block_writer *bw)
++void block_writer_reset(struct reftable_block_writer *bw)
+{
+ bw->restart_len = 0;
+ bw->last_key.len = 0;
+}
+
-+void block_writer_clear(struct block_writer *bw)
++void block_writer_clear(struct reftable_block_writer *bw)
+{
+ FREE_AND_NULL(bw->restarts);
-+ free(slice_yield(&bw->last_key));
++ reftable_free(slice_yield(&bw->last_key));
+ /* the block is not owned. */
+}
@@ -729,7 +788,7 @@
+#include "record.h"
+#include "reftable.h"
+
-+struct block_writer {
++struct reftable_block_writer {
+ byte *buf;
+ uint32_t block_size;
+ uint32_t header_off;
@@ -744,17 +803,17 @@
+ int entries;
+};
+
-+void block_writer_init(struct block_writer *bw, byte typ, byte *buf,
++void block_writer_init(struct reftable_block_writer *bw, byte typ, byte *buf,
+ uint32_t block_size, uint32_t header_off, int hash_size);
-+byte block_writer_type(struct block_writer *bw);
-+int block_writer_add(struct block_writer *w, struct record rec);
-+int block_writer_finish(struct block_writer *w);
-+void block_writer_reset(struct block_writer *bw);
-+void block_writer_clear(struct block_writer *bw);
++byte block_writer_type(struct reftable_block_writer *bw);
++int block_writer_add(struct reftable_block_writer *w, struct record rec);
++int block_writer_finish(struct reftable_block_writer *w);
++void block_writer_reset(struct reftable_block_writer *bw);
++void block_writer_clear(struct reftable_block_writer *bw);
+
-+struct block_reader {
++struct reftable_block_reader {
+ uint32_t header_off;
-+ struct block block;
++ struct reftable_block block;
+ int hash_size;
+
+ /* size of the data, excluding restart data. */
@@ -764,25 +823,30 @@
+ uint16_t restart_count;
+};
+
-+struct block_iter {
++struct reftable_block_iter {
+ uint32_t next_off;
-+ struct block_reader *br;
++ struct reftable_block_reader *br;
+ struct slice last_key;
+};
+
-+int block_reader_init(struct block_reader *br, struct block *bl,
-+ uint32_t header_off, uint32_t table_block_size,
-+ int hash_size);
-+void block_reader_start(struct block_reader *br, struct block_iter *it);
-+int block_reader_seek(struct block_reader *br, struct block_iter *it,
-+ struct slice want);
-+byte block_reader_type(struct block_reader *r);
-+int block_reader_first_key(struct block_reader *br, struct slice *key);
-+
-+void block_iter_copy_from(struct block_iter *dest, struct block_iter *src);
-+int block_iter_next(struct block_iter *it, struct record rec);
-+int block_iter_seek(struct block_iter *it, struct slice want);
-+void block_iter_close(struct block_iter *it);
++int block_reader_init(struct reftable_block_reader *br,
++ struct reftable_block *bl, uint32_t header_off,
++ uint32_t table_block_size, int hash_size);
++void block_reader_start(struct reftable_block_reader *br,
++ struct reftable_block_iter *it);
++int block_reader_seek(struct reftable_block_reader *br,
++ struct reftable_block_iter *it, struct slice want);
++byte block_reader_type(struct reftable_block_reader *r);
++int block_reader_first_key(struct reftable_block_reader *br, struct slice *key);
++
++void block_iter_copy_from(struct reftable_block_iter *dest,
++ struct reftable_block_iter *src);
++int block_iter_next(struct reftable_block_iter *it, struct record rec);
++int block_iter_seek(struct reftable_block_iter *it, struct slice want);
++void block_iter_close(struct reftable_block_iter *it);
++
++int header_size(int version);
++int footer_size(int version);
+
+#endif
@@ -804,11 +868,13 @@
+
+#include "reftable.h"
+
-+uint64_t block_source_size(struct block_source source);
-+int block_source_read_block(struct block_source source, struct block *dest,
-+ uint64_t off, uint32_t size);
-+void block_source_return_block(struct block_source source, struct block *ret);
-+void block_source_close(struct block_source source);
++uint64_t block_source_size(struct reftable_block_source source);
++int block_source_read_block(struct reftable_block_source source,
++ struct reftable_block *dest, uint64_t off,
++ uint32_t size);
++void block_source_return_block(struct reftable_block_source source,
++ struct reftable_block *ret);
++void block_source_close(struct reftable_block_source source);
+
+#endif
@@ -838,10 +904,6 @@
+#ifndef CONSTANTS_H
+#define CONSTANTS_H
+
-+#define VERSION 1
-+#define HEADER_SIZE 24
-+#define FOOTER_SIZE 68
-+
+#define BLOCK_TYPE_LOG 'g'
+#define BLOCK_TYPE_INDEX 'i'
+#define BLOCK_TYPE_REF 'r'
@@ -987,10 +1049,10 @@
+ return ((struct file_block_source *)b)->size;
+}
+
-+static void file_return_block(void *b, struct block *dest)
++static void file_return_block(void *b, struct reftable_block *dest)
+{
+ memset(dest->data, 0xff, dest->len);
-+ free(dest->data);
++ reftable_free(dest->data);
+}
+
+static void file_close(void *b)
@@ -1001,15 +1063,15 @@
+ ((struct file_block_source *)b)->fd = 0;
+ }
+
-+ free(b);
++ reftable_free(b);
+}
+
-+static int file_read_block(void *v, struct block *dest, uint64_t off,
++static int file_read_block(void *v, struct reftable_block *dest, uint64_t off,
+ uint32_t size)
+{
+ struct file_block_source *b = (struct file_block_source *)v;
+ assert(off + size <= b->size);
-+ dest->data = malloc(size);
++ dest->data = reftable_malloc(size);
+ if (pread(b->fd, dest->data, size, off) != size) {
+ return -1;
+ }
@@ -1017,14 +1079,15 @@
+ return size;
+}
+
-+struct block_source_vtable file_vtable = {
++struct reftable_block_source_vtable file_vtable = {
+ .size = &file_size,
+ .read_block = &file_read_block,
+ .return_block = &file_return_block,
+ .close = &file_close,
+};
+
-+int block_source_from_file(struct block_source *bs, const char *name)
++int reftable_block_source_from_file(struct reftable_block_source *bs,
++ const char *name)
+{
+ struct stat st = { 0 };
+ int err = 0;
@@ -1043,7 +1106,7 @@
+
+ {
+ struct file_block_source *p =
-+ calloc(sizeof(struct file_block_source), 1);
++ reftable_calloc(sizeof(struct file_block_source));
+ p->size = st.st_size;
+ p->fd = fd;
+
@@ -1053,7 +1116,7 @@
+ return 0;
+}
+
-+int fd_writer(void *arg, byte *data, int sz)
++int reftable_fd_write(void *arg, byte *data, int sz)
+{
+ int *fdp = (int *)arg;
+ return write(*fdp, data, sz);
@@ -1081,7 +1144,7 @@
+#include "reader.h"
+#include "reftable.h"
+
-+bool iterator_is_null(struct iterator it)
++bool iterator_is_null(struct reftable_iterator it)
+{
+ return it.ops == NULL;
+}
@@ -1095,23 +1158,23 @@
+{
+}
+
-+struct iterator_vtable empty_vtable = {
++struct reftable_iterator_vtable empty_vtable = {
+ .next = &empty_iterator_next,
+ .close = &empty_iterator_close,
+};
+
-+void iterator_set_empty(struct iterator *it)
++void iterator_set_empty(struct reftable_iterator *it)
+{
+ it->iter_arg = NULL;
+ it->ops = &empty_vtable;
+}
+
-+int iterator_next(struct iterator it, struct record rec)
++int iterator_next(struct reftable_iterator it, struct record rec)
+{
+ return it.ops->next(it.iter_arg, rec);
+}
+
-+void iterator_destroy(struct iterator *it)
++void reftable_iterator_destroy(struct reftable_iterator *it)
+{
+ if (it->ops == NULL) {
+ return;
@@ -1121,14 +1184,16 @@
+ FREE_AND_NULL(it->iter_arg);
+}
+
-+int iterator_next_ref(struct iterator it, struct ref_record *ref)
++int reftable_iterator_next_ref(struct reftable_iterator it,
++ struct reftable_ref_record *ref)
+{
+ struct record rec = { 0 };
+ record_from_ref(&rec, ref);
+ return iterator_next(it, rec);
+}
+
-+int iterator_next_log(struct iterator it, struct log_record *log)
++int reftable_iterator_next_log(struct reftable_iterator it,
++ struct reftable_log_record *log)
+{
+ struct record rec = { 0 };
+ record_from_log(&rec, log);
@@ -1139,31 +1204,33 @@
+{
+ struct filtering_ref_iterator *fri =
+ (struct filtering_ref_iterator *)iter_arg;
-+ free(slice_yield(&fri->oid));
-+ iterator_destroy(&fri->it);
++ reftable_free(slice_yield(&fri->oid));
++ reftable_iterator_destroy(&fri->it);
+}
+
+static int filtering_ref_iterator_next(void *iter_arg, struct record rec)
+{
+ struct filtering_ref_iterator *fri =
+ (struct filtering_ref_iterator *)iter_arg;
-+ struct ref_record *ref = (struct ref_record *)rec.data;
++ struct reftable_ref_record *ref =
++ (struct reftable_ref_record *)rec.data;
+
+ while (true) {
-+ int err = iterator_next_ref(fri->it, ref);
++ int err = reftable_iterator_next_ref(fri->it, ref);
+ if (err != 0) {
+ return err;
+ }
+
+ if (fri->double_check) {
-+ struct iterator it = { 0 };
++ struct reftable_iterator it = { 0 };
+
-+ int err = reader_seek_ref(fri->r, &it, ref->ref_name);
++ int err = reftable_reader_seek_ref(fri->r, &it,
++ ref->ref_name);
+ if (err == 0) {
-+ err = iterator_next_ref(it, ref);
++ err = reftable_iterator_next_ref(it, ref);
+ }
+
-+ iterator_destroy(&it);
++ reftable_iterator_destroy(&it);
+
+ if (err < 0) {
+ return err;
@@ -1183,12 +1250,12 @@
+ }
+}
+
-+struct iterator_vtable filtering_ref_iterator_vtable = {
++struct reftable_iterator_vtable filtering_ref_iterator_vtable = {
+ .next = &filtering_ref_iterator_next,
+ .close = &filtering_ref_iterator_close,
+};
+
-+void iterator_from_filtering_ref_iterator(struct iterator *it,
++void iterator_from_filtering_ref_iterator(struct reftable_iterator *it,
+ struct filtering_ref_iterator *fri)
+{
+ it->iter_arg = fri;
@@ -1200,7 +1267,7 @@
+ struct indexed_table_ref_iter *it = (struct indexed_table_ref_iter *)p;
+ block_iter_close(&it->cur);
+ reader_return_block(it->r, &it->block_reader.block);
-+ free(slice_yield(&it->oid));
++ reftable_free(slice_yield(&it->oid));
+}
+
+static int indexed_table_ref_iter_next_block(struct indexed_table_ref_iter *it)
@@ -1231,7 +1298,8 @@
+static int indexed_table_ref_iter_next(void *p, struct record rec)
+{
+ struct indexed_table_ref_iter *it = (struct indexed_table_ref_iter *)p;
-+ struct ref_record *ref = (struct ref_record *)rec.data;
++ struct reftable_ref_record *ref =
++ (struct reftable_ref_record *)rec.data;
+
+ while (true) {
+ int err = block_iter_next(&it->cur, rec);
@@ -1259,11 +1327,11 @@
+}
+
+int new_indexed_table_ref_iter(struct indexed_table_ref_iter **dest,
-+ struct reader *r, byte *oid, int oid_len,
-+ uint64_t *offsets, int offset_len)
++ struct reftable_reader *r, byte *oid,
++ int oid_len, uint64_t *offsets, int offset_len)
+{
+ struct indexed_table_ref_iter *itr =
-+ calloc(sizeof(struct indexed_table_ref_iter), 1);
++ reftable_calloc(sizeof(struct indexed_table_ref_iter));
+ int err = 0;
+
+ itr->r = r;
@@ -1275,19 +1343,19 @@
+
+ err = indexed_table_ref_iter_next_block(itr);
+ if (err < 0) {
-+ free(itr);
++ reftable_free(itr);
+ } else {
+ *dest = itr;
+ }
+ return err;
+}
+
-+struct iterator_vtable indexed_table_ref_iter_vtable = {
++struct reftable_iterator_vtable indexed_table_ref_iter_vtable = {
+ .next = &indexed_table_ref_iter_next,
+ .close = &indexed_table_ref_iter_close,
+};
+
-+void iterator_from_indexed_table_ref_iter(struct iterator *it,
++void iterator_from_indexed_table_ref_iter(struct reftable_iterator *it,
+ struct indexed_table_ref_iter *itr)
+{
+ it->iter_arg = itr;
@@ -1314,27 +1382,27 @@
+#include "record.h"
+#include "slice.h"
+
-+struct iterator_vtable {
++struct reftable_iterator_vtable {
+ int (*next)(void *iter_arg, struct record rec);
+ void (*close)(void *iter_arg);
+};
+
-+void iterator_set_empty(struct iterator *it);
-+int iterator_next(struct iterator it, struct record rec);
-+bool iterator_is_null(struct iterator it);
++void iterator_set_empty(struct reftable_iterator *it);
++int iterator_next(struct reftable_iterator it, struct record rec);
++bool iterator_is_null(struct reftable_iterator it);
+
+struct filtering_ref_iterator {
+ bool double_check;
-+ struct reader *r;
++ struct reftable_reader *r;
+ struct slice oid;
-+ struct iterator it;
++ struct reftable_iterator it;
+};
+
-+void iterator_from_filtering_ref_iterator(struct iterator *,
++void iterator_from_filtering_ref_iterator(struct reftable_iterator *,
+ struct filtering_ref_iterator *);
+
+struct indexed_table_ref_iter {
-+ struct reader *r;
++ struct reftable_reader *r;
+ struct slice oid;
+
+ /* mutable */
@@ -1343,16 +1411,16 @@
+ /* Points to the next offset to read. */
+ int offset_idx;
+ int offset_len;
-+ struct block_reader block_reader;
-+ struct block_iter cur;
++ struct reftable_block_reader block_reader;
++ struct reftable_block_iter cur;
+ bool finished;
+};
+
-+void iterator_from_indexed_table_ref_iter(struct iterator *it,
++void iterator_from_indexed_table_ref_iter(struct reftable_iterator *it,
+ struct indexed_table_ref_iter *itr);
+int new_indexed_table_ref_iter(struct indexed_table_ref_iter **dest,
-+ struct reader *r, byte *oid, int oid_len,
-+ uint64_t *offsets, int offset_len);
++ struct reftable_reader *r, byte *oid,
++ int oid_len, uint64_t *offsets, int offset_len);
+
+#endif
@@ -1389,9 +1457,9 @@
+ }
+
+ if (err > 0) {
-+ iterator_destroy(&mi->stack[i]);
++ reftable_iterator_destroy(&mi->stack[i]);
+ record_clear(rec);
-+ free(record_yield(&rec));
++ reftable_free(record_yield(&rec));
+ } else {
+ struct pq_entry e = {
+ .rec = rec,
@@ -1410,9 +1478,9 @@
+ int i = 0;
+ merged_iter_pqueue_clear(&mi->pq);
+ for (i = 0; i < mi->stack_len; i++) {
-+ iterator_destroy(&mi->stack[i]);
++ reftable_iterator_destroy(&mi->stack[i]);
+ }
-+ free(mi->stack);
++ reftable_free(mi->stack);
+}
+
+static int merged_iter_advance_subiter(struct merged_iter *mi, int idx)
@@ -1433,9 +1501,9 @@
+ }
+
+ if (err > 0) {
-+ iterator_destroy(&mi->stack[idx]);
++ reftable_iterator_destroy(&mi->stack[idx]);
+ record_clear(rec);
-+ free(record_yield(&rec));
++ reftable_free(record_yield(&rec));
+ return 0;
+ }
+
@@ -1462,7 +1530,7 @@
+ record_key(top.rec, &k);
+
+ cmp = slice_compare(k, entry_key);
-+ free(slice_yield(&k));
++ reftable_free(slice_yield(&k));
+
+ if (cmp > 0) {
+ break;
@@ -1474,13 +1542,13 @@
+ return err;
+ }
+ record_clear(top.rec);
-+ free(record_yield(&top.rec));
++ reftable_free(record_yield(&top.rec));
+ }
+
+ record_copy_from(rec, entry.rec, hash_size(mi->hash_id));
+ record_clear(entry.rec);
-+ free(record_yield(&entry.rec));
-+ free(slice_yield(&entry_key));
++ reftable_free(record_yield(&entry.rec));
++ reftable_free(slice_yield(&entry_key));
+ return 0;
+}
+
@@ -1494,41 +1562,42 @@
+ return merged_iter_next(mi, rec);
+}
+
-+struct iterator_vtable merged_iter_vtable = {
++struct reftable_iterator_vtable merged_iter_vtable = {
+ .next = &merged_iter_next_void,
+ .close = &merged_iter_close,
+};
+
-+static void iterator_from_merged_iter(struct iterator *it,
++static void iterator_from_merged_iter(struct reftable_iterator *it,
+ struct merged_iter *mi)
+{
+ it->iter_arg = mi;
+ it->ops = &merged_iter_vtable;
+}
+
-+int new_merged_table(struct merged_table **dest, struct reader **stack, int n,
-+ uint32_t hash_id)
++int reftable_new_merged_table(struct reftable_merged_table **dest,
++ struct reftable_reader **stack, int n,
++ uint32_t hash_id)
+{
+ uint64_t last_max = 0;
+ uint64_t first_min = 0;
+ int i = 0;
+ for (i = 0; i < n; i++) {
-+ struct reader *r = stack[i];
++ struct reftable_reader *r = stack[i];
+ if (r->hash_id != hash_id) {
+ return FORMAT_ERROR;
+ }
-+ if (i > 0 && last_max >= reader_min_update_index(r)) {
++ if (i > 0 && last_max >= reftable_reader_min_update_index(r)) {
+ return FORMAT_ERROR;
+ }
+ if (i == 0) {
-+ first_min = reader_min_update_index(r);
++ first_min = reftable_reader_min_update_index(r);
+ }
+
-+ last_max = reader_max_update_index(r);
++ last_max = reftable_reader_max_update_index(r);
+ }
+
+ {
-+ struct merged_table m = {
++ struct reftable_merged_table m = {
+ .stack = stack,
+ .stack_len = n,
+ .min = first_min,
@@ -1536,52 +1605,56 @@
+ .hash_id = hash_id,
+ };
+
-+ *dest = calloc(sizeof(struct merged_table), 1);
++ *dest = reftable_calloc(sizeof(struct reftable_merged_table));
+ **dest = m;
+ }
+ return 0;
+}
+
-+void merged_table_close(struct merged_table *mt)
++void reftable_merged_table_close(struct reftable_merged_table *mt)
+{
+ int i = 0;
+ for (i = 0; i < mt->stack_len; i++) {
-+ reader_free(mt->stack[i]);
++ reftable_reader_free(mt->stack[i]);
+ }
+ FREE_AND_NULL(mt->stack);
+ mt->stack_len = 0;
+}
+
+/* clears the list of subtable, without affecting the readers themselves. */
-+void merged_table_clear(struct merged_table *mt)
++void merged_table_clear(struct reftable_merged_table *mt)
+{
+ FREE_AND_NULL(mt->stack);
+ mt->stack_len = 0;
+}
+
-+void merged_table_free(struct merged_table *mt)
++void reftable_merged_table_free(struct reftable_merged_table *mt)
+{
+ if (mt == NULL) {
+ return;
+ }
+ merged_table_clear(mt);
-+ free(mt);
++ reftable_free(mt);
+}
+
-+uint64_t merged_max_update_index(struct merged_table *mt)
++uint64_t
++reftable_merged_table_max_update_index(struct reftable_merged_table *mt)
+{
+ return mt->max;
+}
+
-+uint64_t merged_min_update_index(struct merged_table *mt)
++uint64_t
++reftable_merged_table_min_update_index(struct reftable_merged_table *mt)
+{
+ return mt->min;
+}
+
-+static int merged_table_seek_record(struct merged_table *mt,
-+ struct iterator *it, struct record rec)
++static int merged_table_seek_record(struct reftable_merged_table *mt,
++ struct reftable_iterator *it,
++ struct record rec)
+{
-+ struct iterator *iters = calloc(sizeof(struct iterator), mt->stack_len);
++ struct reftable_iterator *iters = reftable_calloc(
++ sizeof(struct reftable_iterator) * mt->stack_len);
+ struct merged_iter merged = {
+ .stack = iters,
+ .typ = record_type(rec),
@@ -1602,9 +1675,9 @@
+ if (err < 0) {
+ int i = 0;
+ for (i = 0; i < n; i++) {
-+ iterator_destroy(&iters[i]);
++ reftable_iterator_destroy(&iters[i]);
+ }
-+ free(iters);
++ reftable_free(iters);
+ return err;
+ }
+
@@ -1615,17 +1688,19 @@
+ }
+
+ {
-+ struct merged_iter *p = malloc(sizeof(struct merged_iter));
++ struct merged_iter *p =
++ reftable_malloc(sizeof(struct merged_iter));
+ *p = merged;
+ iterator_from_merged_iter(it, p);
+ }
+ return 0;
+}
+
-+int merged_table_seek_ref(struct merged_table *mt, struct iterator *it,
-+ const char *name)
++int reftable_merged_table_seek_ref(struct reftable_merged_table *mt,
++ struct reftable_iterator *it,
++ const char *name)
+{
-+ struct ref_record ref = {
++ struct reftable_ref_record ref = {
+ .ref_name = (char *)name,
+ };
+ struct record rec = { 0 };
@@ -1633,10 +1708,11 @@
+ return merged_table_seek_record(mt, it, rec);
+}
+
-+int merged_table_seek_log_at(struct merged_table *mt, struct iterator *it,
-+ const char *name, uint64_t update_index)
++int reftable_merged_table_seek_log_at(struct reftable_merged_table *mt,
++ struct reftable_iterator *it,
++ const char *name, uint64_t update_index)
+{
-+ struct log_record log = {
++ struct reftable_log_record log = {
+ .ref_name = (char *)name,
+ .update_index = update_index,
+ };
@@ -1645,11 +1721,12 @@
+ return merged_table_seek_record(mt, it, rec);
+}
+
-+int merged_table_seek_log(struct merged_table *mt, struct iterator *it,
-+ const char *name)
++int reftable_merged_table_seek_log(struct reftable_merged_table *mt,
++ struct reftable_iterator *it,
++ const char *name)
+{
+ uint64_t max = ~((uint64_t)0);
-+ return merged_table_seek_log_at(mt, it, name, max);
++ return reftable_merged_table_seek_log_at(mt, it, name, max);
+}
diff --git a/reftable/merged.h b/reftable/merged.h
@@ -1671,8 +1748,8 @@
+#include "pq.h"
+#include "reftable.h"
+
-+struct merged_table {
-+ struct reader **stack;
++struct reftable_merged_table {
++ struct reftable_reader **stack;
+ int stack_len;
+ uint32_t hash_id;
+
@@ -1681,14 +1758,14 @@
+};
+
+struct merged_iter {
-+ struct iterator *stack;
++ struct reftable_iterator *stack;
+ uint32_t hash_id;
+ int stack_len;
+ byte typ;
+ struct merged_iter_pqueue pq;
-+} merged_iter;
++};
+
-+void merged_table_clear(struct merged_table *mt);
++void merged_table_clear(struct reftable_merged_table *mt);
+
+#endif
@@ -1719,8 +1796,8 @@
+
+ cmp = slice_compare(ak, bk);
+
-+ free(slice_yield(&ak));
-+ free(slice_yield(&bk));
++ reftable_free(slice_yield(&ak));
++ reftable_free(slice_yield(&bk));
+
+ if (cmp == 0) {
+ return a.index > b.index;
@@ -1784,7 +1861,8 @@
+ int i = 0;
+ if (pq->len == pq->cap) {
+ pq->cap = 2 * pq->cap + 1;
-+ pq->heap = realloc(pq->heap, pq->cap * sizeof(struct pq_entry));
++ pq->heap = reftable_realloc(pq->heap,
++ pq->cap * sizeof(struct pq_entry));
+ }
+
+ pq->heap[pq->len++] = e;
@@ -1806,7 +1884,7 @@
+ int i = 0;
+ for (i = 0; i < pq->len; i++) {
+ record_clear(pq->heap[i].rec);
-+ free(record_yield(&pq->heap[i].rec));
++ reftable_free(record_yield(&pq->heap[i].rec));
+ }
+ FREE_AND_NULL(pq->heap);
+ pq->len = pq->cap = 0;
@@ -1876,20 +1954,22 @@
+#include "reftable.h"
+#include "tree.h"
+
-+uint64_t block_source_size(struct block_source source)
++uint64_t block_source_size(struct reftable_block_source source)
+{
+ return source.ops->size(source.arg);
+}
+
-+int block_source_read_block(struct block_source source, struct block *dest,
-+ uint64_t off, uint32_t size)
++int block_source_read_block(struct reftable_block_source source,
++ struct reftable_block *dest, uint64_t off,
++ uint32_t size)
+{
+ int result = source.ops->read_block(source.arg, dest, off, size);
+ dest->source = source;
+ return result;
+}
+
-+void block_source_return_block(struct block_source source, struct block *blockp)
++void block_source_return_block(struct reftable_block_source source,
++ struct reftable_block *blockp)
+{
+ source.ops->return_block(source.arg, blockp);
+ blockp->data = NULL;
@@ -1898,7 +1978,7 @@
+ blockp->source.arg = NULL;
+}
+
-+void block_source_close(struct block_source *source)
++void block_source_close(struct reftable_block_source *source)
+{
+ if (source->ops == NULL) {
+ return;
@@ -1908,7 +1988,8 @@
+ source->ops = NULL;
+}
+
-+static struct reader_offsets *reader_offsets_for(struct reader *r, byte typ)
++static struct reftable_reader_offsets *
++reader_offsets_for(struct reftable_reader *r, byte typ)
+{
+ switch (typ) {
+ case BLOCK_TYPE_REF:
@@ -1921,7 +2002,8 @@
+ abort();
+}
+
-+static int reader_get_block(struct reader *r, struct block *dest, uint64_t off,
++static int reader_get_block(struct reftable_reader *r,
++ struct reftable_block *dest, uint64_t off,
+ uint32_t sz)
+{
+ if (off >= r->size) {
@@ -1935,22 +2017,22 @@
+ return block_source_read_block(r->source, dest, off, sz);
+}
+
-+void reader_return_block(struct reader *r, struct block *p)
++void reader_return_block(struct reftable_reader *r, struct reftable_block *p)
+{
+ block_source_return_block(r->source, p);
+}
+
-+uint32_t reader_hash_id(struct reader *r)
++uint32_t reftable_reader_hash_id(struct reftable_reader *r)
+{
+ return r->hash_id;
+}
+
-+const char *reader_name(struct reader *r)
++const char *reader_name(struct reftable_reader *r)
+{
+ return r->name;
+}
+
-+static int parse_footer(struct reader *r, byte *footer, byte *header)
++static int parse_footer(struct reftable_reader *r, byte *footer, byte *header)
+{
+ byte *f = footer;
+ int err = 0;
@@ -1960,25 +2042,12 @@
+ }
+ f += 4;
+
-+ if (memcmp(footer, header, HEADER_SIZE)) {
++ if (memcmp(footer, header, header_size(r->version))) {
+ err = FORMAT_ERROR;
+ goto exit;
+ }
+
-+ r->hash_id = SHA1_ID;
-+ {
-+ byte version = *f++;
-+ if (version == 2) {
-+ /* DO NOT SUBMIT. Not yet in the standard. */
-+ r->hash_id = SHA256_ID;
-+ version = 1;
-+ }
-+ if (version != 1) {
-+ err = FORMAT_ERROR;
-+ goto exit;
-+ }
-+ }
-+
++ f++;
+ r->block_size = get_be24(f);
+
+ f += 3;
@@ -1987,6 +2056,22 @@
+ r->max_update_index = get_be64(f);
+ f += 8;
+
++ if (r->version == 1) {
++ r->hash_id = SHA1_ID;
++ } else {
++ r->hash_id = get_be32(f);
++ switch (r->hash_id) {
++ case SHA1_ID:
++ break;
++ case SHA256_ID:
++ break;
++ default:
++ err = FORMAT_ERROR;
++ goto exit;
++ }
++ f += 4;
++ }
++
+ r->ref_offsets.index_offset = get_be64(f);
+ f += 8;
+
@@ -2014,7 +2099,7 @@
+ }
+
+ {
-+ byte first_block_typ = header[HEADER_SIZE];
++ byte first_block_typ = header[header_size(r->version)];
+ r->ref_offsets.present = (first_block_typ == BLOCK_TYPE_REF);
+ r->ref_offsets.offset = 0;
+ r->log_offsets.present = (first_block_typ == BLOCK_TYPE_LOG ||
@@ -2026,27 +2111,40 @@
+ return err;
+}
+
-+int init_reader(struct reader *r, struct block_source source, const char *name)
++int init_reader(struct reftable_reader *r, struct reftable_block_source source,
++ const char *name)
+{
-+ struct block footer = { 0 };
-+ struct block header = { 0 };
++ struct reftable_block footer = { 0 };
++ struct reftable_block header = { 0 };
+ int err = 0;
+
-+ memset(r, 0, sizeof(struct reader));
-+ r->size = block_source_size(source) - FOOTER_SIZE;
-+ r->source = source;
-+ r->name = xstrdup(name);
-+ r->hash_id = 0;
++ memset(r, 0, sizeof(struct reftable_reader));
+
-+ err = block_source_read_block(source, &footer, r->size, FOOTER_SIZE);
-+ if (err != FOOTER_SIZE) {
++ /* Need +1 to read type of first block. */
++ err = block_source_read_block(source, &header, 0, header_size(2) + 1);
++ if (err != header_size(2) + 1) {
+ err = IO_ERROR;
+ goto exit;
+ }
+
-+ /* Need +1 to read type of first block. */
-+ err = reader_get_block(r, &header, 0, HEADER_SIZE + 1);
-+ if (err != HEADER_SIZE + 1) {
++ if (memcmp(header.data, "REFT", 4)) {
++ err = FORMAT_ERROR;
++ goto exit;
++ }
++ r->version = header.data[4];
++ if (r->version != 1 && r->version != 2) {
++ err = FORMAT_ERROR;
++ goto exit;
++ }
++
++ r->size = block_source_size(source) - footer_size(r->version);
++ r->source = source;
++ r->name = xstrdup(name);
++ r->hash_id = 0;
++
++ err = block_source_read_block(source, &footer, r->size,
++ footer_size(r->version));
++ if (err != footer_size(r->version)) {
+ err = IO_ERROR;
+ goto exit;
+ }
@@ -2059,10 +2157,10 @@
+}
+
+struct table_iter {
-+ struct reader *r;
++ struct reftable_reader *r;
+ byte typ;
+ uint64_t block_off;
-+ struct block_iter bi;
++ struct reftable_block_iter bi;
+ bool finished;
+};
+
@@ -2080,7 +2178,7 @@
+{
+ int res = block_iter_next(&ti->bi, rec);
+ if (res == 0 && record_type(rec) == BLOCK_TYPE_REF) {
-+ ((struct ref_record *)rec.data)->update_index +=
++ ((struct reftable_ref_record *)rec.data)->update_index +=
+ ti->r->min_update_index;
+ }
+
@@ -2099,12 +2197,13 @@
+ ti->bi.next_off = 0;
+}
+
-+static int32_t extract_block_size(byte *data, byte *typ, uint64_t off)
++static int32_t extract_block_size(byte *data, byte *typ, uint64_t off,
++ int version)
+{
+ int32_t result = 0;
+
+ if (off == 0) {
-+ data += 24;
++ data += header_size(version);
+ }
+
+ *typ = data[0];
@@ -2114,15 +2213,16 @@
+ return result;
+}
+
-+int reader_init_block_reader(struct reader *r, struct block_reader *br,
++int reader_init_block_reader(struct reftable_reader *r,
++ struct reftable_block_reader *br,
+ uint64_t next_off, byte want_typ)
+{
+ int32_t guess_block_size = r->block_size ? r->block_size :
+ DEFAULT_BLOCK_SIZE;
-+ struct block block = { 0 };
++ struct reftable_block block = { 0 };
+ byte block_typ = 0;
+ int err = 0;
-+ uint32_t header_off = next_off ? 0 : HEADER_SIZE;
++ uint32_t header_off = next_off ? 0 : header_size(r->version);
+ int32_t block_size = 0;
+
+ if (next_off >= r->size) {
@@ -2134,7 +2234,8 @@
+ return err;
+ }
+
-+ block_size = extract_block_size(block.data, &block_typ, next_off);
++ block_size = extract_block_size(block.data, &block_typ, next_off,
++ r->version);
+ if (block_size < 0) {
+ return block_size;
+ }
@@ -2160,7 +2261,7 @@
+ struct table_iter *src)
+{
+ uint64_t next_block_off = src->block_off + src->bi.br->full_block_size;
-+ struct block_reader br = { 0 };
++ struct reftable_block_reader br = { 0 };
+ int err = 0;
+
+ dest->r = src->r;
@@ -2177,7 +2278,8 @@
+ }
+
+ {
-+ struct block_reader *brp = malloc(sizeof(struct block_reader));
++ struct reftable_block_reader *brp =
++ reftable_malloc(sizeof(struct reftable_block_reader));
+ *brp = br;
+
+ dest->finished = false;
@@ -2229,29 +2331,30 @@
+ block_iter_close(&ti->bi);
+}
+
-+struct iterator_vtable table_iter_vtable = {
++struct reftable_iterator_vtable table_iter_vtable = {
+ .next = &table_iter_next_void,
+ .close = &table_iter_close,
+};
+
-+static void iterator_from_table_iter(struct iterator *it, struct table_iter *ti)
++static void iterator_from_table_iter(struct reftable_iterator *it,
++ struct table_iter *ti)
+{
+ it->iter_arg = ti;
+ it->ops = &table_iter_vtable;
+}
+
-+static int reader_table_iter_at(struct reader *r, struct table_iter *ti,
-+ uint64_t off, byte typ)
++static int reader_table_iter_at(struct reftable_reader *r,
++ struct table_iter *ti, uint64_t off, byte typ)
+{
-+ struct block_reader br = { 0 };
-+ struct block_reader *brp = NULL;
++ struct reftable_block_reader br = { 0 };
++ struct reftable_block_reader *brp = NULL;
+
+ int err = reader_init_block_reader(r, &br, off, typ);
+ if (err != 0) {
+ return err;
+ }
+
-+ brp = malloc(sizeof(struct block_reader));
++ brp = reftable_malloc(sizeof(struct reftable_block_reader));
+ *brp = br;
+ ti->r = r;
+ ti->typ = block_reader_type(brp);
@@ -2260,10 +2363,10 @@
+ return 0;
+}
+
-+static int reader_start(struct reader *r, struct table_iter *ti, byte typ,
-+ bool index)
++static int reader_start(struct reftable_reader *r, struct table_iter *ti,
++ byte typ, bool index)
+{
-+ struct reader_offsets *offs = reader_offsets_for(r, typ);
++ struct reftable_reader_offsets *offs = reader_offsets_for(r, typ);
+ uint64_t off = offs->offset;
+ if (index) {
+ off = offs->index_offset;
@@ -2276,7 +2379,7 @@
+ return reader_table_iter_at(r, ti, off, typ);
+}
+
-+static int reader_seek_linear(struct reader *r, struct table_iter *ti,
++static int reader_seek_linear(struct reftable_reader *r, struct table_iter *ti,
+ struct record want)
+{
+ struct record rec = new_record(record_type(want));
@@ -2321,14 +2424,14 @@
+exit:
+ block_iter_close(&next.bi);
+ record_clear(rec);
-+ free(record_yield(&rec));
-+ free(slice_yield(&want_key));
-+ free(slice_yield(&got_key));
++ reftable_free(record_yield(&rec));
++ reftable_free(slice_yield(&want_key));
++ reftable_free(slice_yield(&got_key));
+ return err;
+}
+
-+static int reader_seek_indexed(struct reader *r, struct iterator *it,
-+ struct record rec)
++static int reader_seek_indexed(struct reftable_reader *r,
++ struct reftable_iterator *it, struct record rec)
+{
+ struct index_record want_index = { 0 };
+ struct record want_index_rec = { 0 };
@@ -2380,7 +2483,7 @@
+
+ if (err == 0) {
+ struct table_iter *malloced =
-+ calloc(sizeof(struct table_iter), 1);
++ reftable_calloc(sizeof(struct table_iter));
+ table_iter_copy_from(malloced, &next);
+ iterator_from_table_iter(it, malloced);
+ }
@@ -2392,10 +2495,11 @@
+ return err;
+}
+
-+static int reader_seek_internal(struct reader *r, struct iterator *it,
-+ struct record rec)
++static int reader_seek_internal(struct reftable_reader *r,
++ struct reftable_iterator *it, struct record rec)
+{
-+ struct reader_offsets *offs = reader_offsets_for(r, record_type(rec));
++ struct reftable_reader_offsets *offs =
++ reader_offsets_for(r, record_type(rec));
+ uint64_t idx = offs->index_offset;
+ struct table_iter ti = { 0 };
+ int err = 0;
@@ -2413,7 +2517,8 @@
+ }
+
+ {
-+ struct table_iter *p = malloc(sizeof(struct table_iter));
++ struct table_iter *p =
++ reftable_malloc(sizeof(struct table_iter));
+ *p = ti;
+ iterator_from_table_iter(it, p);
+ }
@@ -2421,11 +2526,12 @@
+ return 0;
+}
+
-+int reader_seek(struct reader *r, struct iterator *it, struct record rec)
++int reader_seek(struct reftable_reader *r, struct reftable_iterator *it,
++ struct record rec)
+{
+ byte typ = record_type(rec);
+
-+ struct reader_offsets *offs = reader_offsets_for(r, typ);
++ struct reftable_reader_offsets *offs = reader_offsets_for(r, typ);
+ if (!offs->present) {
+ iterator_set_empty(it);
+ return 0;
@@ -2434,9 +2540,10 @@
+ return reader_seek_internal(r, it, rec);
+}
+
-+int reader_seek_ref(struct reader *r, struct iterator *it, const char *name)
++int reftable_reader_seek_ref(struct reftable_reader *r,
++ struct reftable_iterator *it, const char *name)
+{
-+ struct ref_record ref = {
++ struct reftable_ref_record ref = {
+ .ref_name = (char *)name,
+ };
+ struct record rec = { 0 };
@@ -2444,10 +2551,11 @@
+ return reader_seek(r, it, rec);
+}
+
-+int reader_seek_log_at(struct reader *r, struct iterator *it, const char *name,
-+ uint64_t update_index)
++int reftable_reader_seek_log_at(struct reftable_reader *r,
++ struct reftable_iterator *it, const char *name,
++ uint64_t update_index)
+{
-+ struct log_record log = {
++ struct reftable_log_record log = {
+ .ref_name = (char *)name,
+ .update_index = update_index,
+ };
@@ -2456,45 +2564,49 @@
+ return reader_seek(r, it, rec);
+}
+
-+int reader_seek_log(struct reader *r, struct iterator *it, const char *name)
++int reftable_reader_seek_log(struct reftable_reader *r,
++ struct reftable_iterator *it, const char *name)
+{
+ uint64_t max = ~((uint64_t)0);
-+ return reader_seek_log_at(r, it, name, max);
++ return reftable_reader_seek_log_at(r, it, name, max);
+}
+
-+void reader_close(struct reader *r)
++void reader_close(struct reftable_reader *r)
+{
+ block_source_close(&r->source);
+ FREE_AND_NULL(r->name);
+}
+
-+int new_reader(struct reader **p, struct block_source src, char const *name)
++int reftable_new_reader(struct reftable_reader **p,
++ struct reftable_block_source src, char const *name)
+{
-+ struct reader *rd = calloc(sizeof(struct reader), 1);
++ struct reftable_reader *rd =
++ reftable_calloc(sizeof(struct reftable_reader));
+ int err = init_reader(rd, src, name);
+ if (err == 0) {
+ *p = rd;
+ } else {
-+ free(rd);
++ reftable_free(rd);
+ }
+ return err;
+}
+
-+void reader_free(struct reader *r)
++void reftable_reader_free(struct reftable_reader *r)
+{
+ reader_close(r);
-+ free(r);
++ reftable_free(r);
+}
+
-+static int reader_refs_for_indexed(struct reader *r, struct iterator *it,
-+ byte *oid)
++static int reftable_reader_refs_for_indexed(struct reftable_reader *r,
++ struct reftable_iterator *it,
++ byte *oid)
+{
+ struct obj_record want = {
+ .hash_prefix = oid,
+ .hash_prefix_len = r->object_id_len,
+ };
+ struct record want_rec = { 0 };
-+ struct iterator oit = { 0 };
++ struct reftable_iterator oit = { 0 };
+ struct obj_record got = { 0 };
+ struct record got_rec = { 0 };
+ int err = 0;
@@ -2508,7 +2620,7 @@
+
+ record_from_obj(&got_rec, &got);
+ err = iterator_next(oit, got_rec);
-+ iterator_destroy(&oit);
++ reftable_iterator_destroy(&oit);
+ if (err < 0) {
+ return err;
+ }
@@ -2537,18 +2649,19 @@
+ return 0;
+}
+
-+static int reader_refs_for_unindexed(struct reader *r, struct iterator *it,
-+ byte *oid, int oid_len)
++static int reftable_reader_refs_for_unindexed(struct reftable_reader *r,
++ struct reftable_iterator *it,
++ byte *oid, int oid_len)
+{
-+ struct table_iter *ti = calloc(sizeof(struct table_iter), 1);
++ struct table_iter *ti = reftable_calloc(sizeof(struct table_iter));
+ struct filtering_ref_iterator *filter = NULL;
+ int err = reader_start(r, ti, BLOCK_TYPE_REF, false);
+ if (err < 0) {
-+ free(ti);
++ reftable_free(ti);
+ return err;
+ }
+
-+ filter = calloc(sizeof(struct filtering_ref_iterator), 1);
++ filter = reftable_calloc(sizeof(struct filtering_ref_iterator));
+ slice_resize(&filter->oid, oid_len);
+ memcpy(filter->oid.buf, oid, oid_len);
+ filter->r = r;
@@ -2559,21 +2672,22 @@
+ return 0;
+}
+
-+int reader_refs_for(struct reader *r, struct iterator *it, byte *oid,
-+ int oid_len)
++int reftable_reader_refs_for(struct reftable_reader *r,
++ struct reftable_iterator *it, byte *oid,
++ int oid_len)
+{
+ if (r->obj_offsets.present) {
-+ return reader_refs_for_indexed(r, it, oid);
++ return reftable_reader_refs_for_indexed(r, it, oid);
+ }
-+ return reader_refs_for_unindexed(r, it, oid, oid_len);
++ return reftable_reader_refs_for_unindexed(r, it, oid, oid_len);
+}
+
-+uint64_t reader_max_update_index(struct reader *r)
++uint64_t reftable_reader_max_update_index(struct reftable_reader *r)
+{
+ return r->max_update_index;
+}
+
-+uint64_t reader_min_update_index(struct reader *r)
++uint64_t reftable_reader_min_update_index(struct reftable_reader *r)
+{
+ return r->min_update_index;
+}
@@ -2598,40 +2712,48 @@
+#include "record.h"
+#include "reftable.h"
+
-+uint64_t block_source_size(struct block_source source);
++uint64_t block_source_size(struct reftable_block_source source);
+
-+int block_source_read_block(struct block_source source, struct block *dest,
-+ uint64_t off, uint32_t size);
-+void block_source_return_block(struct block_source source, struct block *ret);
-+void block_source_close(struct block_source *source);
++int block_source_read_block(struct reftable_block_source source,
++ struct reftable_block *dest, uint64_t off,
++ uint32_t size);
++void block_source_return_block(struct reftable_block_source source,
++ struct reftable_block *ret);
++void block_source_close(struct reftable_block_source *source);
+
-+struct reader_offsets {
++struct reftable_reader_offsets {
+ bool present;
+ uint64_t offset;
+ uint64_t index_offset;
+};
+
-+struct reader {
++struct reftable_reader {
+ char *name;
-+ struct block_source source;
++ struct reftable_block_source source;
+ uint32_t hash_id;
++
++ // Size of the file, excluding the footer.
+ uint64_t size;
+ uint32_t block_size;
+ uint64_t min_update_index;
+ uint64_t max_update_index;
+ int object_id_len;
++ int version;
+
-+ struct reader_offsets ref_offsets;
-+ struct reader_offsets obj_offsets;
-+ struct reader_offsets log_offsets;
++ struct reftable_reader_offsets ref_offsets;
++ struct reftable_reader_offsets obj_offsets;
++ struct reftable_reader_offsets log_offsets;
+};
+
-+int init_reader(struct reader *r, struct block_source source, const char *name);
-+int reader_seek(struct reader *r, struct iterator *it, struct record rec);
-+void reader_close(struct reader *r);
-+const char *reader_name(struct reader *r);
-+void reader_return_block(struct reader *r, struct block *p);
-+int reader_init_block_reader(struct reader *r, struct block_reader *br,
++int init_reader(struct reftable_reader *r, struct reftable_block_source source,
++ const char *name);
++int reader_seek(struct reftable_reader *r, struct reftable_iterator *it,
++ struct record rec);
++void reader_close(struct reftable_reader *r);
++const char *reader_name(struct reftable_reader *r);
++void reader_return_block(struct reftable_reader *r, struct reftable_block *p);
++int reader_init_block_reader(struct reftable_reader *r,
++ struct reftable_block_reader *br,
+ uint64_t next_off, byte want_typ);
+
+#endif
@@ -2649,6 +2771,8 @@
+https://developers.google.com/open-source/licenses/bsd
+*/
+
++/* record.c - methods for different types of records. */
++
+#include "record.h"
+
+#include "system.h"
@@ -2656,18 +2780,6 @@
+#include "constants.h"
+#include "reftable.h"
+
-+int is_block_type(byte typ)
-+{
-+ switch (typ) {
-+ case BLOCK_TYPE_REF:
-+ case BLOCK_TYPE_LOG:
-+ case BLOCK_TYPE_OBJ:
-+ case BLOCK_TYPE_INDEX:
-+ return true;
-+ }
-+ return false;
-+}
-+
+int get_var_int(uint64_t *dest, struct slice in)
+{
+ int ptr = 0;
@@ -2716,17 +2828,16 @@
+ }
+}
+
-+int common_prefix_size(struct slice a, struct slice b)
++int is_block_type(byte typ)
+{
-+ int p = 0;
-+ while (p < a.len && p < b.len) {
-+ if (a.buf[p] != b.buf[p]) {
-+ break;
-+ }
-+ p++;
++ switch (typ) {
++ case BLOCK_TYPE_REF:
++ case BLOCK_TYPE_LOG:
++ case BLOCK_TYPE_OBJ:
++ case BLOCK_TYPE_INDEX:
++ return true;
+ }
-+
-+ return p;
++ return false;
+}
+
+static int decode_string(struct slice *dest, struct slice in)
@@ -2737,8 +2848,7 @@
+ if (n <= 0) {
+ return -1;
+ }
-+ in.buf += n;
-+ in.len -= n;
++ slice_consume(&in, n);
+ if (in.len < tsize) {
+ return -1;
+ }
@@ -2746,12 +2856,29 @@
+ slice_resize(dest, tsize + 1);
+ dest->buf[tsize] = 0;
+ memcpy(dest->buf, in.buf, tsize);
-+ in.buf += tsize;
-+ in.len -= tsize;
++ slice_consume(&in, tsize);
+
+ return start_len - in.len;
+}
+
++static int encode_string(char *str, struct slice s)
++{
++ struct slice start = s;
++ int l = strlen(str);
++ int n = put_var_int(s, l);
++ if (n < 0) {
++ return -1;
++ }
++ slice_consume(&s, n);
++ if (s.len < l) {
++ return -1;
++ }
++ memcpy(s.buf, str, l);
++ slice_consume(&s, l);
++
++ return start.len - s.len;
++}
++
+int encode_key(bool *restart, struct slice dest, struct slice prev_key,
+ struct slice key, byte extra)
+{
@@ -2762,8 +2889,7 @@
+ if (n < 0) {
+ return -1;
+ }
-+ dest.buf += n;
-+ dest.len -= n;
++ slice_consume(&dest, n);
+
+ *restart = (prefix_len == 0);
+
@@ -2771,39 +2897,77 @@
+ if (n < 0) {
+ return -1;
+ }
-+ dest.buf += n;
-+ dest.len -= n;
++ slice_consume(&dest, n);
+
+ if (dest.len < suffix_len) {
+ return -1;
+ }
+ memcpy(dest.buf, key.buf + prefix_len, suffix_len);
-+ dest.buf += suffix_len;
-+ dest.len -= suffix_len;
++ slice_consume(&dest, suffix_len);
+
+ return start.len - dest.len;
+}
+
-+static byte ref_record_type(void)
++int decode_key(struct slice *key, byte *extra, struct slice last_key,
++ struct slice in)
++{
++ int start_len = in.len;
++ uint64_t prefix_len = 0;
++ uint64_t suffix_len = 0;
++ int n = get_var_int(&prefix_len, in);
++ if (n < 0) {
++ return -1;
++ }
++ slice_consume(&in, n);
++
++ if (prefix_len > last_key.len) {
++ return -1;
++ }
++
++ n = get_var_int(&suffix_len, in);
++ if (n <= 0) {
++ return -1;
++ }
++ slice_consume(&in, n);
++
++ *extra = (byte)(suffix_len & 0x7);
++ suffix_len >>= 3;
++
++ if (in.len < suffix_len) {
++ return -1;
++ }
++
++ slice_resize(key, suffix_len + prefix_len);
++ memcpy(key->buf, last_key.buf, prefix_len);
++
++ memcpy(key->buf + prefix_len, in.buf, suffix_len);
++ slice_consume(&in, suffix_len);
++
++ return start_len - in.len;
++}
++
++static byte reftable_ref_record_type(void)
+{
+ return BLOCK_TYPE_REF;
+}
+
-+static void ref_record_key(const void *r, struct slice *dest)
++static void reftable_ref_record_key(const void *r, struct slice *dest)
+{
-+ const struct ref_record *rec = (const struct ref_record *)r;
++ const struct reftable_ref_record *rec =
++ (const struct reftable_ref_record *)r;
+ slice_set_string(dest, rec->ref_name);
+}
+
-+static void ref_record_copy_from(void *rec, const void *src_rec, int hash_size)
++static void reftable_ref_record_copy_from(void *rec, const void *src_rec,
++ int hash_size)
+{
-+ struct ref_record *ref = (struct ref_record *)rec;
-+ struct ref_record *src = (struct ref_record *)src_rec;
++ struct reftable_ref_record *ref = (struct reftable_ref_record *)rec;
++ struct reftable_ref_record *src = (struct reftable_ref_record *)src_rec;
+ assert(hash_size > 0);
+
+ /* This is simple and correct, but we could probably reuse the hash
+ fields. */
-+ ref_record_clear(ref);
++ reftable_ref_record_clear(ref);
+ if (src->ref_name != NULL) {
+ ref->ref_name = xstrdup(src->ref_name);
+ }
@@ -2813,12 +2977,12 @@
+ }
+
+ if (src->target_value != NULL) {
-+ ref->target_value = malloc(hash_size);
++ ref->target_value = reftable_malloc(hash_size);
+ memcpy(ref->target_value, src->target_value, hash_size);
+ }
+
+ if (src->value != NULL) {
-+ ref->value = malloc(hash_size);
++ ref->value = reftable_malloc(hash_size);
+ memcpy(ref->value, src->value, hash_size);
+ }
+ ref->update_index = src->update_index;
@@ -2845,7 +3009,7 @@
+ }
+}
+
-+void ref_record_print(struct ref_record *ref, int hash_size)
++void reftable_ref_record_print(struct reftable_ref_record *ref, int hash_size)
+{
+ char hex[SHA256_SIZE + 1] = { 0 };
+
@@ -2864,23 +3028,24 @@
+ printf("}\n");
+}
+
-+static void ref_record_clear_void(void *rec)
++static void reftable_ref_record_clear_void(void *rec)
+{
-+ ref_record_clear((struct ref_record *)rec);
++ reftable_ref_record_clear((struct reftable_ref_record *)rec);
+}
+
-+void ref_record_clear(struct ref_record *ref)
++void reftable_ref_record_clear(struct reftable_ref_record *ref)
+{
-+ free(ref->ref_name);
-+ free(ref->target);
-+ free(ref->target_value);
-+ free(ref->value);
-+ memset(ref, 0, sizeof(struct ref_record));
++ reftable_free(ref->ref_name);
++ reftable_free(ref->target);
++ reftable_free(ref->target_value);
++ reftable_free(ref->value);
++ memset(ref, 0, sizeof(struct reftable_ref_record));
+}
+
-+static byte ref_record_val_type(const void *rec)
++static byte reftable_ref_record_val_type(const void *rec)
+{
-+ const struct ref_record *r = (const struct ref_record *)rec;
++ const struct reftable_ref_record *r =
++ (const struct reftable_ref_record *)rec;
+ if (r->value != NULL) {
+ if (r->target_value != NULL) {
+ return 2;
@@ -2893,45 +3058,25 @@
+ return 0;
+}
+
-+static int encode_string(char *str, struct slice s)
-+{
-+ struct slice start = s;
-+ int l = strlen(str);
-+ int n = put_var_int(s, l);
-+ if (n < 0) {
-+ return -1;
-+ }
-+ s.buf += n;
-+ s.len -= n;
-+ if (s.len < l) {
-+ return -1;
-+ }
-+ memcpy(s.buf, str, l);
-+ s.buf += l;
-+ s.len -= l;
-+
-+ return start.len - s.len;
-+}
-+
-+static int ref_record_encode(const void *rec, struct slice s, int hash_size)
++static int reftable_ref_record_encode(const void *rec, struct slice s,
++ int hash_size)
+{
-+ const struct ref_record *r = (const struct ref_record *)rec;
++ const struct reftable_ref_record *r =
++ (const struct reftable_ref_record *)rec;
+ struct slice start = s;
+ int n = put_var_int(s, r->update_index);
+ assert(hash_size > 0);
+ if (n < 0) {
+ return -1;
+ }
-+ s.buf += n;
-+ s.len -= n;
++ slice_consume(&s, n);
+
+ if (r->value != NULL) {
+ if (s.len < hash_size) {
+ return -1;
+ }
+ memcpy(s.buf, r->value, hash_size);
-+ s.buf += hash_size;
-+ s.len -= hash_size;
++ slice_consume(&s, hash_size);
+ }
+
+ if (r->target_value != NULL) {
@@ -2939,8 +3084,7 @@
+ return -1;
+ }
+ memcpy(s.buf, r->target_value, hash_size);
-+ s.buf += hash_size;
-+ s.len -= hash_size;
++ slice_consume(&s, hash_size);
+ }
+
+ if (r->target != NULL) {
@@ -2948,17 +3092,17 @@
+ if (n < 0) {
+ return -1;
+ }
-+ s.buf += n;
-+ s.len -= n;
++ slice_consume(&s, n);
+ }
+
+ return start.len - s.len;
+}
+
-+static int ref_record_decode(void *rec, struct slice key, byte val_type,
-+ struct slice in, int hash_size)
++static int reftable_ref_record_decode(void *rec, struct slice key,
++ byte val_type, struct slice in,
++ int hash_size)
+{
-+ struct ref_record *r = (struct ref_record *)rec;
++ struct reftable_ref_record *r = (struct reftable_ref_record *)rec;
+ struct slice start = in;
+ bool seen_value = false;
+ bool seen_target_value = false;
@@ -2970,10 +3114,9 @@
+ }
+ assert(hash_size > 0);
+
-+ in.buf += n;
-+ in.len -= n;
++ slice_consume(&in, n);
+
-+ r->ref_name = realloc(r->ref_name, key.len + 1);
++ r->ref_name = reftable_realloc(r->ref_name, key.len + 1);
+ memcpy(r->ref_name, key.buf, key.len);
+ r->ref_name[key.len] = 0;
+
@@ -2985,22 +3128,20 @@
+ }
+
+ if (r->value == NULL) {
-+ r->value = malloc(hash_size);
++ r->value = reftable_malloc(hash_size);
+ }
+ seen_value = true;
+ memcpy(r->value, in.buf, hash_size);
-+ in.buf += hash_size;
-+ in.len -= hash_size;
++ slice_consume(&in, hash_size);
+ if (val_type == 1) {
+ break;
+ }
+ if (r->target_value == NULL) {
-+ r->target_value = malloc(hash_size);
++ r->target_value = reftable_malloc(hash_size);
+ }
+ seen_target_value = true;
+ memcpy(r->target_value, in.buf, hash_size);
-+ in.buf += hash_size;
-+ in.len -= hash_size;
++ slice_consume(&in, hash_size);
+ break;
+ case 3: {
+ struct slice dest = { 0 };
@@ -3008,8 +3149,7 @@
+ if (n < 0) {
+ return -1;
+ }
-+ in.buf += n;
-+ in.len -= n;
++ slice_consume(&in, n);
+ seen_target = true;
+ r->target = (char *)slice_as_string(&dest);
+ } break;
@@ -3034,55 +3174,14 @@
+ return start.len - in.len;
+}
+
-+int decode_key(struct slice *key, byte *extra, struct slice last_key,
-+ struct slice in)
-+{
-+ int start_len = in.len;
-+ uint64_t prefix_len = 0;
-+ uint64_t suffix_len = 0;
-+ int n = get_var_int(&prefix_len, in);
-+ if (n < 0) {
-+ return -1;
-+ }
-+ in.buf += n;
-+ in.len -= n;
-+
-+ if (prefix_len > last_key.len) {
-+ return -1;
-+ }
-+
-+ n = get_var_int(&suffix_len, in);
-+ if (n <= 0) {
-+ return -1;
-+ }
-+ in.buf += n;
-+ in.len -= n;
-+
-+ *extra = (byte)(suffix_len & 0x7);
-+ suffix_len >>= 3;
-+
-+ if (in.len < suffix_len) {
-+ return -1;
-+ }
-+
-+ slice_resize(key, suffix_len + prefix_len);
-+ memcpy(key->buf, last_key.buf, prefix_len);
-+
-+ memcpy(key->buf + prefix_len, in.buf, suffix_len);
-+ in.buf += suffix_len;
-+ in.len -= suffix_len;
-+
-+ return start_len - in.len;
-+}
-+
-+struct record_vtable ref_record_vtable = {
-+ .key = &ref_record_key,
-+ .type = &ref_record_type,
-+ .copy_from = &ref_record_copy_from,
-+ .val_type = &ref_record_val_type,
-+ .encode = &ref_record_encode,
-+ .decode = &ref_record_decode,
-+ .clear = &ref_record_clear_void,
++struct record_vtable reftable_ref_record_vtable = {
++ .key = &reftable_ref_record_key,
++ .type = &reftable_ref_record_type,
++ .copy_from = &reftable_ref_record_copy_from,
++ .val_type = &reftable_ref_record_val_type,
++ .encode = &reftable_ref_record_encode,
++ .decode = &reftable_ref_record_decode,
++ .clear = &reftable_ref_record_clear_void,
+};
+
+static byte obj_record_type(void)
@@ -3103,12 +3202,12 @@
+ const struct obj_record *src = (const struct obj_record *)src_rec;
+
+ *ref = *src;
-+ ref->hash_prefix = malloc(ref->hash_prefix_len);
++ ref->hash_prefix = reftable_malloc(ref->hash_prefix_len);
+ memcpy(ref->hash_prefix, src->hash_prefix, ref->hash_prefix_len);
+
+ {
+ int olen = ref->offset_len * sizeof(uint64_t);
-+ ref->offsets = malloc(olen);
++ ref->offsets = reftable_malloc(olen);
+ memcpy(ref->offsets, src->offsets, olen);
+ }
+}
@@ -3140,8 +3239,7 @@
+ if (n < 0) {
+ return -1;
+ }
-+ s.buf += n;
-+ s.len -= n;
++ slice_consume(&s, n);
+ }
+ if (r->offset_len == 0) {
+ return start.len - s.len;
@@ -3150,8 +3248,7 @@
+ if (n < 0) {
+ return -1;
+ }
-+ s.buf += n;
-+ s.len -= n;
++ slice_consume(&s, n);
+
+ {
+ uint64_t last = r->offsets[0];
@@ -3161,8 +3258,7 @@
+ if (n < 0) {
+ return -1;
+ }
-+ s.buf += n;
-+ s.len -= n;
++ slice_consume(&s, n);
+ last = r->offsets[i];
+ }
+ }
@@ -3176,7 +3272,7 @@
+ struct obj_record *r = (struct obj_record *)rec;
+ uint64_t count = val_type;
+ int n = 0;
-+ r->hash_prefix = malloc(key.len);
++ r->hash_prefix = reftable_malloc(key.len);
+ memcpy(r->hash_prefix, key.buf, key.len);
+ r->hash_prefix_len = key.len;
+
@@ -3186,8 +3282,7 @@
+ return n;
+ }
+
-+ in.buf += n;
-+ in.len -= n;
++ slice_consume(&in, n);
+ }
+
+ r->offsets = NULL;
@@ -3196,16 +3291,14 @@
+ return start.len - in.len;
+ }
+
-+ r->offsets = malloc(count * sizeof(uint64_t));
++ r->offsets = reftable_malloc(count * sizeof(uint64_t));
+ r->offset_len = count;
+
+ n = get_var_int(&r->offsets[0], in);
+ if (n < 0) {
+ return n;
+ }
-+
-+ in.buf += n;
-+ in.len -= n;
++ slice_consume(&in, n);
+
+ {
+ uint64_t last = r->offsets[0];
@@ -3216,9 +3309,7 @@
+ if (n < 0) {
+ return n;
+ }
-+
-+ in.buf += n;
-+ in.len -= n;
++ slice_consume(&in, n);
+
+ last = r->offsets[j] = (delta + last);
+ j++;
@@ -3237,7 +3328,7 @@
+ .clear = &obj_record_clear,
+};
+
-+void log_record_print(struct log_record *log, int hash_size)
++void reftable_log_record_print(struct reftable_log_record *log, int hash_size)
+{
+ char hex[SHA256_SIZE + 1] = { 0 };
+
@@ -3250,14 +3341,15 @@
+ printf("%s\n\n%s\n}\n", hex, log->message);
+}
+
-+static byte log_record_type(void)
++static byte reftable_log_record_type(void)
+{
+ return BLOCK_TYPE_LOG;
+}
+
-+static void log_record_key(const void *r, struct slice *dest)
++static void reftable_log_record_key(const void *r, struct slice *dest)
+{
-+ const struct log_record *rec = (const struct log_record *)r;
++ const struct reftable_log_record *rec =
++ (const struct reftable_log_record *)r;
+ int len = strlen(rec->ref_name);
+ uint64_t ts = 0;
+ slice_resize(dest, len + 9);
@@ -3266,10 +3358,12 @@
+ put_be64(dest->buf + 1 + len, ts);
+}
+
-+static void log_record_copy_from(void *rec, const void *src_rec, int hash_size)
++static void reftable_log_record_copy_from(void *rec, const void *src_rec,
++ int hash_size)
+{
-+ struct log_record *dst = (struct log_record *)rec;
-+ const struct log_record *src = (const struct log_record *)src_rec;
++ struct reftable_log_record *dst = (struct reftable_log_record *)rec;
++ const struct reftable_log_record *src =
++ (const struct reftable_log_record *)src_rec;
+
+ *dst = *src;
+ dst->ref_name = xstrdup(dst->ref_name);
@@ -3277,46 +3371,54 @@
+ dst->name = xstrdup(dst->name);
+ dst->message = xstrdup(dst->message);
+ if (dst->new_hash != NULL) {
-+ dst->new_hash = malloc(hash_size);
++ dst->new_hash = reftable_malloc(hash_size);
+ memcpy(dst->new_hash, src->new_hash, hash_size);
+ }
+ if (dst->old_hash != NULL) {
-+ dst->old_hash = malloc(hash_size);
++ dst->old_hash = reftable_malloc(hash_size);
+ memcpy(dst->old_hash, src->old_hash, hash_size);
+ }
+}
+
-+static void log_record_clear_void(void *rec)
++static void reftable_log_record_clear_void(void *rec)
+{
-+ struct log_record *r = (struct log_record *)rec;
-+ log_record_clear(r);
++ struct reftable_log_record *r = (struct reftable_log_record *)rec;
++ reftable_log_record_clear(r);
+}
+
-+void log_record_clear(struct log_record *r)
++void reftable_log_record_clear(struct reftable_log_record *r)
+{
-+ free(r->ref_name);
-+ free(r->new_hash);
-+ free(r->old_hash);
-+ free(r->name);
-+ free(r->email);
-+ free(r->message);
-+ memset(r, 0, sizeof(struct log_record));
++ reftable_free(r->ref_name);
++ reftable_free(r->new_hash);
++ reftable_free(r->old_hash);
++ reftable_free(r->name);
++ reftable_free(r->email);
++ reftable_free(r->message);
++ memset(r, 0, sizeof(struct reftable_log_record));
+}
+
-+static byte log_record_val_type(const void *rec)
++static byte reftable_log_record_val_type(const void *rec)
+{
-+ return 1;
++ const struct reftable_log_record *log =
++ (const struct reftable_log_record *)rec;
++
++ return reftable_log_record_is_deletion(log) ? 0 : 1;
+}
+
+static byte zero[SHA256_SIZE] = { 0 };
+
-+static int log_record_encode(const void *rec, struct slice s, int hash_size)
++static int reftable_log_record_encode(const void *rec, struct slice s,
++ int hash_size)
+{
-+ struct log_record *r = (struct log_record *)rec;
++ struct reftable_log_record *r = (struct reftable_log_record *)rec;
+ struct slice start = s;
+ int n = 0;
+ byte *oldh = r->old_hash;
+ byte *newh = r->new_hash;
++ if (reftable_log_record_is_deletion(r)) {
++ return 0;
++ }
++
+ if (oldh == NULL) {
+ oldh = zero;
+ }
@@ -3330,53 +3432,48 @@
+
+ memcpy(s.buf, oldh, hash_size);
+ memcpy(s.buf + hash_size, newh, hash_size);
-+ s.buf += 2 * hash_size;
-+ s.len -= 2 * hash_size;
++ slice_consume(&s, 2 * hash_size);
+
+ n = encode_string(r->name ? r->name : "", s);
+ if (n < 0) {
+ return -1;
+ }
-+ s.len -= n;
-+ s.buf += n;
++ slice_consume(&s, n);
+
+ n = encode_string(r->email ? r->email : "", s);
+ if (n < 0) {
+ return -1;
+ }
-+ s.len -= n;
-+ s.buf += n;
++ slice_consume(&s, n);
+
+ n = put_var_int(s, r->time);
+ if (n < 0) {
+ return -1;
+ }
-+ s.buf += n;
-+ s.len -= n;
++ slice_consume(&s, n);
+
+ if (s.len < 2) {
+ return -1;
+ }
+
+ put_be16(s.buf, r->tz_offset);
-+ s.buf += 2;
-+ s.len -= 2;
++ slice_consume(&s, 2);
+
+ n = encode_string(r->message ? r->message : "", s);
+ if (n < 0) {
+ return -1;
+ }
-+ s.len -= n;
-+ s.buf += n;
++ slice_consume(&s, n);
+
+ return start.len - s.len;
+}
+
-+static int log_record_decode(void *rec, struct slice key, byte val_type,
-+ struct slice in, int hash_size)
++static int reftable_log_record_decode(void *rec, struct slice key,
++ byte val_type, struct slice in,
++ int hash_size)
+{
+ struct slice start = in;
-+ struct log_record *r = (struct log_record *)rec;
++ struct reftable_log_record *r = (struct reftable_log_record *)rec;
+ uint64_t max = 0;
+ uint64_t ts = 0;
+ struct slice dest = { 0 };
@@ -3386,33 +3483,35 @@
+ return FORMAT_ERROR;
+ }
+
-+ r->ref_name = realloc(r->ref_name, key.len - 8);
++ r->ref_name = reftable_realloc(r->ref_name, key.len - 8);
+ memcpy(r->ref_name, key.buf, key.len - 8);
+ ts = get_be64(key.buf + key.len - 8);
+
+ r->update_index = (~max) - ts;
+
++ if (val_type == 0) {
++ return 0;
++ }
++
+ if (in.len < 2 * hash_size) {
+ return FORMAT_ERROR;
+ }
+
-+ r->old_hash = realloc(r->old_hash, hash_size);
-+ r->new_hash = realloc(r->new_hash, hash_size);
++ r->old_hash = reftable_realloc(r->old_hash, hash_size);
++ r->new_hash = reftable_realloc(r->new_hash, hash_size);
+
+ memcpy(r->old_hash, in.buf, hash_size);
+ memcpy(r->new_hash, in.buf + hash_size, hash_size);
+
-+ in.buf += 2 * hash_size;
-+ in.len -= 2 * hash_size;
++ slice_consume(&in, 2 * hash_size);
+
+ n = decode_string(&dest, in);
+ if (n < 0) {
+ goto error;
+ }
-+ in.len -= n;
-+ in.buf += n;
++ slice_consume(&in, n);
+
-+ r->name = realloc(r->name, dest.len + 1);
++ r->name = reftable_realloc(r->name, dest.len + 1);
+ memcpy(r->name, dest.buf, dest.len);
+ r->name[dest.len] = 0;
+
@@ -3421,10 +3520,9 @@
+ if (n < 0) {
+ goto error;
+ }
-+ in.len -= n;
-+ in.buf += n;
++ slice_consume(&in, n);
+
-+ r->email = realloc(r->email, dest.len + 1);
++ r->email = reftable_realloc(r->email, dest.len + 1);
+ memcpy(r->email, dest.buf, dest.len);
+ r->email[dest.len] = 0;
+
@@ -3433,33 +3531,30 @@
+ if (n < 0) {
+ goto error;
+ }
-+ in.len -= n;
-+ in.buf += n;
++ slice_consume(&in, n);
+ r->time = ts;
+ if (in.len < 2) {
+ goto error;
+ }
+
+ r->tz_offset = get_be16(in.buf);
-+ in.buf += 2;
-+ in.len -= 2;
++ slice_consume(&in, 2);
+
+ slice_resize(&dest, 0);
+ n = decode_string(&dest, in);
+ if (n < 0) {
+ goto error;
+ }
-+ in.len -= n;
-+ in.buf += n;
++ slice_consume(&in, n);
+
-+ r->message = realloc(r->message, dest.len + 1);
++ r->message = reftable_realloc(r->message, dest.len + 1);
+ memcpy(r->message, dest.buf, dest.len);
+ r->message[dest.len] = 0;
+
+ return start.len - in.len;
+
+error:
-+ free(slice_yield(&dest));
++ reftable_free(slice_yield(&dest));
+ return FORMAT_ERROR;
+}
+
@@ -3486,7 +3581,8 @@
+ return !memcmp(a, b, sz);
+}
+
-+bool log_record_equal(struct log_record *a, struct log_record *b, int hash_size)
++bool reftable_log_record_equal(struct reftable_log_record *a,
++ struct reftable_log_record *b, int hash_size)
+{
+ return null_streq(a->name, b->name) && null_streq(a->email, b->email) &&
+ null_streq(a->message, b->message) &&
@@ -3496,14 +3592,14 @@
+ a->update_index == b->update_index;
+}
+
-+struct record_vtable log_record_vtable = {
-+ .key = &log_record_key,
-+ .type = &log_record_type,
-+ .copy_from = &log_record_copy_from,
-+ .val_type = &log_record_val_type,
-+ .encode = &log_record_encode,
-+ .decode = &log_record_decode,
-+ .clear = &log_record_clear_void,
++struct record_vtable reftable_log_record_vtable = {
++ .key = &reftable_log_record_key,
++ .type = &reftable_log_record_type,
++ .copy_from = &reftable_log_record_copy_from,
++ .val_type = &reftable_log_record_val_type,
++ .encode = &reftable_log_record_encode,
++ .decode = &reftable_log_record_decode,
++ .clear = &reftable_log_record_clear_void,
+};
+
+struct record new_record(byte typ)
@@ -3511,23 +3607,27 @@
+ struct record rec;
+ switch (typ) {
+ case BLOCK_TYPE_REF: {
-+ struct ref_record *r = calloc(1, sizeof(struct ref_record));
++ struct reftable_ref_record *r =
++ reftable_calloc(sizeof(struct reftable_ref_record));
+ record_from_ref(&rec, r);
+ return rec;
+ }
+
+ case BLOCK_TYPE_OBJ: {
-+ struct obj_record *r = calloc(1, sizeof(struct obj_record));
++ struct obj_record *r =
++ reftable_calloc(sizeof(struct obj_record));
+ record_from_obj(&rec, r);
+ return rec;
+ }
+ case BLOCK_TYPE_LOG: {
-+ struct log_record *r = calloc(1, sizeof(struct log_record));
++ struct reftable_log_record *r =
++ reftable_calloc(sizeof(struct reftable_log_record));
+ record_from_log(&rec, r);
+ return rec;
+ }
+ case BLOCK_TYPE_INDEX: {
-+ struct index_record *r = calloc(1, sizeof(struct index_record));
++ struct index_record *r =
++ reftable_calloc(sizeof(struct index_record));
+ record_from_index(&rec, r);
+ return rec;
+ }
@@ -3560,7 +3660,7 @@
+static void index_record_clear(void *rec)
+{
+ struct index_record *idx = (struct index_record *)rec;
-+ free(slice_yield(&idx->last_key));
++ reftable_free(slice_yield(&idx->last_key));
+}
+
+static byte index_record_val_type(const void *rec)
@@ -3578,8 +3678,7 @@
+ return n;
+ }
+
-+ out.buf += n;
-+ out.len -= n;
++ slice_consume(&out, n);
+
+ return start.len - out.len;
+}
@@ -3598,8 +3697,7 @@
+ return n;
+ }
+
-+ in.buf += n;
-+ in.len -= n;
++ slice_consume(&in, n);
+ return start.len - in.len;
+}
+
@@ -3651,10 +3749,10 @@
+ return rec.ops->clear(rec.data);
+}
+
-+void record_from_ref(struct record *rec, struct ref_record *ref_rec)
++void record_from_ref(struct record *rec, struct reftable_ref_record *ref_rec)
+{
+ rec->data = ref_rec;
-+ rec->ops = &ref_record_vtable;
++ rec->ops = &reftable_ref_record_vtable;
+}
+
+void record_from_obj(struct record *rec, struct obj_record *obj_rec)
@@ -3669,10 +3767,10 @@
+ rec->ops = &index_record_vtable;
+}
+
-+void record_from_log(struct record *rec, struct log_record *log_rec)
++void record_from_log(struct record *rec, struct reftable_log_record *log_rec)
+{
+ rec->data = log_rec;
-+ rec->ops = &log_record_vtable;
++ rec->ops = &reftable_log_record_vtable;
+}
+
+void *record_yield(struct record *rec)
@@ -3682,10 +3780,10 @@
+ return p;
+}
+
-+struct ref_record *record_as_ref(struct record rec)
++struct reftable_ref_record *record_as_ref(struct record rec)
+{
+ assert(record_type(rec) == BLOCK_TYPE_REF);
-+ return (struct ref_record *)rec.data;
++ return (struct reftable_ref_record *)rec.data;
+}
+
+static bool hash_equal(byte *a, byte *b, int hash_size)
@@ -3706,7 +3804,8 @@
+ return a == b;
+}
+
-+bool ref_record_equal(struct ref_record *a, struct ref_record *b, int hash_size)
++bool reftable_ref_record_equal(struct reftable_ref_record *a,
++ struct reftable_ref_record *b, int hash_size)
+{
+ assert(hash_size > 0);
+ return 0 == strcmp(a->ref_name, b->ref_name) &&
@@ -3716,22 +3815,22 @@
+ str_equal(a->target, b->target);
+}
+
-+int ref_record_compare_name(const void *a, const void *b)
++int reftable_ref_record_compare_name(const void *a, const void *b)
+{
-+ return strcmp(((struct ref_record *)a)->ref_name,
-+ ((struct ref_record *)b)->ref_name);
++ return strcmp(((struct reftable_ref_record *)a)->ref_name,
++ ((struct reftable_ref_record *)b)->ref_name);
+}
+
-+bool ref_record_is_deletion(const struct ref_record *ref)
++bool reftable_ref_record_is_deletion(const struct reftable_ref_record *ref)
+{
+ return ref->value == NULL && ref->target == NULL &&
+ ref->target_value == NULL;
+}
+
-+int log_record_compare_key(const void *a, const void *b)
++int reftable_log_record_compare_key(const void *a, const void *b)
+{
-+ struct log_record *la = (struct log_record *)a;
-+ struct log_record *lb = (struct log_record *)b;
++ struct reftable_log_record *la = (struct reftable_log_record *)a;
++ struct reftable_log_record *lb = (struct reftable_log_record *)b;
+
+ int cmp = strcmp(la->ref_name, lb->ref_name);
+ if (cmp) {
@@ -3743,10 +3842,12 @@
+ return (la->update_index < lb->update_index) ? 1 : 0;
+}
+
-+bool log_record_is_deletion(const struct log_record *log)
++bool reftable_log_record_is_deletion(const struct reftable_log_record *log)
+{
-+ /* XXX */
-+ return false;
++ return (log->new_hash == NULL && log->old_hash == NULL &&
++ log->name == NULL && log->email == NULL &&
++ log->message == NULL && log->time == 0 && log->tz_offset == 0 &&
++ log->message == NULL);
+}
+
+int hash_size(uint32_t id)
@@ -3780,14 +3881,33 @@
+#include "reftable.h"
+#include "slice.h"
+
++/* utilities for de/encoding varints */
++
++int get_var_int(uint64_t *dest, struct slice in);
++int put_var_int(struct slice dest, uint64_t val);
++
++/* Methods for records. */
+struct record_vtable {
++ /* encode the key of to a byte slice. */
+ void (*key)(const void *rec, struct slice *dest);
++
++ /* The record type of ('r' for ref). */
+ byte (*type)(void);
-+ void (*copy_from)(void *rec, const void *src, int hash_size);
++
++ void (*copy_from)(void *dest, const void *src, int hash_size);
++
++ /* a value of [0..7], indicating record subvariants (eg. ref vs. symref
++ * vs ref deletion) */
+ byte (*val_type)(const void *rec);
++
++ /* encodes rec into dest, returning how much space was used. */
+ int (*encode)(const void *rec, struct slice dest, int hash_size);
++
++ /* decode data from `src` into the record. */
+ int (*decode)(void *rec, struct slice key, byte extra, struct slice src,
+ int hash_size);
++
++ /* deallocate and null the record. */
+ void (*clear)(void *rec);
+};
+
@@ -3797,32 +3917,34 @@
+ struct record_vtable *ops;
+};
+
-+int get_var_int(uint64_t *dest, struct slice in);
-+int put_var_int(struct slice dest, uint64_t val);
-+int common_prefix_size(struct slice a, struct slice b);
-+
+int is_block_type(byte typ);
++
+struct record new_record(byte typ);
+
-+extern struct record_vtable ref_record_vtable;
++extern struct record_vtable reftable_ref_record_vtable;
+
+int encode_key(bool *restart, struct slice dest, struct slice prev_key,
+ struct slice key, byte extra);
+int decode_key(struct slice *key, byte *extra, struct slice last_key,
+ struct slice in);
+
++/* index_record are used internally to speed up lookups. */
+struct index_record {
-+ uint64_t offset;
-+ struct slice last_key;
++ uint64_t offset; /* Offset of block */
++ struct slice last_key; /* Last key of the block. */
+};
+
++/* obj_record stores an object ID => ref mapping. */
+struct obj_record {
-+ byte *hash_prefix;
-+ int hash_prefix_len;
-+ uint64_t *offsets;
++ byte *hash_prefix; /* leading bytes of the object ID */
++ int hash_prefix_len; /* number of leading bytes. Constant
++ * across a single table. */
++ uint64_t *offsets; /* a vector of file offsets. */
+ int offset_len;
+};
+
++/* see struct record_vtable */
++
+void record_key(struct record rec, struct slice *dest);
+byte record_type(struct record rec);
+void record_copy_from(struct record rec, struct record src, int hash_size);
@@ -3831,18 +3953,24 @@
+int record_decode(struct record rec, struct slice key, byte extra,
+ struct slice src, int hash_size);
+void record_clear(struct record rec);
++
++/* clear out the record, yielding the record data that was encapsulated. */
+void *record_yield(struct record *rec);
++
++/* initialize generic records from concrete records. The generic record should
++ * be zeroed out. */
++
+void record_from_obj(struct record *rec, struct obj_record *objrec);
+void record_from_index(struct record *rec, struct index_record *idxrec);
-+void record_from_ref(struct record *rec, struct ref_record *refrec);
-+void record_from_log(struct record *rec, struct log_record *logrec);
-+struct ref_record *record_as_ref(struct record ref);
++void record_from_ref(struct record *rec, struct reftable_ref_record *refrec);
++void record_from_log(struct record *rec, struct reftable_log_record *logrec);
++struct reftable_ref_record *record_as_ref(struct record ref);
+
+/* for qsort. */
-+int ref_record_compare_name(const void *a, const void *b);
++int reftable_ref_record_compare_name(const void *a, const void *b);
+
+/* for qsort. */
-+int log_record_compare_key(const void *a, const void *b);
++int reftable_log_record_compare_key(const void *a, const void *b);
+
+#endif
@@ -3863,65 +3991,20 @@
+#define REFTABLE_H
+
+#include <stdint.h>
++#include <stddef.h>
+
-+/* block_source is a generic wrapper for a seekable readable file.
-+ It is generally passed around by value.
-+ */
-+struct block_source {
-+ struct block_source_vtable *ops;
-+ void *arg;
-+};
++void reftable_set_alloc(void *(*malloc)(size_t),
++ void *(*realloc)(void *, size_t), void (*free)(void *));
+
-+/* a contiguous segment of bytes. It keeps track of its generating block_source
-+ so it can return itself into the pool.
-+*/
-+struct block {
-+ uint8_t *data;
-+ int len;
-+ struct block_source source;
-+};
++/****************************************************************
++ Basic data types
+
-+/* block_source_vtable are the operations that make up block_source */
-+struct block_source_vtable {
-+ /* returns the size of a block source */
-+ uint64_t (*size)(void *source);
++ Reftables store the state of each ref in struct reftable_ref_record, and they
++ store a sequence of reflog updates in struct reftable_log_record.
++ ****************************************************************/
+
-+ /* reads a segment from the block source. It is an error to read
-+ beyond the end of the block */
-+ int (*read_block)(void *source, struct block *dest, uint64_t off,
-+ uint32_t size);
-+ /* mark the block as read; may return the data back to malloc */
-+ void (*return_block)(void *source, struct block *blockp);
-+
-+ /* release all resources associated with the block source */
-+ void (*close)(void *source);
-+};
-+
-+/* opens a file on the file system as a block_source */
-+int block_source_from_file(struct block_source *block_src, const char *name);
-+
-+/* write_options sets options for writing a single reftable. */
-+struct write_options {
-+ /* boolean: do not pad out blocks to block size. */
-+ int unpadded;
-+
-+ /* the blocksize. Should be less than 2^24. */
-+ uint32_t block_size;
-+
-+ /* boolean: do not generate a SHA1 => ref index. */
-+ int skip_index_objects;
-+
-+ /* how often to write complete keys in each block. */
-+ int restart_interval;
-+
-+ /* 4-byte identifier ("sha1", "s256") of the hash.
-+ * Defaults to SHA1 if unset
-+ */
-+ uint32_t hash_id;
-+};
-+
-+/* ref_record holds a ref database entry target_value */
-+struct ref_record {
++/* reftable_ref_record holds a ref database entry target_value */
++struct reftable_ref_record {
+ char *ref_name; /* Name of the ref, malloced. */
+ uint64_t update_index; /* Logical timestamp at which this value is
+ written */
@@ -3931,21 +4014,23 @@
+};
+
+/* returns whether 'ref' represents a deletion */
-+int ref_record_is_deletion(const struct ref_record *ref);
++int reftable_ref_record_is_deletion(const struct reftable_ref_record *ref);
+
-+/* prints a ref_record onto stdout */
-+void ref_record_print(struct ref_record *ref, int hash_size);
++/* prints a reftable_ref_record onto stdout */
++void reftable_ref_record_print(struct reftable_ref_record *ref, int hash_size);
+
+/* frees and nulls all pointer values. */
-+void ref_record_clear(struct ref_record *ref);
++void reftable_ref_record_clear(struct reftable_ref_record *ref);
+
-+/* returns whether two ref_records are the same */
-+int ref_record_equal(struct ref_record *a, struct ref_record *b, int hash_size);
++/* returns whether two reftable_ref_records are the same */
++int reftable_ref_record_equal(struct reftable_ref_record *a,
++ struct reftable_ref_record *b, int hash_size);
+
-+/* log_record holds a reflog entry */
-+struct log_record {
++/* reftable_log_record holds a reflog entry */
++struct reftable_log_record {
+ char *ref_name;
-+ uint64_t update_index;
++ uint64_t update_index; /* logical timestamp of a transactional update.
++ */
+ uint8_t *new_hash;
+ uint8_t *old_hash;
+ char *name;
@@ -3956,39 +4041,88 @@
+};
+
+/* returns whether 'ref' represents the deletion of a log record. */
-+int log_record_is_deletion(const struct log_record *log);
++int reftable_log_record_is_deletion(const struct reftable_log_record *log);
+
+/* frees and nulls all pointer values. */
-+void log_record_clear(struct log_record *log);
++void reftable_log_record_clear(struct reftable_log_record *log);
+
+/* returns whether two records are equal. */
-+int log_record_equal(struct log_record *a, struct log_record *b, int hash_size);
++int reftable_log_record_equal(struct reftable_log_record *a,
++ struct reftable_log_record *b, int hash_size);
+
-+void log_record_print(struct log_record *log, int hash_size);
++/* dumps a reftable_log_record on stdout, for debugging/testing. */
++void reftable_log_record_print(struct reftable_log_record *log, int hash_size);
+
-+/* iterator is the generic interface for walking over data stored in a
-+ reftable. It is generally passed around by value.
-+*/
-+struct iterator {
-+ struct iterator_vtable *ops;
-+ void *iter_arg;
++/****************************************************************
++ Error handling
++
++ Error are signaled with negative integer return values. 0 means success.
++ ****************************************************************/
++
++/* different types of errors */
++enum reftable_error {
++ /* Unexpected file system behavior */
++ IO_ERROR = -2,
++
++ /* Format inconsistency on reading data
++ */
++ FORMAT_ERROR = -3,
++
++ /* File does not exist. Returned from block_source_from_file(), because
++ it needs special handling in stack.
++ */
++ NOT_EXIST_ERROR = -4,
++
++ /* Trying to write out-of-date data. */
++ LOCK_ERROR = -5,
++
++ /* Misuse of the API:
++ - on writing a record with NULL ref_name.
++ - on writing a reftable_ref_record outside the table limits
++ - on writing a ref or log record before the stack's next_update_index
++ - on reading a reftable_ref_record from log iterator, or vice versa.
++ */
++ API_ERROR = -6,
++
++ /* Decompression error */
++ ZLIB_ERROR = -7,
++
++ /* Wrote a table without blocks. */
++ EMPTY_TABLE_ERROR = -8,
+};
+
-+/* reads the next ref_record. Returns < 0 for error, 0 for OK and > 0:
-+ end of iteration.
-+*/
-+int iterator_next_ref(struct iterator it, struct ref_record *ref);
++/* convert the numeric error code to a string. The string should not be
++ * deallocated. */
++const char *reftable_error_str(int err);
+
-+/* reads the next log_record. Returns < 0 for error, 0 for OK and > 0:
-+ end of iteration.
-+*/
-+int iterator_next_log(struct iterator it, struct log_record *log);
++/****************************************************************
++ Writing
+
-+/* releases resources associated with an iterator. */
-+void iterator_destroy(struct iterator *it);
++ Writing single reftables
++ ****************************************************************/
++
++/* reftable_write_options sets options for writing a single reftable. */
++struct reftable_write_options {
++ /* boolean: do not pad out blocks to block size. */
++ int unpadded;
++
++ /* the blocksize. Should be less than 2^24. */
++ uint32_t block_size;
+
-+/* block_stats holds statistics for a single block type */
-+struct block_stats {
++ /* boolean: do not generate a SHA1 => ref index. */
++ int skip_index_objects;
++
++ /* how often to write complete keys in each block. */
++ int restart_interval;
++
++ /* 4-byte identifier ("sha1", "s256") of the hash.
++ * Defaults to SHA1 if unset
++ */
++ uint32_t hash_id;
++};
++
++/* reftable_block_stats holds statistics for a single block type */
++struct reftable_block_stats {
+ /* total number of entries written */
+ int entries;
+ /* total number of key restarts */
@@ -4008,119 +4142,175 @@
+};
+
+/* stats holds overall statistics for a single reftable */
-+struct stats {
++struct reftable_stats {
+ /* total number of blocks written. */
+ int blocks;
+ /* stats for ref data */
-+ struct block_stats ref_stats;
++ struct reftable_block_stats ref_stats;
+ /* stats for the SHA1 to ref map. */
-+ struct block_stats obj_stats;
++ struct reftable_block_stats obj_stats;
+ /* stats for index blocks */
-+ struct block_stats idx_stats;
++ struct reftable_block_stats idx_stats;
+ /* stats for log blocks */
-+ struct block_stats log_stats;
++ struct reftable_block_stats log_stats;
+
+ /* disambiguation length of shortened object IDs. */
+ int object_id_len;
+};
+
-+/* different types of errors */
-+
-+/* Unexpected file system behavior */
-+#define IO_ERROR -2
-+
-+/* Format inconsistency on reading data
-+ */
-+#define FORMAT_ERROR -3
-+
-+/* File does not exist. Returned from block_source_from_file(), because it
-+ needs special handling in stack.
-+*/
-+#define NOT_EXIST_ERROR -4
-+
-+/* Trying to write out-of-date data. */
-+#define LOCK_ERROR -5
-+
-+/* Misuse of the API:
-+ - on writing a record with NULL ref_name.
-+ - on writing a ref_record outside the table limits
-+ - on writing a ref or log record before the stack's next_update_index
-+ - on reading a ref_record from log iterator, or vice versa.
-+ */
-+#define API_ERROR -6
-+
-+/* Decompression error */
-+#define ZLIB_ERROR -7
-+
-+/* Wrote a table without blocks. */
-+#define EMPTY_TABLE_ERROR -8
-+
-+const char *error_str(int err);
-+
-+/* new_writer creates a new writer */
-+struct writer *new_writer(int (*writer_func)(void *, uint8_t *, int),
-+ void *writer_arg, struct write_options *opts);
++/* reftable_new_writer creates a new writer */
++struct reftable_writer *
++reftable_new_writer(int (*writer_func)(void *, uint8_t *, int),
++ void *writer_arg, struct reftable_write_options *opts);
+
+/* write to a file descriptor. fdp should be an int* pointing to the fd. */
-+int fd_writer(void *fdp, uint8_t *data, int size);
++int reftable_fd_write(void *fdp, uint8_t *data, int size);
+
+/* Set the range of update indices for the records we will add. When
+ writing a table into a stack, the min should be at least
-+ stack_next_update_index(), or API_ERROR is returned.
++ reftable_stack_next_update_index(), or API_ERROR is returned.
++
++ For transactional updates, typically min==max. When converting an existing
++ ref database into a single reftable, this would be a range of update-index
++ timestamps.
+ */
-+void writer_set_limits(struct writer *w, uint64_t min, uint64_t max);
++void reftable_writer_set_limits(struct reftable_writer *w, uint64_t min,
++ uint64_t max);
+
-+/* adds a ref_record. Must be called in ascending
++/* adds a reftable_ref_record. Must be called in ascending
+ order. The update_index must be within the limits set by
-+ writer_set_limits(), or API_ERROR is returned.
++ reftable_writer_set_limits(), or API_ERROR is returned.
++
++ It is an error to write a ref record after a log record.
+ */
-+int writer_add_ref(struct writer *w, struct ref_record *ref);
++int reftable_writer_add_ref(struct reftable_writer *w,
++ struct reftable_ref_record *ref);
+
+/* Convenience function to add multiple refs. Will sort the refs by
+ name before adding. */
-+int writer_add_refs(struct writer *w, struct ref_record *refs, int n);
++int reftable_writer_add_refs(struct reftable_writer *w,
++ struct reftable_ref_record *refs, int n);
+
-+/* adds a log_record. Must be called in ascending order (with more
++/* adds a reftable_log_record. Must be called in ascending order (with more
+ recent log entries first.)
+ */
-+int writer_add_log(struct writer *w, struct log_record *log);
++int reftable_writer_add_log(struct reftable_writer *w,
++ struct reftable_log_record *log);
+
+/* Convenience function to add multiple logs. Will sort the records by
+ key before adding. */
-+int writer_add_logs(struct writer *w, struct log_record *logs, int n);
++int reftable_writer_add_logs(struct reftable_writer *w,
++ struct reftable_log_record *logs, int n);
++
++/* reftable_writer_close finalizes the reftable. The writer is retained so
++ * statistics can be inspected. */
++int reftable_writer_close(struct reftable_writer *w);
++
++/* writer_stats returns the statistics on the reftable being written.
++
++ This struct becomes invalid when the writer is freed.
++ */
++const struct reftable_stats *writer_stats(struct reftable_writer *w);
++
++/* reftable_writer_free deallocates memory for the writer */
++void reftable_writer_free(struct reftable_writer *w);
++
++/****************************************************************
++ * ITERATING
++ ****************************************************************/
+
-+/* writer_close finalizes the reftable. The writer is retained so statistics can
-+ * be inspected. */
-+int writer_close(struct writer *w);
++/* iterator is the generic interface for walking over data stored in a
++ reftable. It is generally passed around by value.
++*/
++struct reftable_iterator {
++ struct reftable_iterator_vtable *ops;
++ void *iter_arg;
++};
+
-+/* writer_stats returns the statistics on the reftable being written. */
-+struct stats *writer_stats(struct writer *w);
++/* reads the next reftable_ref_record. Returns < 0 for error, 0 for OK and > 0:
++ end of iteration.
++*/
++int reftable_iterator_next_ref(struct reftable_iterator it,
++ struct reftable_ref_record *ref);
+
-+/* writer_free deallocates memory for the writer */
-+void writer_free(struct writer *w);
++/* reads the next reftable_log_record. Returns < 0 for error, 0 for OK and > 0:
++ end of iteration.
++*/
++int reftable_iterator_next_log(struct reftable_iterator it,
++ struct reftable_log_record *log);
+
-+struct reader;
++/* releases resources associated with an iterator. */
++void reftable_iterator_destroy(struct reftable_iterator *it);
++
++/****************************************************************
++ Reading single tables
+
-+/* new_reader opens a reftable for reading. If successful, returns 0
++ The follow routines are for reading single files. For an application-level
++ interface, skip ahead to struct reftable_merged_table and struct
++ reftable_stack.
++ ****************************************************************/
++
++/* block_source is a generic wrapper for a seekable readable file.
++ It is generally passed around by value.
++ */
++struct reftable_block_source {
++ struct reftable_block_source_vtable *ops;
++ void *arg;
++};
++
++/* a contiguous segment of bytes. It keeps track of its generating block_source
++ so it can return itself into the pool.
++*/
++struct reftable_block {
++ uint8_t *data;
++ int len;
++ struct reftable_block_source source;
++};
++
++/* block_source_vtable are the operations that make up block_source */
++struct reftable_block_source_vtable {
++ /* returns the size of a block source */
++ uint64_t (*size)(void *source);
++
++ /* reads a segment from the block source. It is an error to read
++ beyond the end of the block */
++ int (*read_block)(void *source, struct reftable_block *dest,
++ uint64_t off, uint32_t size);
++ /* mark the block as read; may return the data back to malloc */
++ void (*return_block)(void *source, struct reftable_block *blockp);
++
++ /* release all resources associated with the block source */
++ void (*close)(void *source);
++};
++
++/* opens a file on the file system as a block_source */
++int reftable_block_source_from_file(struct reftable_block_source *block_src,
++ const char *name);
++
++/* The reader struct is a handle to an open reftable file. */
++struct reftable_reader;
++
++/* reftable_new_reader opens a reftable for reading. If successful, returns 0
+ * code and sets pp. The name is used for creating a
+ * stack. Typically, it is the basename of the file.
+ */
-+int new_reader(struct reader **pp, struct block_source, const char *name);
++int reftable_new_reader(struct reftable_reader **pp,
++ struct reftable_block_source, const char *name);
+
-+/* reader_seek_ref returns an iterator where 'name' would be inserted in the
-+ table.
++/* reftable_reader_seek_ref returns an iterator where 'name' would be inserted
++ in the table. To seek to the start of the table, use name = "".
+
+ example:
+
-+ struct reader *r = NULL;
-+ int err = new_reader(&r, src, "filename");
++ struct reftable_reader *r = NULL;
++ int err = reftable_new_reader(&r, src, "filename");
+ if (err < 0) { ... }
-+ struct iterator it = {0};
-+ err = reader_seek_ref(r, &it, "refs/heads/master");
++ struct reftable_iterator it = {0};
++ err = reftable_reader_seek_ref(r, &it, "refs/heads/master");
+ if (err < 0) { ... }
-+ struct ref_record ref = {0};
++ struct reftable_ref_record ref = {0};
+ while (1) {
-+ err = iterator_next_ref(it, &ref);
++ err = reftable_iterator_next_ref(it, &ref);
+ if (err > 0) {
+ break;
+ }
@@ -4129,103 +4319,149 @@
+ }
+ ..found..
+ }
-+ iterator_destroy(&it);
-+ ref_record_clear(&ref);
++ reftable_iterator_destroy(&it);
++ reftable_ref_record_clear(&ref);
+ */
-+int reader_seek_ref(struct reader *r, struct iterator *it, const char *name);
++int reftable_reader_seek_ref(struct reftable_reader *r,
++ struct reftable_iterator *it, const char *name);
+
+/* returns the hash ID used in this table. */
-+uint32_t reader_hash_id(struct reader *r);
++uint32_t reftable_reader_hash_id(struct reftable_reader *r);
+
-+/* seek to logs for the given name, older than update_index. */
-+int reader_seek_log_at(struct reader *r, struct iterator *it, const char *name,
-+ uint64_t update_index);
++/* seek to logs for the given name, older than update_index. To seek to the
++ start of the table, use name = "".
++ */
++int reftable_reader_seek_log_at(struct reftable_reader *r,
++ struct reftable_iterator *it, const char *name,
++ uint64_t update_index);
+
+/* seek to newest log entry for given name. */
-+int reader_seek_log(struct reader *r, struct iterator *it, const char *name);
++int reftable_reader_seek_log(struct reftable_reader *r,
++ struct reftable_iterator *it, const char *name);
+
+/* closes and deallocates a reader. */
-+void reader_free(struct reader *);
++void reftable_reader_free(struct reftable_reader *);
+
+/* return an iterator for the refs pointing to oid */
-+int reader_refs_for(struct reader *r, struct iterator *it, uint8_t *oid,
-+ int oid_len);
++int reftable_reader_refs_for(struct reftable_reader *r,
++ struct reftable_iterator *it, uint8_t *oid,
++ int oid_len);
+
+/* return the max_update_index for a table */
-+uint64_t reader_max_update_index(struct reader *r);
++uint64_t reftable_reader_max_update_index(struct reftable_reader *r);
+
+/* return the min_update_index for a table */
-+uint64_t reader_min_update_index(struct reader *r);
++uint64_t reftable_reader_min_update_index(struct reftable_reader *r);
++
++/****************************************************************
++ Merged tables
+
-+/* a merged table is implements seeking/iterating over a stack of tables. */
-+struct merged_table;
++ A ref database kept in a sequence of table files. The merged_table presents a
++ unified view to reading (seeking, iterating) a sequence of immutable tables.
++ ****************************************************************/
+
-+/* new_merged_table creates a new merged table. It takes ownership of the stack
-+ array.
++/* A merged table is implements seeking/iterating over a stack of tables. */
++struct reftable_merged_table;
++
++/* reftable_new_merged_table creates a new merged table. It takes ownership of
++ the stack array.
+*/
-+int new_merged_table(struct merged_table **dest, struct reader **stack, int n,
-+ uint32_t hash_id);
++int reftable_new_merged_table(struct reftable_merged_table **dest,
++ struct reftable_reader **stack, int n,
++ uint32_t hash_id);
+
+/* returns the hash id used in this merged table. */
-+uint32_t merged_hash_id(struct merged_table *mt);
++uint32_t reftable_merged_table_hash_id(struct reftable_merged_table *mt);
+
+/* returns an iterator positioned just before 'name' */
-+int merged_table_seek_ref(struct merged_table *mt, struct iterator *it,
-+ const char *name);
++int reftable_merged_table_seek_ref(struct reftable_merged_table *mt,
++ struct reftable_iterator *it,
++ const char *name);
+
+/* returns an iterator for log entry, at given update_index */
-+int merged_table_seek_log_at(struct merged_table *mt, struct iterator *it,
-+ const char *name, uint64_t update_index);
++int reftable_merged_table_seek_log_at(struct reftable_merged_table *mt,
++ struct reftable_iterator *it,
++ const char *name, uint64_t update_index);
+
-+/* like merged_table_seek_log_at but look for the newest entry. */
-+int merged_table_seek_log(struct merged_table *mt, struct iterator *it,
-+ const char *name);
++/* like reftable_merged_table_seek_log_at but look for the newest entry. */
++int reftable_merged_table_seek_log(struct reftable_merged_table *mt,
++ struct reftable_iterator *it,
++ const char *name);
+
+/* returns the max update_index covered by this merged table. */
-+uint64_t merged_max_update_index(struct merged_table *mt);
++uint64_t
++reftable_merged_table_max_update_index(struct reftable_merged_table *mt);
+
+/* returns the min update_index covered by this merged table. */
-+uint64_t merged_min_update_index(struct merged_table *mt);
++uint64_t
++reftable_merged_table_min_update_index(struct reftable_merged_table *mt);
+
+/* closes readers for the merged tables */
-+void merged_table_close(struct merged_table *mt);
++void reftable_merged_table_close(struct reftable_merged_table *mt);
+
+/* releases memory for the merged_table */
-+void merged_table_free(struct merged_table *m);
++void reftable_merged_table_free(struct reftable_merged_table *m);
++
++/****************************************************************
++ Mutable ref database
++
++ The stack presents an interface to a mutable sequence of reftables.
++ ****************************************************************/
+
+/* a stack is a stack of reftables, which can be mutated by pushing a table to
+ * the top of the stack */
-+struct stack;
++struct reftable_stack;
+
-+/* open a new reftable stack. The tables will be stored in 'dir', while the list
-+ of tables is in 'list_file'. Typically, this should be .git/reftables and
-+ .git/refs respectively.
++/* open a new reftable stack. The tables along with the table list will be
++ stored in 'dir'. Typically, this should be .git/reftables.
+*/
-+int new_stack(struct stack **dest, const char *dir, const char *list_file,
-+ struct write_options config);
++int reftable_new_stack(struct reftable_stack **dest, const char *dir,
++ struct reftable_write_options config);
+
+/* returns the update_index at which a next table should be written. */
-+uint64_t stack_next_update_index(struct stack *st);
++uint64_t reftable_stack_next_update_index(struct reftable_stack *st);
++
++/* holds a transaction to add tables at the top of a stack. */
++struct reftable_addition;
++
++/*
++ returns a new transaction to add reftables to the given stack. As a side
++ effect, the ref database is locked.
++*/
++int reftable_stack_new_addition(struct reftable_addition **dest, struct reftable_stack *st);
++
++/* Adds a reftable to transaction. */
++int reftable_addition_add(struct reftable_addition *add,
++ int (*write_table)(struct reftable_writer *wr, void *arg),
++ void *arg);
++
++/* Commits the transaction, releasing the lock. */
++int reftable_addition_commit(struct reftable_addition *add);
++
++/* Release all non-committed data from the transaction; releases the lock if held. */
++void reftable_addition_close(struct reftable_addition *add);
+
+/* add a new table to the stack. The write_table function must call
-+ writer_set_limits, add refs and return an error value. */
-+int stack_add(struct stack *st,
-+ int (*write_table)(struct writer *wr, void *write_arg),
-+ void *write_arg);
++ reftable_writer_set_limits, add refs and return an error value. */
++int reftable_stack_add(struct reftable_stack *st,
++ int (*write_table)(struct reftable_writer *wr,
++ void *write_arg),
++ void *write_arg);
+
+/* returns the merged_table for seeking. This table is valid until the
+ next write or reload, and should not be closed or deleted.
+*/
-+struct merged_table *stack_merged_table(struct stack *st);
++struct reftable_merged_table *
++reftable_stack_merged_table(struct reftable_stack *st);
+
+/* frees all resources associated with the stack. */
-+void stack_destroy(struct stack *st);
++void reftable_stack_destroy(struct reftable_stack *st);
+
+/* reloads the stack if necessary. */
-+int stack_reload(struct stack *st);
++int reftable_stack_reload(struct reftable_stack *st);
+
+/* Policy for expiring reflog entries. */
-+struct log_expiry_config {
++struct reftable_log_expiry_config {
+ /* Drop entries older than this timestamp */
+ uint64_t time;
+
@@ -4235,29 +4471,32 @@
+
+/* compacts all reftables into a giant table. Expire reflog entries if config is
+ * non-NULL */
-+int stack_compact_all(struct stack *st, struct log_expiry_config *config);
++int reftable_stack_compact_all(struct reftable_stack *st,
++ struct reftable_log_expiry_config *config);
+
+/* heuristically compact unbalanced table stack. */
-+int stack_auto_compact(struct stack *st);
++int reftable_stack_auto_compact(struct reftable_stack *st);
+
+/* convenience function to read a single ref. Returns < 0 for error, 0
+ for success, and 1 if ref not found. */
-+int stack_read_ref(struct stack *st, const char *refname,
-+ struct ref_record *ref);
++int reftable_stack_read_ref(struct reftable_stack *st, const char *refname,
++ struct reftable_ref_record *ref);
+
+/* convenience function to read a single log. Returns < 0 for error, 0
+ for success, and 1 if ref not found. */
-+int stack_read_log(struct stack *st, const char *refname,
-+ struct log_record *log);
++int reftable_stack_read_log(struct reftable_stack *st, const char *refname,
++ struct reftable_log_record *log);
+
+/* statistics on past compactions. */
-+struct compaction_stats {
-+ uint64_t bytes;
-+ int attempts;
-+ int failures;
++struct reftable_compaction_stats {
++ uint64_t bytes; /* total number of bytes written */
++ int attempts; /* how often we tried to compact */
++ int failures; /* failures happen on concurrent updates */
+};
+
-+struct compaction_stats *stack_compaction_stats(struct stack *st);
++/* return statistics for compaction up till now. */
++struct reftable_compaction_stats *
++reftable_stack_compaction_stats(struct reftable_stack *st);
+
+#endif
@@ -4304,7 +4543,7 @@
+ c = l;
+ }
+ s->cap = c;
-+ s->buf = realloc(s->buf, s->cap);
++ s->buf = reftable_realloc(s->buf, s->cap);
+ }
+ s->len = l;
+}
@@ -4325,6 +4564,12 @@
+ memcpy(s->buf + end, a.buf, a.len);
+}
+
++void slice_consume(struct slice *s, int n)
++{
++ s->buf += n;
++ s->len -= n;
++}
++
+byte *slice_yield(struct slice *s)
+{
+ byte *p = s->buf;
@@ -4394,7 +4639,7 @@
+ if (newcap < b->len + sz) {
+ newcap = (b->len + sz);
+ }
-+ b->buf = realloc(b->buf, newcap);
++ b->buf = reftable_realloc(b->buf, newcap);
+ b->cap = newcap;
+ }
+
@@ -4413,57 +4658,71 @@
+ return ((struct slice *)b)->len;
+}
+
-+static void slice_return_block(void *b, struct block *dest)
++static void slice_return_block(void *b, struct reftable_block *dest)
+{
+ memset(dest->data, 0xff, dest->len);
-+ free(dest->data);
++ reftable_free(dest->data);
+}
+
+static void slice_close(void *b)
+{
+}
+
-+static int slice_read_block(void *v, struct block *dest, uint64_t off,
++static int slice_read_block(void *v, struct reftable_block *dest, uint64_t off,
+ uint32_t size)
+{
+ struct slice *b = (struct slice *)v;
+ assert(off + size <= b->len);
-+ dest->data = calloc(size, 1);
++ dest->data = reftable_calloc(size);
+ memcpy(dest->data, b->buf + off, size);
+ dest->len = size;
+ return size;
+}
+
-+struct block_source_vtable slice_vtable = {
++struct reftable_block_source_vtable slice_vtable = {
+ .size = &slice_size,
+ .read_block = &slice_read_block,
+ .return_block = &slice_return_block,
+ .close = &slice_close,
+};
+
-+void block_source_from_slice(struct block_source *bs, struct slice *buf)
++void block_source_from_slice(struct reftable_block_source *bs,
++ struct slice *buf)
+{
+ bs->ops = &slice_vtable;
+ bs->arg = buf;
+}
+
-+static void malloc_return_block(void *b, struct block *dest)
++static void malloc_return_block(void *b, struct reftable_block *dest)
+{
+ memset(dest->data, 0xff, dest->len);
-+ free(dest->data);
++ reftable_free(dest->data);
+}
+
-+struct block_source_vtable malloc_vtable = {
++struct reftable_block_source_vtable malloc_vtable = {
+ .return_block = &malloc_return_block,
+};
+
-+struct block_source malloc_block_source_instance = {
++struct reftable_block_source malloc_block_source_instance = {
+ .ops = &malloc_vtable,
+};
+
-+struct block_source malloc_block_source(void)
++struct reftable_block_source malloc_block_source(void)
+{
+ return malloc_block_source_instance;
++}
++
++int common_prefix_size(struct slice a, struct slice b)
++{
++ int p = 0;
++ while (p < a.len && p < b.len) {
++ if (a.buf[p] != b.buf[p]) {
++ break;
++ }
++ p++;
++ }
++
++ return p;
+}
diff --git a/reftable/slice.h b/reftable/slice.h
@@ -4498,16 +4757,19 @@
+bool slice_equal(struct slice a, struct slice b);
+byte *slice_yield(struct slice *s);
+void slice_copy(struct slice *dest, struct slice src);
++void slice_consume(struct slice *s, int n);
+void slice_resize(struct slice *s, int l);
+int slice_compare(struct slice a, struct slice b);
+int slice_write(struct slice *b, byte *data, int sz);
+int slice_write_void(void *b, byte *data, int sz);
+void slice_append(struct slice *dest, struct slice add);
++int common_prefix_size(struct slice a, struct slice b);
+
-+struct block_source;
-+void block_source_from_slice(struct block_source *bs, struct slice *buf);
++struct reftable_block_source;
++void block_source_from_slice(struct reftable_block_source *bs,
++ struct slice *buf);
+
-+struct block_source malloc_block_source(void);
++struct reftable_block_source malloc_block_source(void);
+
+#endif
@@ -4532,19 +4794,26 @@
+#include "reftable.h"
+#include "writer.h"
+
-+int new_stack(struct stack **dest, const char *dir, const char *list_file,
-+ struct write_options config)
++int reftable_new_stack(struct reftable_stack **dest, const char *dir,
++ struct reftable_write_options config)
+{
-+ struct stack *p = calloc(sizeof(struct stack), 1);
++ struct reftable_stack *p =
++ reftable_calloc(sizeof(struct reftable_stack));
++ struct slice list_file_name = {};
+ int err = 0;
+ *dest = NULL;
-+ p->list_file = xstrdup(list_file);
++
++ slice_set_string(&list_file_name, dir);
++ slice_append_string(&list_file_name, "/reftables.list");
++
++ p->list_file = slice_to_string(list_file_name);
++ reftable_free(slice_yield(&list_file_name));
+ p->reftable_dir = xstrdup(dir);
+ p->config = config;
+
-+ err = stack_reload(p);
++ err = reftable_stack_reload(p);
+ if (err < 0) {
-+ stack_destroy(p);
++ reftable_stack_destroy(p);
+ } else {
+ *dest = p;
+ }
@@ -4571,7 +4840,7 @@
+ goto exit;
+ }
+
-+ buf = malloc(size + 1);
++ buf = reftable_malloc(size + 1);
+ if (fread(buf, 1, size, f) != size) {
+ err = IO_ERROR;
+ goto exit;
@@ -4580,7 +4849,7 @@
+
+ parse_names(buf, size, namesp);
+exit:
-+ free(buf);
++ reftable_free(buf);
+ return err;
+}
+
@@ -4590,7 +4859,7 @@
+ int err = 0;
+ if (f == NULL) {
+ if (errno == ENOENT) {
-+ *namesp = calloc(sizeof(char *), 1);
++ *namesp = reftable_calloc(sizeof(char *));
+ return 0;
+ }
+
@@ -4601,30 +4870,33 @@
+ return err;
+}
+
-+struct merged_table *stack_merged_table(struct stack *st)
++struct reftable_merged_table *
++reftable_stack_merged_table(struct reftable_stack *st)
+{
+ return st->merged;
+}
+
+/* Close and free the stack */
-+void stack_destroy(struct stack *st)
++void reftable_stack_destroy(struct reftable_stack *st)
+{
+ if (st->merged == NULL) {
+ return;
+ }
+
-+ merged_table_close(st->merged);
-+ merged_table_free(st->merged);
++ reftable_merged_table_close(st->merged);
++ reftable_merged_table_free(st->merged);
+ st->merged = NULL;
+
+ FREE_AND_NULL(st->list_file);
+ FREE_AND_NULL(st->reftable_dir);
-+ free(st);
++ reftable_free(st);
+}
+
-+static struct reader **stack_copy_readers(struct stack *st, int cur_len)
++static struct reftable_reader **stack_copy_readers(struct reftable_stack *st,
++ int cur_len)
+{
-+ struct reader **cur = calloc(sizeof(struct reader *), cur_len);
++ struct reftable_reader **cur =
++ reftable_calloc(sizeof(struct reftable_reader *) * cur_len);
+ int i = 0;
+ for (i = 0; i < cur_len; i++) {
+ cur[i] = st->merged->stack[i];
@@ -4632,21 +4904,22 @@
+ return cur;
+}
+
-+static int stack_reload_once(struct stack *st, char **names, bool reuse_open)
++static int reftable_stack_reload_once(struct reftable_stack *st, char **names,
++ bool reuse_open)
+{
+ int cur_len = st->merged == NULL ? 0 : st->merged->stack_len;
-+ struct reader **cur = stack_copy_readers(st, cur_len);
++ struct reftable_reader **cur = stack_copy_readers(st, cur_len);
+ int err = 0;
+ int names_len = names_length(names);
-+ struct reader **new_tables =
-+ malloc(sizeof(struct reader *) * names_len);
++ struct reftable_reader **new_tables =
++ reftable_malloc(sizeof(struct reftable_reader *) * names_len);
+ int new_tables_len = 0;
-+ struct merged_table *new_merged = NULL;
++ struct reftable_merged_table *new_merged = NULL;
+
+ struct slice table_path = { 0 };
+
+ while (*names) {
-+ struct reader *rd = NULL;
++ struct reftable_reader *rd = NULL;
+ char *name = *names++;
+
+ /* this is linear; we assume compaction keeps the number of
@@ -4661,18 +4934,18 @@
+ }
+
+ if (rd == NULL) {
-+ struct block_source src = { 0 };
++ struct reftable_block_source src = { 0 };
+ slice_set_string(&table_path, st->reftable_dir);
+ slice_append_string(&table_path, "/");
+ slice_append_string(&table_path, name);
+
-+ err = block_source_from_file(
++ err = reftable_block_source_from_file(
+ &src, slice_as_string(&table_path));
+ if (err < 0) {
+ goto exit;
+ }
+
-+ err = new_reader(&rd, src, name);
++ err = reftable_new_reader(&rd, src, name);
+ if (err < 0) {
+ goto exit;
+ }
@@ -4682,8 +4955,8 @@
+ }
+
+ /* success! */
-+ err = new_merged_table(&new_merged, new_tables, new_tables_len,
-+ st->config.hash_id);
++ err = reftable_new_merged_table(&new_merged, new_tables, new_tables_len,
++ st->config.hash_id);
+ if (err < 0) {
+ goto exit;
+ }
@@ -4692,7 +4965,7 @@
+ new_tables_len = 0;
+ if (st->merged != NULL) {
+ merged_table_clear(st->merged);
-+ merged_table_free(st->merged);
++ reftable_merged_table_free(st->merged);
+ }
+ st->merged = new_merged;
+
@@ -4701,20 +4974,20 @@
+ for (i = 0; i < cur_len; i++) {
+ if (cur[i] != NULL) {
+ reader_close(cur[i]);
-+ reader_free(cur[i]);
++ reftable_reader_free(cur[i]);
+ }
+ }
+ }
+exit:
-+ free(slice_yield(&table_path));
++ reftable_free(slice_yield(&table_path));
+ {
+ int i = 0;
+ for (i = 0; i < new_tables_len; i++) {
+ reader_close(new_tables[i]);
+ }
+ }
-+ free(new_tables);
-+ free(cur);
++ reftable_free(new_tables);
++ reftable_free(cur);
+ return err;
+}
+
@@ -4731,7 +5004,8 @@
+ return udiff;
+}
+
-+static int stack_reload_maybe_reuse(struct stack *st, bool reuse_open)
++static int reftable_stack_reload_maybe_reuse(struct reftable_stack *st,
++ bool reuse_open)
+{
+ struct timeval deadline = { 0 };
+ int err = gettimeofday(&deadline, NULL);
@@ -4763,7 +5037,7 @@
+ free_names(names);
+ return err;
+ }
-+ err = stack_reload_once(st, names, reuse_open);
++ err = reftable_stack_reload_once(st, names, reuse_open);
+ if (err == 0) {
+ free_names(names);
+ break;
@@ -4794,15 +5068,15 @@
+ return 0;
+}
+
-+int stack_reload(struct stack *st)
++int reftable_stack_reload(struct reftable_stack *st)
+{
-+ return stack_reload_maybe_reuse(st, true);
++ return reftable_stack_reload_maybe_reuse(st, true);
+}
+
+/* -1 = error
+ 0 = up to date
+ 1 = changed. */
-+static int stack_uptodate(struct stack *st)
++static int stack_uptodate(struct reftable_stack *st)
+{
+ char **names = NULL;
+ int err = read_lines(st->list_file, &names);
@@ -4833,18 +5107,19 @@
+ return err;
+}
+
-+int stack_add(struct stack *st, int (*write)(struct writer *wr, void *arg),
-+ void *arg)
++int reftable_stack_add(struct reftable_stack *st,
++ int (*write)(struct reftable_writer *wr, void *arg),
++ void *arg)
+{
+ int err = stack_try_add(st, write, arg);
+ if (err < 0) {
+ if (err == LOCK_ERROR) {
-+ err = stack_reload(st);
++ err = reftable_stack_reload(st);
+ }
+ return err;
+ }
+
-+ return stack_auto_compact(st);
++ return reftable_stack_auto_compact(st);
+}
+
+static void format_name(struct slice *dest, uint64_t min, uint64_t max)
@@ -4854,34 +5129,35 @@
+ slice_set_string(dest, buf);
+}
+
-+int stack_try_add(struct stack *st,
-+ int (*write_table)(struct writer *wr, void *arg), void *arg)
++struct reftable_addition {
++ int lock_file_fd;
++ struct slice lock_file_name;
++ struct reftable_stack *stack;
++ char **names;
++ char **new_tables;
++ int new_tables_len;
++ uint64_t next_update_index;
++};
++
++static int reftable_stack_init_addition(struct reftable_addition *add,
++ struct reftable_stack *st)
+{
-+ struct slice lock_name = { 0 };
-+ struct slice temp_tab_name = { 0 };
-+ struct slice tab_name = { 0 };
-+ struct slice next_name = { 0 };
-+ struct slice table_list = { 0 };
-+ struct writer *wr = NULL;
+ int err = 0;
-+ int tab_fd = 0;
-+ int lock_fd = 0;
-+ uint64_t next_update_index = 0;
++ add->stack = st;
+
-+ slice_set_string(&lock_name, st->list_file);
-+ slice_append_string(&lock_name, ".lock");
++ slice_set_string(&add->lock_file_name, st->list_file);
++ slice_append_string(&add->lock_file_name, ".lock");
+
-+ lock_fd = open(slice_as_string(&lock_name), O_EXCL | O_CREAT | O_WRONLY,
-+ 0644);
-+ if (lock_fd < 0) {
++ add->lock_file_fd = open(slice_as_string(&add->lock_file_name),
++ O_EXCL | O_CREAT | O_WRONLY, 0644);
++ if (add->lock_file_fd < 0) {
+ if (errno == EEXIST) {
+ err = LOCK_ERROR;
-+ goto exit;
++ } else {
++ err = IO_ERROR;
+ }
-+ err = IO_ERROR;
+ goto exit;
+ }
-+
+ err = stack_uptodate(st);
+ if (err < 0) {
+ goto exit;
@@ -4892,29 +5168,160 @@
+ goto exit;
+ }
+
-+ next_update_index = stack_next_update_index(st);
++ add->next_update_index = reftable_stack_next_update_index(st);
++exit:
++ if (err) {
++ reftable_addition_close(add);
++ }
++ return err;
++}
++
++void reftable_addition_close(struct reftable_addition *add)
++{
++ int i = 0;
++ struct slice nm = {};
++ for (i = 0; i < add->new_tables_len; i++) {
++ slice_set_string(&nm, add->stack->list_file);
++ slice_append_string(&nm, "/");
++ slice_append_string(&nm, add->new_tables[i]);
++ unlink(slice_as_string(&nm));
++
++ reftable_free(add->new_tables[i]);
++ add->new_tables[i] = NULL;
++ }
++ reftable_free(add->new_tables);
++ add->new_tables = NULL;
++ add->new_tables_len = 0;
++
++ if (add->lock_file_fd > 0) {
++ close(add->lock_file_fd);
++ add->lock_file_fd = 0;
++ }
++ if (add->lock_file_name.len > 0) {
++ unlink(slice_as_string(&add->lock_file_name));
++ reftable_free(slice_yield(&add->lock_file_name));
++ }
++
++ free_names(add->names);
++ add->names = NULL;
++}
++
++int reftable_addition_commit(struct reftable_addition *add)
++{
++ struct slice table_list = { 0 };
++ int i = 0;
++ int err = 0;
++ if (add->new_tables_len == 0) {
++ goto exit;
++ }
++
++ for (i = 0; i < add->stack->merged->stack_len; i++) {
++ slice_append_string(&table_list,
++ add->stack->merged->stack[i]->name);
++ slice_append_string(&table_list, "\n");
++ }
++ for (i = 0; i < add->new_tables_len; i++) {
++ slice_append_string(&table_list, add->new_tables[i]);
++ slice_append_string(&table_list, "\n");
++ }
++
++ err = write(add->lock_file_fd, table_list.buf, table_list.len);
++ free(slice_yield(&table_list));
++ if (err < 0) {
++ err = IO_ERROR;
++ goto exit;
++ }
++
++ err = close(add->lock_file_fd);
++ add->lock_file_fd = 0;
++ if (err < 0) {
++ err = IO_ERROR;
++ goto exit;
++ }
++
++ err = rename(slice_as_string(&add->lock_file_name),
++ add->stack->list_file);
++ if (err < 0) {
++ err = IO_ERROR;
++ goto exit;
++ }
++
++ err = reftable_stack_reload(add->stack);
++
++exit:
++ reftable_addition_close(add);
++ return err;
++}
++
++int reftable_stack_new_addition(struct reftable_addition **dest,
++ struct reftable_stack *st)
++{
++ int err = 0;
++ *dest = reftable_malloc(sizeof(**dest));
++ err = reftable_stack_init_addition(*dest, st);
++ if (err) {
++ reftable_free(*dest);
++ *dest = NULL;
++ }
++ return err;
++}
++
++int stack_try_add(struct reftable_stack *st,
++ int (*write_table)(struct reftable_writer *wr, void *arg),
++ void *arg)
++{
++ struct reftable_addition add = { 0 };
++ int err = reftable_stack_init_addition(&add, st);
++ if (err < 0) {
++ goto exit;
++ }
++
++ err = reftable_addition_add(&add, write_table, arg);
++ if (err < 0) {
++ goto exit;
++ }
++
++ err = reftable_addition_commit(&add);
++exit:
++ reftable_addition_close(&add);
++ return err;
++}
++
++int reftable_addition_add(struct reftable_addition *add,
++ int (*write_table)(struct reftable_writer *wr,
++ void *arg),
++ void *arg)
++{
++ struct slice temp_tab_file_name = { 0 };
++ struct slice tab_file_name = { 0 };
++ struct slice next_name = { 0 };
++ struct reftable_writer *wr = NULL;
++ int err = 0;
++ int tab_fd = 0;
++ uint64_t next_update_index = 0;
+
+ slice_resize(&next_name, 0);
+ format_name(&next_name, next_update_index, next_update_index);
+
-+ slice_set_string(&temp_tab_name, st->reftable_dir);
-+ slice_append_string(&temp_tab_name, "/");
-+ slice_append(&temp_tab_name, next_name);
-+ slice_append_string(&temp_tab_name, ".temp.XXXXXX");
++ slice_set_string(&temp_tab_file_name, add->stack->reftable_dir);
++ slice_append_string(&temp_tab_file_name, "/");
++ slice_append(&temp_tab_file_name, next_name);
++ slice_append_string(&temp_tab_file_name, ".temp.XXXXXX");
+
-+ tab_fd = mkstemp((char *)slice_as_string(&temp_tab_name));
++ tab_fd = mkstemp((char *)slice_as_string(&temp_tab_file_name));
+ if (tab_fd < 0) {
+ err = IO_ERROR;
+ goto exit;
+ }
+
-+ wr = new_writer(fd_writer, &tab_fd, &st->config);
++ wr = reftable_new_writer(reftable_fd_write, &tab_fd,
++ &add->stack->config);
+ err = write_table(wr, arg);
+ if (err < 0) {
+ goto exit;
+ }
+
-+ err = writer_close(wr);
++ err = reftable_writer_close(wr);
+ if (err == EMPTY_TABLE_ERROR) {
+ err = 0;
+ goto exit;
@@ -4935,98 +5342,64 @@
+ goto exit;
+ }
+
-+ {
-+ int i = 0;
-+ for (i = 0; i < st->merged->stack_len; i++) {
-+ slice_append_string(&table_list,
-+ st->merged->stack[i]->name);
-+ slice_append_string(&table_list, "\n");
-+ }
-+ }
-+
+ format_name(&next_name, wr->min_update_index, wr->max_update_index);
+ slice_append_string(&next_name, ".ref");
-+ slice_append(&table_list, next_name);
-+ slice_append_string(&table_list, "\n");
+
-+ slice_set_string(&tab_name, st->reftable_dir);
-+ slice_append_string(&tab_name, "/");
-+ slice_append(&tab_name, next_name);
++ slice_set_string(&tab_file_name, add->stack->reftable_dir);
++ slice_append_string(&tab_file_name, "/");
++ slice_append(&tab_file_name, next_name);
+
-+ err = rename(slice_as_string(&temp_tab_name),
-+ slice_as_string(&tab_name));
++ err = rename(slice_as_string(&temp_tab_file_name),
++ slice_as_string(&tab_file_name));
+ if (err < 0) {
+ err = IO_ERROR;
+ goto exit;
+ }
-+ free(slice_yield(&temp_tab_name));
+
-+ err = write(lock_fd, table_list.buf, table_list.len);
-+ if (err < 0) {
-+ err = IO_ERROR;
-+ goto exit;
-+ }
-+ err = close(lock_fd);
-+ lock_fd = 0;
-+ if (err < 0) {
-+ unlink(slice_as_string(&tab_name));
-+ err = IO_ERROR;
-+ goto exit;
-+ }
-+
-+ err = rename(slice_as_string(&lock_name), st->list_file);
-+ if (err < 0) {
-+ unlink(slice_as_string(&tab_name));
-+ err = IO_ERROR;
-+ goto exit;
-+ }
-+
-+ err = stack_reload(st);
++ add->new_tables = reftable_realloc(add->new_tables,
++ sizeof(*add->new_tables) *
++ (add->new_tables_len + 1));
++ add->new_tables[add->new_tables_len] = slice_to_string(next_name);
++ add->new_tables_len++;
+exit:
+ if (tab_fd > 0) {
+ close(tab_fd);
+ tab_fd = 0;
+ }
-+ if (temp_tab_name.len > 0) {
-+ unlink(slice_as_string(&temp_tab_name));
-+ }
-+ unlink(slice_as_string(&lock_name));
-+
-+ if (lock_fd > 0) {
-+ close(lock_fd);
-+ lock_fd = 0;
++ if (temp_tab_file_name.len > 0) {
++ unlink(slice_as_string(&temp_tab_file_name));
+ }
+
-+ free(slice_yield(&lock_name));
-+ free(slice_yield(&temp_tab_name));
-+ free(slice_yield(&tab_name));
-+ free(slice_yield(&next_name));
-+ free(slice_yield(&table_list));
-+ writer_free(wr);
++ reftable_free(slice_yield(&temp_tab_file_name));
++ reftable_free(slice_yield(&tab_file_name));
++ reftable_free(slice_yield(&next_name));
++ reftable_writer_free(wr);
+ return err;
+}
+
-+uint64_t stack_next_update_index(struct stack *st)
++uint64_t reftable_stack_next_update_index(struct reftable_stack *st)
+{
+ int sz = st->merged->stack_len;
+ if (sz > 0) {
-+ return reader_max_update_index(st->merged->stack[sz - 1]) + 1;
++ return reftable_reader_max_update_index(
++ st->merged->stack[sz - 1]) +
++ 1;
+ }
+ return 1;
+}
+
-+static int stack_compact_locked(struct stack *st, int first, int last,
++static int stack_compact_locked(struct reftable_stack *st, int first, int last,
+ struct slice *temp_tab,
-+ struct log_expiry_config *config)
++ struct reftable_log_expiry_config *config)
+{
+ struct slice next_name = { 0 };
+ int tab_fd = -1;
-+ struct writer *wr = NULL;
++ struct reftable_writer *wr = NULL;
+ int err = 0;
+
+ format_name(&next_name,
-+ reader_min_update_index(st->merged->stack[first]),
-+ reader_max_update_index(st->merged->stack[first]));
++ reftable_reader_min_update_index(st->merged->stack[first]),
++ reftable_reader_max_update_index(st->merged->stack[first]));
+
+ slice_set_string(temp_tab, st->reftable_dir);
+ slice_append_string(temp_tab, "/");
@@ -5034,17 +5407,17 @@
+ slice_append_string(temp_tab, ".temp.XXXXXX");
+
+ tab_fd = mkstemp((char *)slice_as_string(temp_tab));
-+ wr = new_writer(fd_writer, &tab_fd, &st->config);
++ wr = reftable_new_writer(reftable_fd_write, &tab_fd, &st->config);
+
+ err = stack_write_compact(st, wr, first, last, config);
+ if (err < 0) {
+ goto exit;
+ }
-+ err = writer_close(wr);
++ err = reftable_writer_close(wr);
+ if (err < 0) {
+ goto exit;
+ }
-+ writer_free(wr);
++ reftable_writer_free(wr);
+
+ err = close(tab_fd);
+ tab_fd = 0;
@@ -5056,46 +5429,49 @@
+ }
+ if (err != 0 && temp_tab->len > 0) {
+ unlink(slice_as_string(temp_tab));
-+ free(slice_yield(temp_tab));
++ reftable_free(slice_yield(temp_tab));
+ }
-+ free(slice_yield(&next_name));
++ reftable_free(slice_yield(&next_name));
+ return err;
+}
+
-+int stack_write_compact(struct stack *st, struct writer *wr, int first,
-+ int last, struct log_expiry_config *config)
++int stack_write_compact(struct reftable_stack *st, struct reftable_writer *wr,
++ int first, int last,
++ struct reftable_log_expiry_config *config)
+{
+ int subtabs_len = last - first + 1;
-+ struct reader **subtabs =
-+ calloc(sizeof(struct reader *), last - first + 1);
-+ struct merged_table *mt = NULL;
++ struct reftable_reader **subtabs = reftable_calloc(
++ sizeof(struct reftable_reader *) * (last - first + 1));
++ struct reftable_merged_table *mt = NULL;
+ int err = 0;
-+ struct iterator it = { 0 };
-+ struct ref_record ref = { 0 };
-+ struct log_record log = { 0 };
++ struct reftable_iterator it = { 0 };
++ struct reftable_ref_record ref = { 0 };
++ struct reftable_log_record log = { 0 };
+
+ int i = 0, j = 0;
+ for (i = first, j = 0; i <= last; i++) {
-+ struct reader *t = st->merged->stack[i];
++ struct reftable_reader *t = st->merged->stack[i];
+ subtabs[j++] = t;
+ st->stats.bytes += t->size;
+ }
-+ writer_set_limits(wr, st->merged->stack[first]->min_update_index,
-+ st->merged->stack[last]->max_update_index);
++ reftable_writer_set_limits(wr,
++ st->merged->stack[first]->min_update_index,
++ st->merged->stack[last]->max_update_index);
+
-+ err = new_merged_table(&mt, subtabs, subtabs_len, st->config.hash_id);
++ err = reftable_new_merged_table(&mt, subtabs, subtabs_len,
++ st->config.hash_id);
+ if (err < 0) {
-+ free(subtabs);
++ reftable_free(subtabs);
+ goto exit;
+ }
+
-+ err = merged_table_seek_ref(mt, &it, "");
++ err = reftable_merged_table_seek_ref(mt, &it, "");
+ if (err < 0) {
+ goto exit;
+ }
+
+ while (true) {
-+ err = iterator_next_ref(it, &ref);
++ err = reftable_iterator_next_ref(it, &ref);
+ if (err > 0) {
+ err = 0;
+ break;
@@ -5103,23 +5479,23 @@
+ if (err < 0) {
+ break;
+ }
-+ if (first == 0 && ref_record_is_deletion(&ref)) {
++ if (first == 0 && reftable_ref_record_is_deletion(&ref)) {
+ continue;
+ }
+
-+ err = writer_add_ref(wr, &ref);
++ err = reftable_writer_add_ref(wr, &ref);
+ if (err < 0) {
+ break;
+ }
+ }
+
-+ err = merged_table_seek_log(mt, &it, "");
++ err = reftable_merged_table_seek_log(mt, &it, "");
+ if (err < 0) {
+ goto exit;
+ }
+
+ while (true) {
-+ err = iterator_next_log(it, &log);
++ err = reftable_iterator_next_log(it, &log);
+ if (err > 0) {
+ err = 0;
+ break;
@@ -5127,7 +5503,7 @@
+ if (err < 0) {
+ break;
+ }
-+ if (first == 0 && log_record_is_deletion(&log)) {
++ if (first == 0 && reftable_log_record_is_deletion(&log)) {
+ continue;
+ }
+
@@ -5143,28 +5519,28 @@
+ continue;
+ }
+
-+ err = writer_add_log(wr, &log);
++ err = reftable_writer_add_log(wr, &log);
+ if (err < 0) {
+ break;
+ }
+ }
+
+exit:
-+ iterator_destroy(&it);
++ reftable_iterator_destroy(&it);
+ if (mt != NULL) {
+ merged_table_clear(mt);
-+ merged_table_free(mt);
++ reftable_merged_table_free(mt);
+ }
-+ ref_record_clear(&ref);
++ reftable_ref_record_clear(&ref);
+
+ return err;
+}
+
+/* < 0: error. 0 == OK, > 0 attempt failed; could retry. */
-+static int stack_compact_range(struct stack *st, int first, int last,
-+ struct log_expiry_config *expiry)
++static int stack_compact_range(struct reftable_stack *st, int first, int last,
++ struct reftable_log_expiry_config *expiry)
+{
-+ struct slice temp_tab_name = { 0 };
++ struct slice temp_tab_file_name = { 0 };
+ struct slice new_table_name = { 0 };
+ struct slice lock_file_name = { 0 };
+ struct slice ref_list_contents = { 0 };
@@ -5173,8 +5549,10 @@
+ bool have_lock = false;
+ int lock_file_fd = 0;
+ int compact_count = last - first + 1;
-+ char **delete_on_success = calloc(sizeof(char *), compact_count + 1);
-+ char **subtable_locks = calloc(sizeof(char *), compact_count + 1);
++ char **delete_on_success =
++ reftable_calloc(sizeof(char *) * (compact_count + 1));
++ char **subtable_locks =
++ reftable_calloc(sizeof(char *) * (compact_count + 1));
+ int i = 0;
+ int j = 0;
+ bool is_empty_table = false;
@@ -5206,14 +5584,14 @@
+ }
+
+ for (i = first, j = 0; i <= last; i++) {
-+ struct slice subtab_name = { 0 };
++ struct slice subtab_file_name = { 0 };
+ struct slice subtab_lock = { 0 };
-+ slice_set_string(&subtab_name, st->reftable_dir);
-+ slice_append_string(&subtab_name, "/");
-+ slice_append_string(&subtab_name,
++ slice_set_string(&subtab_file_name, st->reftable_dir);
++ slice_append_string(&subtab_file_name, "/");
++ slice_append_string(&subtab_file_name,
+ reader_name(st->merged->stack[i]));
+
-+ slice_copy(&subtab_lock, subtab_name);
++ slice_copy(&subtab_lock, subtab_file_name);
+ slice_append_string(&subtab_lock, ".lock");
+
+ {
@@ -5231,7 +5609,8 @@
+ }
+
+ subtable_locks[j] = (char *)slice_as_string(&subtab_lock);
-+ delete_on_success[j] = (char *)slice_as_string(&subtab_name);
++ delete_on_success[j] =
++ (char *)slice_as_string(&subtab_file_name);
+ j++;
+
+ if (err != 0) {
@@ -5245,7 +5624,8 @@
+ }
+ have_lock = false;
+
-+ err = stack_compact_locked(st, first, last, &temp_tab_name, expiry);
++ err = stack_compact_locked(st, first, last, &temp_tab_file_name,
++ expiry);
+ /* Compaction + tombstones can create an empty table out of non-empty
+ * tables. */
+ is_empty_table = (err == EMPTY_TABLE_ERROR);
@@ -5278,7 +5658,7 @@
+ slice_append(&new_table_path, new_table_name);
+
+ if (!is_empty_table) {
-+ err = rename(slice_as_string(&temp_tab_name),
++ err = rename(slice_as_string(&temp_tab_file_name),
+ slice_as_string(&new_table_path));
+ if (err < 0) {
+ goto exit;
@@ -5329,7 +5709,7 @@
+ }
+ }
+
-+ err = stack_reload_maybe_reuse(st, first < last);
++ err = reftable_stack_reload_maybe_reuse(st, first < last);
+exit:
+ free_names(delete_on_success);
+ {
@@ -5347,21 +5727,23 @@
+ if (have_lock) {
+ unlink(slice_as_string(&lock_file_name));
+ }
-+ free(slice_yield(&new_table_name));
-+ free(slice_yield(&new_table_path));
-+ free(slice_yield(&ref_list_contents));
-+ free(slice_yield(&temp_tab_name));
-+ free(slice_yield(&lock_file_name));
++ reftable_free(slice_yield(&new_table_name));
++ reftable_free(slice_yield(&new_table_path));
++ reftable_free(slice_yield(&ref_list_contents));
++ reftable_free(slice_yield(&temp_tab_file_name));
++ reftable_free(slice_yield(&lock_file_name));
+ return err;
+}
+
-+int stack_compact_all(struct stack *st, struct log_expiry_config *config)
++int reftable_stack_compact_all(struct reftable_stack *st,
++ struct reftable_log_expiry_config *config)
+{
+ return stack_compact_range(st, 0, st->merged->stack_len - 1, config);
+}
+
-+static int stack_compact_range_stats(struct stack *st, int first, int last,
-+ struct log_expiry_config *config)
++static int stack_compact_range_stats(struct reftable_stack *st, int first,
++ int last,
++ struct reftable_log_expiry_config *config)
+{
+ int err = stack_compact_range(st, first, last, config);
+ if (err > 0) {
@@ -5387,7 +5769,7 @@
+
+struct segment *sizes_to_segments(int *seglen, uint64_t *sizes, int n)
+{
-+ struct segment *segs = calloc(sizeof(struct segment), n);
++ struct segment *segs = reftable_calloc(sizeof(struct segment) * n);
+ int next = 0;
+ struct segment cur = { 0 };
+ int i = 0;
@@ -5406,8 +5788,9 @@
+ cur.end = i + 1;
+ cur.bytes += sizes[i];
+ }
-+
-+ segs[next++] = cur;
++ if (next > 0) {
++ segs[next++] = cur;
++ }
+ *seglen = next;
+ return segs;
+}
@@ -5440,13 +5823,14 @@
+ min_seg.bytes += sizes[prev];
+ }
+
-+ free(segs);
++ reftable_free(segs);
+ return min_seg;
+}
+
-+static uint64_t *stack_table_sizes_for_compaction(struct stack *st)
++static uint64_t *stack_table_sizes_for_compaction(struct reftable_stack *st)
+{
-+ uint64_t *sizes = calloc(sizeof(uint64_t), st->merged->stack_len);
++ uint64_t *sizes =
++ reftable_calloc(sizeof(uint64_t) * st->merged->stack_len);
+ int i = 0;
+ for (i = 0; i < st->merged->stack_len; i++) {
+ /* overhead is 24 + 68 = 92. */
@@ -5455,12 +5839,12 @@
+ return sizes;
+}
+
-+int stack_auto_compact(struct stack *st)
++int reftable_stack_auto_compact(struct reftable_stack *st)
+{
+ uint64_t *sizes = stack_table_sizes_for_compaction(st);
+ struct segment seg =
+ suggest_compaction_segment(sizes, st->merged->stack_len);
-+ free(sizes);
++ reftable_free(sizes);
+ if (segment_size(&seg) > 0) {
+ return stack_compact_range_stats(st, seg.start, seg.end - 1,
+ NULL);
@@ -5469,58 +5853,61 @@
+ return 0;
+}
+
-+struct compaction_stats *stack_compaction_stats(struct stack *st)
++struct reftable_compaction_stats *
++reftable_stack_compaction_stats(struct reftable_stack *st)
+{
+ return &st->stats;
+}
+
-+int stack_read_ref(struct stack *st, const char *refname,
-+ struct ref_record *ref)
++int reftable_stack_read_ref(struct reftable_stack *st, const char *refname,
++ struct reftable_ref_record *ref)
+{
-+ struct iterator it = { 0 };
-+ struct merged_table *mt = stack_merged_table(st);
-+ int err = merged_table_seek_ref(mt, &it, refname);
++ struct reftable_iterator it = { 0 };
++ struct reftable_merged_table *mt = reftable_stack_merged_table(st);
++ int err = reftable_merged_table_seek_ref(mt, &it, refname);
+ if (err) {
+ goto exit;
+ }
+
-+ err = iterator_next_ref(it, ref);
++ err = reftable_iterator_next_ref(it, ref);
+ if (err) {
+ goto exit;
+ }
+
-+ if (strcmp(ref->ref_name, refname) || ref_record_is_deletion(ref)) {
++ if (strcmp(ref->ref_name, refname) ||
++ reftable_ref_record_is_deletion(ref)) {
+ err = 1;
+ goto exit;
+ }
+
+exit:
-+ iterator_destroy(&it);
++ reftable_iterator_destroy(&it);
+ return err;
+}
+
-+int stack_read_log(struct stack *st, const char *refname,
-+ struct log_record *log)
++int reftable_stack_read_log(struct reftable_stack *st, const char *refname,
++ struct reftable_log_record *log)
+{
-+ struct iterator it = { 0 };
-+ struct merged_table *mt = stack_merged_table(st);
-+ int err = merged_table_seek_log(mt, &it, refname);
++ struct reftable_iterator it = { 0 };
++ struct reftable_merged_table *mt = reftable_stack_merged_table(st);
++ int err = reftable_merged_table_seek_log(mt, &it, refname);
+ if (err) {
+ goto exit;
+ }
+
-+ err = iterator_next_log(it, log);
++ err = reftable_iterator_next_log(it, log);
+ if (err) {
+ goto exit;
+ }
+
-+ if (strcmp(log->ref_name, refname) || log_record_is_deletion(log)) {
++ if (strcmp(log->ref_name, refname) ||
++ reftable_log_record_is_deletion(log)) {
+ err = 1;
+ goto exit;
+ }
+
+exit:
-+ iterator_destroy(&it);
++ reftable_iterator_destroy(&it);
+ return err;
+}
@@ -5542,21 +5929,23 @@
+
+#include "reftable.h"
+
-+struct stack {
++struct reftable_stack {
+ char *list_file;
+ char *reftable_dir;
+
-+ struct write_options config;
++ struct reftable_write_options config;
+
-+ struct merged_table *merged;
-+ struct compaction_stats stats;
++ struct reftable_merged_table *merged;
++ struct reftable_compaction_stats stats;
+};
+
+int read_lines(const char *filename, char ***lines);
-+int stack_try_add(struct stack *st,
-+ int (*write_table)(struct writer *wr, void *arg), void *arg);
-+int stack_write_compact(struct stack *st, struct writer *wr, int first,
-+ int last, struct log_expiry_config *config);
++int stack_try_add(struct reftable_stack *st,
++ int (*write_table)(struct reftable_writer *wr, void *arg),
++ void *arg);
++int stack_write_compact(struct reftable_stack *st, struct reftable_writer *wr,
++ int first, int last,
++ struct reftable_log_expiry_config *config);
+int fastlog2(uint64_t sz);
+
+struct segment {
@@ -5644,6 +6033,7 @@
+
+#include "tree.h"
+
++#include "basics.h"
+#include "system.h"
+
+struct tree_node *tree_search(void *key, struct tree_node **rootp,
@@ -5655,7 +6045,7 @@
+ return NULL;
+ } else {
+ struct tree_node *n =
-+ calloc(sizeof(struct tree_node), 1);
++ reftable_calloc(sizeof(struct tree_node));
+ n->key = key;
+ *rootp = n;
+ return *rootp;
@@ -5698,7 +6088,7 @@
+ if (t->right != NULL) {
+ tree_free(t->right);
+ }
-+ free(t);
++ reftable_free(t);
+}
diff --git a/reftable/tree.h b/reftable/tree.h
@@ -5743,6 +6133,7 @@
+((cd reftable-repo && git fetch origin && git checkout origin/master ) ||
+git clone https://github.com/google/reftable reftable-repo) && \
+cp reftable-repo/c/*.[ch] reftable/ && \
++cp reftable-repo/c/include/*.[ch] reftable/ && \
+cp reftable-repo/LICENSE reftable/ &&
+git --git-dir reftable-repo/.git show --no-patch origin/master \
+> reftable/VERSION && \
@@ -5773,7 +6164,8 @@
+#include "reftable.h"
+#include "tree.h"
+
-+static struct block_stats *writer_block_stats(struct writer *w, byte typ)
++static struct reftable_block_stats *
++writer_reftable_block_stats(struct reftable_writer *w, byte typ)
+{
+ switch (typ) {
+ case 'r':
@@ -5791,18 +6183,19 @@
+
+/* write data, queuing the padding for the next write. Returns negative for
+ * error. */
-+static int padded_write(struct writer *w, byte *data, size_t len, int padding)
++static int padded_write(struct reftable_writer *w, byte *data, size_t len,
++ int padding)
+{
+ int n = 0;
+ if (w->pending_padding > 0) {
-+ byte *zeroed = calloc(w->pending_padding, 1);
++ byte *zeroed = reftable_calloc(w->pending_padding);
+ int n = w->write(w->write_arg, zeroed, w->pending_padding);
+ if (n < 0) {
+ return n;
+ }
+
+ w->pending_padding = 0;
-+ free(zeroed);
++ reftable_free(zeroed);
+ }
+
+ w->pending_padding = padding;
@@ -5814,7 +6207,7 @@
+ return 0;
+}
+
-+static void options_set_defaults(struct write_options *opts)
++static void options_set_defaults(struct reftable_write_options *opts)
+{
+ if (opts->restart_interval == 0) {
+ opts->restart_interval = 16;
@@ -5828,25 +6221,31 @@
+ }
+}
+
-+static int writer_write_header(struct writer *w, byte *dest)
++static int writer_version(struct reftable_writer *w)
++{
++ return (w->opts.hash_id == 0 || w->opts.hash_id == SHA1_ID) ? 1 : 2;
++}
++
++static int writer_write_header(struct reftable_writer *w, byte *dest)
+{
+ memcpy((char *)dest, "REFT", 4);
+
-+ /* DO NOT SUBMIT. This has not been encoded in the standard yet. */
-+ dest[4] = (hash_size(w->opts.hash_id) == SHA1_SIZE) ? 1 : 2; /* version
-+ */
++ dest[4] = writer_version(w);
+
+ put_be24(dest + 5, w->opts.block_size);
+ put_be64(dest + 8, w->min_update_index);
+ put_be64(dest + 16, w->max_update_index);
-+ return 24;
++ if (writer_version(w) == 2) {
++ put_be32(dest + 24, w->opts.hash_id);
++ }
++ return header_size(writer_version(w));
+}
+
-+static void writer_reinit_block_writer(struct writer *w, byte typ)
++static void writer_reinit_block_writer(struct reftable_writer *w, byte typ)
+{
+ int block_start = 0;
+ if (w->next == 0) {
-+ block_start = HEADER_SIZE;
++ block_start = header_size(writer_version(w));
+ }
+
+ block_writer_init(&w->block_writer_data, typ, w->block,
@@ -5856,16 +6255,18 @@
+ w->block_writer->restart_interval = w->opts.restart_interval;
+}
+
-+struct writer *new_writer(int (*writer_func)(void *, byte *, int),
-+ void *writer_arg, struct write_options *opts)
++struct reftable_writer *
++reftable_new_writer(int (*writer_func)(void *, byte *, int), void *writer_arg,
++ struct reftable_write_options *opts)
+{
-+ struct writer *wp = calloc(sizeof(struct writer), 1);
++ struct reftable_writer *wp =
++ reftable_calloc(sizeof(struct reftable_writer));
+ options_set_defaults(opts);
+ if (opts->block_size >= (1 << 24)) {
+ /* TODO - error return? */
+ abort();
+ }
-+ wp->block = calloc(opts->block_size, 1);
++ wp->block = reftable_calloc(opts->block_size);
+ wp->write = writer_func;
+ wp->write_arg = writer_arg;
+ wp->opts = *opts;
@@ -5874,16 +6275,17 @@
+ return wp;
+}
+
-+void writer_set_limits(struct writer *w, uint64_t min, uint64_t max)
++void reftable_writer_set_limits(struct reftable_writer *w, uint64_t min,
++ uint64_t max)
+{
+ w->min_update_index = min;
+ w->max_update_index = max;
+}
+
-+void writer_free(struct writer *w)
++void reftable_writer_free(struct reftable_writer *w)
+{
-+ free(w->block);
-+ free(w);
++ reftable_free(w->block);
++ reftable_free(w);
+}
+
+struct obj_index_tree_node {
@@ -5899,7 +6301,7 @@
+ ((const struct obj_index_tree_node *)b)->hash);
+}
+
-+static void writer_index_hash(struct writer *w, struct slice hash)
++static void writer_index_hash(struct reftable_writer *w, struct slice hash)
+{
+ uint64_t off = w->next;
+
@@ -5909,7 +6311,7 @@
+ &obj_index_tree_node_compare, 0);
+ struct obj_index_tree_node *key = NULL;
+ if (node == NULL) {
-+ key = calloc(sizeof(struct obj_index_tree_node), 1);
++ key = reftable_calloc(sizeof(struct obj_index_tree_node));
+ slice_copy(&key->hash, hash);
+ tree_search((void *)key, &w->obj_index_tree,
+ &obj_index_tree_node_compare, 1);
@@ -5923,14 +6325,14 @@
+
+ if (key->offset_len == key->offset_cap) {
+ key->offset_cap = 2 * key->offset_cap + 1;
-+ key->offsets = realloc(key->offsets,
-+ sizeof(uint64_t) * key->offset_cap);
++ key->offsets = reftable_realloc(
++ key->offsets, sizeof(uint64_t) * key->offset_cap);
+ }
+
+ key->offsets[key->offset_len++] = off;
+}
+
-+static int writer_add_record(struct writer *w, struct record rec)
++static int writer_add_record(struct reftable_writer *w, struct record rec)
+{
+ int result = -1;
+ struct slice key = { 0 };
@@ -5967,14 +6369,15 @@
+
+ result = 0;
+exit:
-+ free(slice_yield(&key));
++ reftable_free(slice_yield(&key));
+ return result;
+}
+
-+int writer_add_ref(struct writer *w, struct ref_record *ref)
++int reftable_writer_add_ref(struct reftable_writer *w,
++ struct reftable_ref_record *ref)
+{
+ struct record rec = { 0 };
-+ struct ref_record copy = *ref;
++ struct reftable_ref_record copy = *ref;
+ int err = 0;
+
+ if (ref->ref_name == NULL) {
@@ -6010,18 +6413,20 @@
+ return 0;
+}
+
-+int writer_add_refs(struct writer *w, struct ref_record *refs, int n)
++int reftable_writer_add_refs(struct reftable_writer *w,
++ struct reftable_ref_record *refs, int n)
+{
+ int err = 0;
+ int i = 0;
-+ QSORT(refs, n, ref_record_compare_name);
++ QSORT(refs, n, reftable_ref_record_compare_name);
+ for (i = 0; err == 0 && i < n; i++) {
-+ err = writer_add_ref(w, &refs[i]);
++ err = reftable_writer_add_ref(w, &refs[i]);
+ }
+ return err;
+}
+
-+int writer_add_log(struct writer *w, struct log_record *log)
++int reftable_writer_add_log(struct reftable_writer *w,
++ struct reftable_log_record *log)
+{
+ if (log->ref_name == NULL) {
+ return API_ERROR;
@@ -6047,18 +6452,19 @@
+ }
+}
+
-+int writer_add_logs(struct writer *w, struct log_record *logs, int n)
++int reftable_writer_add_logs(struct reftable_writer *w,
++ struct reftable_log_record *logs, int n)
+{
+ int err = 0;
+ int i = 0;
-+ QSORT(logs, n, log_record_compare_key);
++ QSORT(logs, n, reftable_log_record_compare_key);
+ for (i = 0; err == 0 && i < n; i++) {
-+ err = writer_add_log(w, &logs[i]);
++ err = reftable_writer_add_log(w, &logs[i]);
+ }
+ return err;
+}
+
-+static int writer_finish_section(struct writer *w)
++static int writer_finish_section(struct reftable_writer *w)
+{
+ byte typ = block_writer_type(w->block_writer);
+ uint64_t index_start = 0;
@@ -6105,9 +6511,9 @@
+ assert(err == 0);
+ }
+ for (i = 0; i < idx_len; i++) {
-+ free(slice_yield(&idx[i].last_key));
++ reftable_free(slice_yield(&idx[i].last_key));
+ }
-+ free(idx);
++ reftable_free(idx);
+ }
+
+ writer_clear_index(w);
@@ -6118,7 +6524,8 @@
+ }
+
+ {
-+ struct block_stats *bstats = writer_block_stats(w, typ);
++ struct reftable_block_stats *bstats =
++ writer_reftable_block_stats(w, typ);
+ bstats->index_blocks =
+ w->stats.idx_stats.blocks - before_blocks;
+ bstats->index_offset = index_start;
@@ -6150,7 +6557,7 @@
+}
+
+struct write_record_arg {
-+ struct writer *w;
++ struct reftable_writer *w;
+ int err;
+};
+
@@ -6199,11 +6606,11 @@
+ struct obj_index_tree_node *entry = (struct obj_index_tree_node *)key;
+
+ FREE_AND_NULL(entry->offsets);
-+ free(slice_yield(&entry->hash));
-+ free(entry);
++ reftable_free(slice_yield(&entry->hash));
++ reftable_free(entry);
+}
+
-+static int writer_dump_object_index(struct writer *w)
++static int writer_dump_object_index(struct reftable_writer *w)
+{
+ struct write_record_arg closure = { .w = w };
+ struct common_prefix_arg common = { 0 };
@@ -6224,7 +6631,7 @@
+ return writer_finish_section(w);
+}
+
-+int writer_finish_public_section(struct writer *w)
++int writer_finish_public_section(struct reftable_writer *w)
+{
+ byte typ = 0;
+ int err = 0;
@@ -6256,9 +6663,9 @@
+ return 0;
+}
+
-+int writer_close(struct writer *w)
++int reftable_writer_close(struct reftable_writer *w)
+{
-+ byte footer[68];
++ byte footer[72];
+ byte *p = footer;
+
+ int err = writer_finish_public_section(w);
@@ -6266,8 +6673,7 @@
+ goto exit;
+ }
+
-+ writer_write_header(w, footer);
-+ p += 24;
++ p += writer_write_header(w, footer);
+ put_be64(p, w->stats.ref_stats.index_offset);
+ p += 8;
+ put_be64(p, (w->stats.obj_stats.offset) << 5 | w->stats.object_id_len);
@@ -6284,7 +6690,7 @@
+ p += 4;
+ w->pending_padding = 0;
+
-+ err = padded_write(w, footer, sizeof(footer), 0);
++ err = padded_write(w, footer, footer_size(writer_version(w)), 0);
+ if (err < 0) {
+ goto exit;
+ }
@@ -6298,15 +6704,15 @@
+ /* free up memory. */
+ block_writer_clear(&w->block_writer_data);
+ writer_clear_index(w);
-+ free(slice_yield(&w->last_key));
++ reftable_free(slice_yield(&w->last_key));
+ return err;
+}
+
-+void writer_clear_index(struct writer *w)
++void writer_clear_index(struct reftable_writer *w)
+{
+ int i = 0;
+ for (i = 0; i < w->index_len; i++) {
-+ free(slice_yield(&w->index[i].last_key));
++ reftable_free(slice_yield(&w->index[i].last_key));
+ }
+
+ FREE_AND_NULL(w->index);
@@ -6316,10 +6722,11 @@
+
+const int debug = 0;
+
-+static int writer_flush_nonempty_block(struct writer *w)
++static int writer_flush_nonempty_block(struct reftable_writer *w)
+{
+ byte typ = block_writer_type(w->block_writer);
-+ struct block_stats *bstats = writer_block_stats(w, typ);
++ struct reftable_block_stats *bstats =
++ writer_reftable_block_stats(w, typ);
+ uint64_t block_typ_off = (bstats->blocks == 0) ? w->next : 0;
+ int raw_bytes = block_writer_finish(w->block_writer);
+ int padding = 0;
@@ -6358,8 +6765,8 @@
+
+ if (w->index_cap == w->index_len) {
+ w->index_cap = 2 * w->index_cap + 1;
-+ w->index = realloc(w->index,
-+ sizeof(struct index_record) * w->index_cap);
++ w->index = reftable_realloc(
++ w->index, sizeof(struct index_record) * w->index_cap);
+ }
+
+ {
@@ -6377,7 +6784,7 @@
+ return 0;
+}
+
-+int writer_flush_block(struct writer *w)
++int writer_flush_block(struct reftable_writer *w)
+{
+ if (w->block_writer == NULL) {
+ return 0;
@@ -6388,7 +6795,7 @@
+ return writer_flush_nonempty_block(w);
+}
+
-+struct stats *writer_stats(struct writer *w)
++const struct reftable_stats *writer_stats(struct reftable_writer *w)
+{
+ return &w->stats;
+}
@@ -6415,7 +6822,7 @@
+#include "slice.h"
+#include "tree.h"
+
-+struct writer {
++struct reftable_writer {
+ int (*write)(void *, byte *, int);
+ void *write_arg;
+ int pending_padding;
@@ -6423,11 +6830,11 @@
+
+ uint64_t next;
+ uint64_t min_update_index, max_update_index;
-+ struct write_options opts;
++ struct reftable_write_options opts;
+
+ byte *block;
-+ struct block_writer *block_writer;
-+ struct block_writer block_writer_data;
++ struct reftable_block_writer *block_writer;
++ struct reftable_block_writer block_writer_data;
+ struct index_record *index;
+ int index_len;
+ int index_cap;
@@ -6435,12 +6842,12 @@
+ /* tree for use with tsearch */
+ struct tree_node *obj_index_tree;
+
-+ struct stats stats;
++ struct reftable_stats stats;
+};
+
-+int writer_flush_block(struct writer *w);
-+void writer_clear_index(struct writer *w);
-+int writer_finish_public_section(struct writer *w);
++int writer_flush_block(struct reftable_writer *w);
++void writer_clear_index(struct reftable_writer *w);
++int writer_finish_public_section(struct reftable_writer *w);
+
+#endif 6: a622d8066c7 ! 9: b29c4ecc1c4 Reftable support for git-core
@@ -6,6 +6,7 @@
TODO:
+ * "git show-ref" shows "HEAD"
* Resolve spots marked with XXX
* Test strategy?
@@ -104,8 +105,9 @@ $^
}
}
-- init_db(git_dir, real_git_dir, option_template, INIT_DB_QUIET);
+- init_db(git_dir, real_git_dir, option_template, GIT_HASH_UNKNOWN, INIT_DB_QUIET);
+ init_db(git_dir, real_git_dir, option_template,
++ GIT_HASH_UNKNOWN,
+ DEFAULT_REF_STORAGE, /* XXX */
+ INIT_DB_QUIET);
@@ -116,20 +118,28 @@ $^
--- a/builtin/init-db.c
+++ b/builtin/init-db.c
@@
+ return 1;
}
- static int create_default_files(const char *template_path,
-- const char *original_git_dir)
-+ const char *original_git_dir,
-+ const char *ref_storage_format, int flags)
+-void initialize_repository_version(int hash_algo)
++void initialize_repository_version(int hash_algo, const char *ref_storage_format)
{
- struct stat st1;
- struct strbuf buf = STRBUF_INIT;
+ char repo_version_string[10];
+ int repo_version = GIT_REPO_VERSION;
+@@
+ die(_("The hash algorithm %s is not supported in this build."), hash_algos[hash_algo].name);
+ #endif
+
+- if (hash_algo != GIT_HASH_SHA1)
++ if (hash_algo != GIT_HASH_SHA1 || !strcmp(ref_storage_format, "reftable"))
+ repo_version = GIT_REPO_VERSION_READ;
+
+ /* This forces creation of new config file */
@@
is_bare_repository_cfg = init_is_bare_repository;
if (init_shared_repository != -1)
set_shared_repository(init_shared_repository);
-+ the_repository->ref_storage_format = xstrdup(ref_storage_format);
++ the_repository->ref_storage_format = xstrdup(fmt->ref_storage);
/*
* We would have created the above under user's umask -- under
@@ -166,35 +176,28 @@ $^
+ */
}
- /* This forces creation of new config file */
-- xsnprintf(repo_version_string, sizeof(repo_version_string),
-- "%d", GIT_REPO_VERSION);
-+ xsnprintf(repo_version_string, sizeof(repo_version_string), "%d",
-+ !strcmp(ref_storage_format, "reftable") ?
-+ GIT_REPO_VERSION_READ :
-+ GIT_REPO_VERSION);
- git_config_set("core.repositoryformatversion", repo_version_string);
+- initialize_repository_version(fmt->hash_algo);
++ initialize_repository_version(fmt->hash_algo, fmt->ref_storage);
/* Check filemode trustability */
+ path = git_path_buf(&buf, "config");
@@
}
int init_db(const char *git_dir, const char *real_git_dir,
-- const char *template_dir, unsigned int flags)
-+ const char *template_dir, const char *ref_storage_format,
-+ unsigned int flags)
+- const char *template_dir, int hash, unsigned int flags)
++ const char *template_dir, int hash, const char *ref_storage_format,
++ unsigned int flags)
{
int reinit;
int exist_ok = flags & INIT_DB_EXIST_OK;
@@
+ * is an attempt to reinitialize new repository with an old tool.
*/
- check_repository_format();
-
-- reinit = create_default_files(template_dir, original_git_dir);
-+ reinit = create_default_files(template_dir, original_git_dir,
-+ ref_storage_format, flags);
+ check_repository_format(&repo_fmt);
++ repo_fmt.ref_storage = xstrdup(ref_storage_format);
- create_object_directory();
+ validate_hash_algorithm(&repo_fmt, hash);
@@
git_config_set("receive.denyNonFastforwards", "true");
@@ -213,7 +216,9 @@ $^
const char *real_git_dir = NULL;
const char *work_tree;
const char *template_dir = NULL;
- unsigned int flags = 0;
+@@
+ const char *object_format = NULL;
+ int hash_algo = GIT_HASH_UNKNOWN;
const struct option init_db_options[] = {
- OPT_STRING(0, "template", &template_dir, N_("template-directory"),
- N_("directory from which templates will be used")),
@@ -235,7 +240,7 @@ $^
+ N_("the ref storage format to use")),
OPT_STRING(0, "separate-git-dir", &real_git_dir, N_("gitdir"),
N_("separate git dir from working tree")),
- OPT_END()
+ OPT_STRING(0, "object-format", &object_format, N_("hash"),
@@
}
@@ -245,21 +250,21 @@ $^
UNLEAK(work_tree);
flags |= INIT_DB_EXIST_OK;
-- return init_db(git_dir, real_git_dir, template_dir, flags);
-+ return init_db(git_dir, real_git_dir, template_dir, ref_storage_format,
-+ flags);
+- return init_db(git_dir, real_git_dir, template_dir, hash_algo, flags);
++ return init_db(git_dir, real_git_dir, template_dir, hash_algo, ref_storage_format, flags);
}
diff --git a/cache.h b/cache.h
--- a/cache.h
+++ b/cache.h
@@
- #define INIT_DB_EXIST_OK 0x0002
int init_db(const char *git_dir, const char *real_git_dir,
-- const char *template_dir, unsigned int flags);
-+ const char *template_dir, const char *ref_storage_format,
-+ unsigned int flags);
+ const char *template_dir, int hash_algo,
++ const char *ref_storage_format,
+ unsigned int flags);
+-void initialize_repository_version(int hash_algo);
++void initialize_repository_version(int hash_algo, const char *ref_storage_format);
void sanitize_stdfds(void);
int daemonize(void);
@@ -350,7 +355,6 @@ $^
struct string_list_item;
struct worktree;
-+/* XXX where should this be? */
+#define DEFAULT_REF_STORAGE "files"
+
/*
@@ -386,26 +390,25 @@ $^
+
+extern struct ref_storage_be refs_be_reftable;
+
-+struct reftable_ref_store {
++struct git_reftable_ref_store {
+ struct ref_store base;
+ unsigned int store_flags;
+
+ int err;
+ char *reftable_dir;
-+ char *table_list_file;
-+ struct stack *stack;
++ struct reftable_stack *stack;
+};
+
-+static void clear_log_record(struct log_record *log)
++static void clear_reftable_log_record(struct reftable_log_record *log)
+{
+ log->old_hash = NULL;
+ log->new_hash = NULL;
+ log->message = NULL;
+ log->ref_name = NULL;
-+ log_record_clear(log);
++ reftable_log_record_clear(log);
+}
+
-+static void fill_log_record(struct log_record *log)
++static void fill_reftable_log_record(struct reftable_log_record *log)
+{
+ const char *info = git_committer_info(0);
+ struct ident_split split = {};
@@ -413,7 +416,7 @@ $^
+ int sign = 1;
+ assert(0 == result);
+
-+ log_record_clear(log);
++ reftable_log_record_clear(log);
+ log->name =
+ xstrndup(split.name_begin, split.name_end - split.name_begin);
+ log->email =
@@ -431,12 +434,12 @@ $^
+ log->tz_offset = sign * atoi(split.tz_begin);
+}
+
-+static struct ref_store *reftable_ref_store_create(const char *path,
++static struct ref_store *git_reftable_ref_store_create(const char *path,
+ unsigned int store_flags)
+{
-+ struct reftable_ref_store *refs = xcalloc(1, sizeof(*refs));
++ struct git_reftable_ref_store *refs = xcalloc(1, sizeof(*refs));
+ struct ref_store *ref_store = (struct ref_store *)refs;
-+ struct write_options cfg = {
++ struct reftable_write_options cfg = {
+ .block_size = 4096,
+ .hash_id = the_hash_algo->format_id,
+ };
@@ -449,10 +452,6 @@ $^
+ refs->reftable_dir = xstrdup(sb.buf);
+ strbuf_reset(&sb);
+
-+ strbuf_addf(&sb, "%s/reftable/tables.list", path);
-+ refs->table_list_file = xstrdup(sb.buf);
-+ strbuf_reset(&sb);
-+
+ strbuf_addf(&sb, "%s/refs", path);
+ safe_create_dir(sb.buf, 1);
+ strbuf_reset(&sb);
@@ -464,31 +463,23 @@ $^
+ strbuf_addf(&sb, "%s/refs/heads", path);
+ write_file(sb.buf, "this repository uses the reftable format");
+
-+ refs->err = new_stack(&refs->stack, refs->reftable_dir,
-+ refs->table_list_file, cfg);
++ refs->err = reftable_new_stack(&refs->stack, refs->reftable_dir, cfg);
+ strbuf_release(&sb);
+ return ref_store;
+}
+
+static int reftable_init_db(struct ref_store *ref_store, struct strbuf *err)
+{
-+ struct reftable_ref_store *refs =
-+ (struct reftable_ref_store *)ref_store;
-+ FILE *file;
++ struct git_reftable_ref_store *refs =
++ (struct git_reftable_ref_store *)ref_store;
+ safe_create_dir(refs->reftable_dir, 1);
-+
-+ file = fopen(refs->table_list_file, "a");
-+ if (file == NULL) {
-+ return -1;
-+ }
-+ fclose(file);
+ return 0;
+}
+
-+struct reftable_iterator {
++struct git_reftable_iterator {
+ struct ref_iterator base;
-+ struct iterator iter;
-+ struct ref_record ref;
++ struct reftable_iterator iter;
++ struct reftable_ref_record ref;
+ struct object_id oid;
+ struct ref_store *ref_store;
+ unsigned int flags;
@@ -498,9 +489,9 @@ $^
+
+static int reftable_ref_iterator_advance(struct ref_iterator *ref_iterator)
+{
-+ struct reftable_iterator *ri = (struct reftable_iterator *)ref_iterator;
++ struct git_reftable_iterator *ri = (struct git_reftable_iterator *)ref_iterator;
+ while (ri->err == 0) {
-+ ri->err = iterator_next_ref(ri->iter, &ri->ref);
++ ri->err = reftable_iterator_next_ref(ri->iter, &ri->ref);
+ if (ri->err) {
+ break;
+ }
@@ -554,7 +545,7 @@ $^
+static int reftable_ref_iterator_peel(struct ref_iterator *ref_iterator,
+ struct object_id *peeled)
+{
-+ struct reftable_iterator *ri = (struct reftable_iterator *)ref_iterator;
++ struct git_reftable_iterator *ri = (struct git_reftable_iterator *)ref_iterator;
+ if (ri->ref.target_value != NULL) {
+ hashcpy(peeled->hash, ri->ref.target_value);
+ return 0;
@@ -565,9 +556,9 @@ $^
+
+static int reftable_ref_iterator_abort(struct ref_iterator *ref_iterator)
+{
-+ struct reftable_iterator *ri = (struct reftable_iterator *)ref_iterator;
-+ ref_record_clear(&ri->ref);
-+ iterator_destroy(&ri->iter);
++ struct git_reftable_iterator *ri = (struct git_reftable_iterator *)ref_iterator;
++ reftable_ref_record_clear(&ri->ref);
++ reftable_iterator_destroy(&ri->iter);
+ return 0;
+}
+
@@ -580,16 +571,16 @@ $^
+reftable_ref_iterator_begin(struct ref_store *ref_store, const char *prefix,
+ unsigned int flags)
+{
-+ struct reftable_ref_store *refs =
-+ (struct reftable_ref_store *)ref_store;
-+ struct reftable_iterator *ri = xcalloc(1, sizeof(*ri));
-+ struct merged_table *mt = NULL;
++ struct git_reftable_ref_store *refs =
++ (struct git_reftable_ref_store *)ref_store;
++ struct git_reftable_iterator *ri = xcalloc(1, sizeof(*ri));
++ struct reftable_merged_table *mt = NULL;
+
+ if (refs->err < 0) {
+ ri->err = refs->err;
+ } else {
-+ mt = stack_merged_table(refs->stack);
-+ ri->err = merged_table_seek_ref(mt, &ri->iter, prefix);
++ mt = reftable_stack_merged_table(refs->stack);
++ ri->err = reftable_merged_table_seek_ref(mt, &ri->iter, prefix);
+ }
+
+ base_ref_iterator_init(&ri->base, &reftable_ref_iterator_vtable, 1);
@@ -610,8 +601,8 @@ $^
+ struct ref_transaction *transaction,
+ struct strbuf *err)
+{
-+ struct reftable_ref_store *refs =
-+ (struct reftable_ref_store *)ref_store;
++ struct git_reftable_ref_store *refs =
++ (struct git_reftable_ref_store *)ref_store;
+ (void)refs;
+ return 0;
+}
@@ -640,19 +631,19 @@ $^
+ ((struct ref_update *)b)->refname);
+}
+
-+static int write_transaction_table(struct writer *writer, void *arg)
++static int write_transaction_table(struct reftable_writer *writer, void *arg)
+{
+ struct ref_transaction *transaction = (struct ref_transaction *)arg;
-+ struct reftable_ref_store *refs =
-+ (struct reftable_ref_store *)transaction->ref_store;
-+ uint64_t ts = stack_next_update_index(refs->stack);
++ struct git_reftable_ref_store *refs =
++ (struct git_reftable_ref_store *)transaction->ref_store;
++ uint64_t ts = reftable_stack_next_update_index(refs->stack);
+ int err = 0;
-+ struct log_record *logs = calloc(transaction->nr, sizeof(*logs));
++ struct reftable_log_record *logs = calloc(transaction->nr, sizeof(*logs));
+ struct ref_update **sorted =
+ malloc(transaction->nr * sizeof(struct ref_update *));
+ COPY_ARRAY(sorted, transaction->updates, transaction->nr);
+ QSORT(sorted, transaction->nr, ref_update_cmp);
-+ writer_set_limits(writer, ts, ts);
++ reftable_writer_set_limits(writer, ts, ts);
+
+ for (int i = 0; i < transaction->nr; i++) {
+ struct ref_update *u = sorted[i];
@@ -667,8 +658,8 @@ $^
+
+ for (int i = 0; i < transaction->nr; i++) {
+ struct ref_update *u = sorted[i];
-+ struct log_record *log = &logs[i];
-+ fill_log_record(log);
++ struct reftable_log_record *log = &logs[i];
++ fill_reftable_log_record(log);
+ log->ref_name = (char *)u->refname;
+ log->old_hash = u->old_oid.hash;
+ log->new_hash = u->new_oid.hash;
@@ -683,13 +674,13 @@ $^
+ const char *resolved = refs_resolve_ref_unsafe(
+ transaction->ref_store, u->refname, 0, &out_oid,
+ &out_flags);
-+ struct ref_record ref = {};
++ struct reftable_ref_record ref = {};
+ ref.ref_name =
+ (char *)(resolved ? resolved : u->refname);
+ log->ref_name = ref.ref_name;
+ ref.value = u->new_oid.hash;
+ ref.update_index = ts;
-+ err = writer_add_ref(writer, &ref);
++ err = reftable_writer_add_ref(writer, &ref);
+ if (err < 0) {
+ goto exit;
+ }
@@ -697,8 +688,8 @@ $^
+ }
+
+ for (int i = 0; i < transaction->nr; i++) {
-+ err = writer_add_log(writer, &logs[i]);
-+ clear_log_record(&logs[i]);
++ err = reftable_writer_add_log(writer, &logs[i]);
++ clear_reftable_log_record(&logs[i]);
+ if (err < 0) {
+ goto exit;
+ }
@@ -714,17 +705,17 @@ $^
+ struct ref_transaction *transaction,
+ struct strbuf *errmsg)
+{
-+ struct reftable_ref_store *refs =
-+ (struct reftable_ref_store *)ref_store;
++ struct git_reftable_ref_store *refs =
++ (struct git_reftable_ref_store *)ref_store;
+ int err = 0;
+ if (refs->err < 0) {
+ return refs->err;
+ }
+
-+ err = stack_add(refs->stack, &write_transaction_table, transaction);
++ err = reftable_stack_add(refs->stack, &write_transaction_table, transaction);
+ if (err < 0) {
+ strbuf_addf(errmsg, "reftable: transaction failure %s",
-+ error_str(err));
++ reftable_error_str(err));
+ return -1;
+ }
+
@@ -739,49 +730,49 @@ $^
+}
+
+struct write_delete_refs_arg {
-+ struct stack *stack;
++ struct reftable_stack *stack;
+ struct string_list *refnames;
+ const char *logmsg;
+ unsigned int flags;
+};
+
-+static int write_delete_refs_table(struct writer *writer, void *argv)
++static int write_delete_refs_table(struct reftable_writer *writer, void *argv)
+{
+ struct write_delete_refs_arg *arg =
+ (struct write_delete_refs_arg *)argv;
-+ uint64_t ts = stack_next_update_index(arg->stack);
++ uint64_t ts = reftable_stack_next_update_index(arg->stack);
+ int err = 0;
+
-+ writer_set_limits(writer, ts, ts);
++ reftable_writer_set_limits(writer, ts, ts);
+ for (int i = 0; i < arg->refnames->nr; i++) {
-+ struct ref_record ref = {
++ struct reftable_ref_record ref = {
+ .ref_name = (char *)arg->refnames->items[i].string,
+ .update_index = ts,
+ };
-+ err = writer_add_ref(writer, &ref);
++ err = reftable_writer_add_ref(writer, &ref);
+ if (err < 0) {
+ return err;
+ }
+ }
+
+ for (int i = 0; i < arg->refnames->nr; i++) {
-+ struct log_record log = {};
-+ struct ref_record current = {};
-+ fill_log_record(&log);
++ struct reftable_log_record log = {};
++ struct reftable_ref_record current = {};
++ fill_reftable_log_record(&log);
+ log.message = xstrdup(arg->logmsg);
+ log.new_hash = NULL;
+ log.old_hash = NULL;
+ log.update_index = ts;
+ log.ref_name = (char *)arg->refnames->items[i].string;
+
-+ if (stack_read_ref(arg->stack, log.ref_name, ¤t) == 0) {
++ if (reftable_stack_read_ref(arg->stack, log.ref_name, ¤t) == 0) {
+ log.old_hash = current.value;
+ }
-+ err = writer_add_log(writer, &log);
++ err = reftable_writer_add_log(writer, &log);
+ log.old_hash = NULL;
-+ ref_record_clear(¤t);
++ reftable_ref_record_clear(¤t);
+
-+ clear_log_record(&log);
++ clear_reftable_log_record(&log);
+ if (err < 0) {
+ return err;
+ }
@@ -793,8 +784,8 @@ $^
+ struct string_list *refnames,
+ unsigned int flags)
+{
-+ struct reftable_ref_store *refs =
-+ (struct reftable_ref_store *)ref_store;
++ struct git_reftable_ref_store *refs =
++ (struct git_reftable_ref_store *)ref_store;
+ struct write_delete_refs_arg arg = {
+ .stack = refs->stack,
+ .refnames = refnames,
@@ -805,52 +796,52 @@ $^
+ return refs->err;
+ }
+
-+ return stack_add(refs->stack, &write_delete_refs_table, &arg);
++ return reftable_stack_add(refs->stack, &write_delete_refs_table, &arg);
+}
+
+static int reftable_pack_refs(struct ref_store *ref_store, unsigned int flags)
+{
-+ struct reftable_ref_store *refs =
-+ (struct reftable_ref_store *)ref_store;
++ struct git_reftable_ref_store *refs =
++ (struct git_reftable_ref_store *)ref_store;
+ if (refs->err < 0) {
+ return refs->err;
+ }
-+ return stack_compact_all(refs->stack, NULL);
++ return reftable_stack_compact_all(refs->stack, NULL);
+}
+
+struct write_create_symref_arg {
-+ struct reftable_ref_store *refs;
++ struct git_reftable_ref_store *refs;
+ const char *refname;
+ const char *target;
+ const char *logmsg;
+};
+
-+static int write_create_symref_table(struct writer *writer, void *arg)
++static int write_create_symref_table(struct reftable_writer *writer, void *arg)
+{
+ struct write_create_symref_arg *create =
+ (struct write_create_symref_arg *)arg;
-+ uint64_t ts = stack_next_update_index(create->refs->stack);
++ uint64_t ts = reftable_stack_next_update_index(create->refs->stack);
+ int err = 0;
+
-+ struct ref_record ref = {
++ struct reftable_ref_record ref = {
+ .ref_name = (char *)create->refname,
+ .target = (char *)create->target,
+ .update_index = ts,
+ };
-+ writer_set_limits(writer, ts, ts);
-+ err = writer_add_ref(writer, &ref);
++ reftable_writer_set_limits(writer, ts, ts);
++ err = reftable_writer_add_ref(writer, &ref);
+ if (err < 0) {
+ return err;
+ }
+
+ {
-+ struct log_record log = {};
++ struct reftable_log_record log = {};
+ struct object_id new_oid = {};
+ struct object_id old_oid = {};
-+ struct ref_record current = {};
-+ stack_read_ref(create->refs->stack, create->refname, ¤t);
++ struct reftable_ref_record current = {};
++ reftable_stack_read_ref(create->refs->stack, create->refname, ¤t);
+
-+ fill_log_record(&log);
++ fill_reftable_log_record(&log);
+ log.ref_name = current.ref_name;
+ if (refs_resolve_ref_unsafe(
+ (struct ref_store *)create->refs, create->refname,
@@ -865,12 +856,12 @@ $^
+ }
+
+ if (log.old_hash != NULL || log.new_hash != NULL) {
-+ writer_add_log(writer, &log);
++ reftable_writer_add_log(writer, &log);
+ }
+ log.ref_name = NULL;
+ log.old_hash = NULL;
+ log.new_hash = NULL;
-+ clear_log_record(&log);
++ clear_reftable_log_record(&log);
+ }
+ return 0;
+}
@@ -879,8 +870,8 @@ $^
+ const char *refname, const char *target,
+ const char *logmsg)
+{
-+ struct reftable_ref_store *refs =
-+ (struct reftable_ref_store *)ref_store;
++ struct git_reftable_ref_store *refs =
++ (struct git_reftable_ref_store *)ref_store;
+ struct write_create_symref_arg arg = { .refs = refs,
+ .refname = refname,
+ .target = target,
@@ -888,55 +879,55 @@ $^
+ if (refs->err < 0) {
+ return refs->err;
+ }
-+ return stack_add(refs->stack, &write_create_symref_table, &arg);
++ return reftable_stack_add(refs->stack, &write_create_symref_table, &arg);
+}
+
+struct write_rename_arg {
-+ struct stack *stack;
++ struct reftable_stack *stack;
+ const char *oldname;
+ const char *newname;
+ const char *logmsg;
+};
+
-+static int write_rename_table(struct writer *writer, void *argv)
++static int write_rename_table(struct reftable_writer *writer, void *argv)
+{
+ struct write_rename_arg *arg = (struct write_rename_arg *)argv;
-+ uint64_t ts = stack_next_update_index(arg->stack);
-+ struct ref_record ref = {};
-+ int err = stack_read_ref(arg->stack, arg->oldname, &ref);
++ uint64_t ts = reftable_stack_next_update_index(arg->stack);
++ struct reftable_ref_record ref = {};
++ int err = reftable_stack_read_ref(arg->stack, arg->oldname, &ref);
+
+ if (err) {
+ goto exit;
+ }
+
+ /* XXX do ref renames overwrite the target? */
-+ if (stack_read_ref(arg->stack, arg->newname, &ref) == 0) {
++ if (reftable_stack_read_ref(arg->stack, arg->newname, &ref) == 0) {
+ goto exit;
+ }
+
+ free(ref.ref_name);
+ ref.ref_name = strdup(arg->newname);
-+ writer_set_limits(writer, ts, ts);
++ reftable_writer_set_limits(writer, ts, ts);
+ ref.update_index = ts;
+
+ {
-+ struct ref_record todo[2] = {};
++ struct reftable_ref_record todo[2] = {};394 lines not shown · raw message
1 line not shown · raw message
Part of this message is not shown here. View the raw message.
PATCH v8 1/9 refs.h: clarify reflog iteration order
From: Han-Wen Nienhuys <[email protected]>
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- refs.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
refs.h +4 -1
@@ -444,18 +444,21 @@ int delete_refs(const char *msg, struct string_list *refnames,int refs_delete_reflog (struct ref_store *refs,const char *refname);int delete_reflog (const char *refname); -/* iterate over reflog entries */ +/* Iterate over reflog entries. */ typedef int each_reflog_ent_fn (struct object_id *old_oid,struct object_id *new_oid,const char *committer,timestamp_t timestamp,int tz,const char *msg,void *cb_data); +/* Iterate in over reflog entries, oldest entry first. */ int refs_for_each_reflog_ent (struct ref_store *refs,const char *refname,each_reflog_ent_fn fn,void *cb_data);int refs_for_each_reflog_ent_reverse (struct ref_store *refs,const char *refname,each_reflog_ent_fn fn,void *cb_data); + +/* Call a function for each reflog entry, oldest entry first. */ int for_each_reflog_ent (const char *refname,each_reflog_ent_fn fn,void *cb_data);int for_each_reflog_ent_reverse (const char *refname,each_reflog_ent_fn fn,void *cb_data);
Signature · 1 line
gitgitgadget
PATCH v8 2/9 create .git/refs in files-backend.c
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This prepares for supporting the reftable format, which will want create its own file system layout in .git
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- builtin/init-db.c | 2 -- refs/files-backend.c | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-)
builtin/init-db.c +0 -2
@@ -251,8 +251,6 @@ static int create_default_files(const char *template_path,* We need to create a "refs" dir in any case so that older * versions of git can tell that this is a repository. */ -safe_create_dir (git_path ("refs" ),1 ); -adjust_shared_perm (git_path ("refs" ));if (refs_init_db (&err))die ("failed to set up refs db: %s" , err.buf);
refs/files-backend.c +6 -0
@@ -3157,9 +3157,15 @@ static int files_init_db(struct ref_store *ref_store, struct strbuf *err)files_downcast (ref_store,REF_STORE_WRITE ,"init_db" );struct strbuf sb =STRBUF_INIT ; +files_ref_path (refs, &sb,"refs" ); +safe_create_dir (sb.buf,1 ); +/* adjust permissions even if directory already exists. */ +adjust_shared_perm (sb.buf); +/* * Create .git/refs/{heads,tags} */ +strbuf_reset (&sb);files_ref_path (refs, &sb,"refs/heads" );safe_create_dir (sb.buf,1 );
Signature · 1 line
gitgitgadget
PATCH v8 3/9 refs: document how ref_iterator_advance_fn should handle symrefs
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- refs/refs-internal.h | 5 +++++ 1 file changed, 5 insertions(+)
refs/refs-internal.h +5 -0
@@ -438,6 +438,11 @@ void base_ref_iterator_free(struct ref_iterator *iter);/* Virtual function declarations for ref_iterators: */ +/* +* backend-specific implementation of ref_iterator_advance. +* For symrefs, the function should set REF_ISSYMREF, and it should also +* dereference the symref to provide the OID referent. +*/ typedef int ref_iterator_advance_fn (struct ref_iterator *ref_iterator); typedef int ref_iterator_peel_fn(struct ref_iterator *ref_iterator,
Signature · 1 line
gitgitgadget
PATCH v8 4/9 Add .gitattributes for the reftable/ directory
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- reftable/.gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 reftable/.gitattributes
reftable/.gitattributes new file +1 -0
@@ -0,0 +1 @@ +/zlib-compat.c whitespace=-indent-with-non-tab,-trailing-space
Signature · 1 line
gitgitgadget
PATCH v8 5/9 reftable: file format documentation
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Jonathan Nieder <[email protected]>
Shawn Pearce explains:
Some repositories contain a lot of references (e.g. android at 866k, rails at 31k). The reftable format provides:
- Near constant time lookup for any single reference, even when the repository is cold and not in process or kernel cache.
- Near constant time verification a SHA-1 is referred to by at least one reference (for allow-tip-sha1-in-want).
- Efficient lookup of an entire namespace, such as refs/tags/.
- Support atomic push O(size_of_update) operations.
- Combine reflog storage with ref storage.
This file format spec was originally written in July, 2017 by Shawn Pearce. Some refinements since then were made by Shawn and by Han-Wen Nienhuys based on experiences implementing and experimenting with the format. (All of this was in the context of our work at Google and Google is happy to contribute the result to the Git project.)
Imported from JGit[1]'s current version (c217d33ff, "Documentation/technical/reftable: improve repo layout", 2020-02-04) of Documentation/technical/reftable.md and converted to asciidoc by running
pandoc -t asciidoc -f markdown reftable.md >reftable.txt
using pandoc 2.2.1. The result required the following additional minor changes:
- removed the [TOC] directive to add a table of contents, since asciidoc does not support it
- replaced git-scm.com/docs links with linkgit: directives that link to other pages within Git's documentation
[1] https://eclipse.googlesource.com/jgit/jgit
Signed-off-by Jonathan Nieder <[email protected]>
--- Documentation/Makefile | 1 + Documentation/technical/reftable.txt | 1067 ++++++++++++++++++++++++++ 2 files changed, 1068 insertions(+) create mode 100644 Documentation/technical/reftable.txt
Documentation/Makefile +1 -0
@@ -92,6 +92,7 @@ TECH_DOCS += technical/protocol-capabilities TECH_DOCS += technical/protocol-common TECH_DOCS += technical/protocol-v2 TECH_DOCS += technical/racy-git +TECH_DOCS += technical/reftable TECH_DOCS += technical/send-pack-pipeline TECH_DOCS += technical/shallow TECH_DOCS += technical/signature-format
Documentation/technical/reftable.txt new file +1067 -0
1,067 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v8 6/9 reftable: define version 2 of the spec to accomodate SHA256
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- Documentation/technical/reftable.txt | 50 ++++++++++++++++------------ 1 file changed, 28 insertions(+), 22 deletions(-)
Documentation/technical/reftable.txt +28 -22
@@ -193,8 +193,8 @@ and non-aligned files. Very small files (e.g. 1 only ref block) may omit `padding` and the ref index to reduce total file size. -Header -^^^^^^ +Header (version 1) +^^^^^^^^^^^^^^^^^^ A 24-byte header appears at the beginning of the file:
@@ -215,6 +215,24 @@ used in a stack for link:#Update-transactions[transactions], these fields can order the files such that the prior file’s `max_update_index + 1` is the next file’s `min_update_index`. +Header (version 2) +^^^^^^^^^^^^^^^^^^ + +A 28-byte header appears at the beginning of the file: + +.... +'REFT' +uint8( version_number = 1 ) +uint24( block_size ) +uint64( min_update_index ) +uint64( max_update_index ) +uint32( hash_id ) +.... + +The header is identical to `version_number=1`, with the 4-byte hash ID +("sha1" for SHA1 and "s256" for SHA-256) append to the header. + + First ref block ^^^^^^^^^^^^^^^
@@ -671,14 +689,8 @@ Footer After the last block of the file, a file footer is written. It begins like the file header, but is extended with additional data. -A 68-byte footer appears at the end: - .... - 'REFT' - uint8( version_number = 1 ) - uint24( block_size ) - uint64( min_update_index ) - uint64( max_update_index ) + HEADER uint64( ref_index_position ) uint64( (obj_position << 5) | obj_id_len )
@@ -701,12 +713,16 @@ obj blocks. * `obj_index_position`: byte position for the start of the obj index. * `log_index_position`: byte position for the start of the log index. +The size of the footer is 68 bytes for version 1, and 72 bytes for +version 2. + Reading the footer ++++++++++++++++++ -Readers must seek to `file_length - 68` to access the footer. A trusted -external source (such as `stat(2)`) is necessary to obtain -`file_length`. When reading the footer, readers must verify: +Readers must first read the file start to determine the version +number. Then they seek to `file_length - FOOTER_LENGTH` to access the +footer. A trusted external source (such as `stat(2)`) is necessary to +obtain `file_length`. When reading the footer, readers must verify: * 4-byte magic is correct * 1-byte version number is recognized
@@ -1055,13 +1071,3 @@ impossible. A common format that can be supported by all major Git implementations (git-core, JGit, libgit2) is strongly preferred. - -Future -~~~~~~ - -Longer hashes -^^^^^^^^^^^^^ - -Version will bump (e.g. 2) to indicate `value` uses a different object -id length other than 20. The length could be stored in an expanded file -header, or hardcoded as part of the version.
Signature · 1 line
gitgitgadget
PATCH v8 7/9 reftable: clarify how empty tables should be written
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The format allows for some ambiguity, as a lone footer also starts with a valid file header. However, the current JGit code will barf on this. This commit codifies this behavior into the standard.
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- Documentation/technical/reftable.txt | 7 +++++++ 1 file changed, 7 insertions(+)
Documentation/technical/reftable.txt +7 -0
@@ -731,6 +731,13 @@ version) Once verified, the other fields of the footer can be accessed. +Empty tables +++++++++++++ + +A reftable may be empty. In this case, the file starts with a header +and is immediately followed by a footer. + + Varint encoding ^^^^^^^^^^^^^^^
Signature · 1 line
gitgitgadget
PATCH v8 8/9 Add reftable library
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Reftable is a format for storing the ref database. Its rationale and specification is in the preceding commit.
This imports the upstream library as one big commit. For understanding the code, it is suggested to read the code in the following order:
* The specification under Documentation/technical/reftable.txt
* reftable.h - the public API
* block.{c,h} - reading and writing blocks.
* writer.{c,h} - writing a complete reftable file.
* merged.{c,h} and pq.{c,h} - reading a stack of reftables
* stack.{c,h} - writing and compacting stacks of reftable on the filesystem.
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- reftable/LICENSE | 31 ++ reftable/README.md | 11 + reftable/VERSION | 5 + reftable/basics.c | 194 +++++++ reftable/basics.h | 35 ++ reftable/block.c | 433 +++++++++++++++ reftable/block.h | 76 +++ reftable/blocksource.h | 22 + reftable/bytes.c | 0 reftable/config.h | 1 + reftable/constants.h | 21 + reftable/dump.c | 97 ++++ reftable/file.c | 98 ++++ reftable/iter.c | 234 +++++++++ reftable/iter.h | 56 ++ reftable/merged.c | 299 +++++++++++ reftable/merged.h | 34 ++ reftable/pq.c | 115 ++++ reftable/pq.h | 34 ++ reftable/reader.c | 756 +++++++++++++++++++++++++++ reftable/reader.h | 60 +++ reftable/record.c | 1098 ++++++++++++++++++++++++++++++++++++++ reftable/record.h | 106 ++++ reftable/reftable.h | 520 ++++++++++++++++++ reftable/slice.c | 219 ++++++++ reftable/slice.h | 42 ++ reftable/stack.c | 1132 ++++++++++++++++++++++++++++++++++++++++ reftable/stack.h | 42 ++ reftable/system.h | 53 ++ reftable/tree.c | 67 +++ reftable/tree.h | 24 + reftable/update.sh | 14 + reftable/writer.c | 653 +++++++++++++++++++++++ reftable/writer.h | 45 ++ reftable/zlib-compat.c | 92 ++++ 35 files changed, 6719 insertions(+) create mode 100644 reftable/LICENSE create mode 100644 reftable/README.md create mode 100644 reftable/VERSION create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/blocksource.h create mode 100644 reftable/bytes.c create mode 100644 reftable/config.h create mode 100644 reftable/constants.h create mode 100644 reftable/dump.c create mode 100644 reftable/file.c create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/reftable.h create mode 100644 reftable/slice.c create mode 100644 reftable/slice.h create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/system.h create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100644 reftable/update.sh create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h create mode 100644 reftable/zlib-compat.c
reftable/LICENSE new file +31 -0
@@ -0,0 +1,31 @@ +BSD License + +Copyright (c) 2020, Google LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of Google LLC nor the names of its contributors may +be used to endorse or promote products derived from this software +without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
reftable/README.md new file +11 -0
@@ -0,0 +1,11 @@ + +The source code in this directory comes from https://github.com/google/reftable. + +The VERSION file keeps track of the current version of the reftable library. + +To update the library, do: + + sh reftable/update.sh + +Bugfixes should be accompanied by a test and applied to upstream project at +https://github.com/google/reftable.
reftable/VERSION new file +5 -0
@@ -0,0 +1,5 @@ +commit 920f7413f7a25f308d9a203f584873f6753192ec +Author: Han-Wen Nienhuys <[email protected]> +Date: Wed Apr 1 12:41:41 2020 +0200 + + C: remove stray debug printfs
reftable/basics.c new file +194 -0
194 lines not on this page · raw message
reftable/basics.h new file +35 -0
@@ -0,0 +1,35 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef BASICS_H +#define BASICS_H + +#include "system.h" + +#include "reftable.h" + +#define true 1 +#define false 0 + +void put_be24 (byte *out,uint32_t i); +uint32_t get_be24 (byte *in); +void put_be16 (uint8_t *out,uint16_t i); + +int binsearch (int sz,int (*f)(int k,void *args),void *args); + +void free_names (char **a); +void parse_names (char *buf,int size,char ***namesp); +int names_equal (char **a,char **b); +int names_length (char **names); + +void *reftable_malloc (size_t sz); +void *reftable_realloc (void *p,size_t sz); +void reftable_free (void *p); +void *reftable_calloc (size_t sz); + +#endif
reftable/block.c new file +433 -0
433 lines not on this page · raw message
reftable/block.h new file +76 -0
76 lines not on this page · raw message
reftable/blocksource.h new file +22 -0
@@ -0,0 +1,22 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef BLOCKSOURCE_H +#define BLOCKSOURCE_H + +#include "reftable.h" + +uint64_t block_source_size (struct reftable_block_source source); +int block_source_read_block (struct reftable_block_source source, +struct reftable_block *dest,uint64_t off, +uint32_t size); +void block_source_return_block (struct reftable_block_source source, +struct reftable_block *ret); +void block_source_close (struct reftable_block_source source); + +#endif
reftable/bytes.c new file +0 -0
reftable/config.h new file +1 -0
@@ -0,0 +1 @@ +/* empty */
reftable/constants.h new file +21 -0
@@ -0,0 +1,21 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef CONSTANTS_H +#define CONSTANTS_H + +#define BLOCK_TYPE_LOG 'g' +#define BLOCK_TYPE_INDEX 'i' +#define BLOCK_TYPE_REF 'r' +#define BLOCK_TYPE_OBJ 'o' +#define BLOCK_TYPE_ANY 0 + +#define MAX_RESTARTS ((1 << 16) - 1) +#define DEFAULT_BLOCK_SIZE 4096 + +#endif
reftable/dump.c new file +97 -0
97 lines not on this page · raw message
reftable/file.c new file +98 -0
98 lines not on this page · raw message
reftable/iter.c new file +234 -0
234 lines not on this page · raw message
reftable/iter.h new file +56 -0
56 lines not on this page · raw message
reftable/merged.c new file +299 -0
299 lines not on this page · raw message
reftable/merged.h new file +34 -0
@@ -0,0 +1,34 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef MERGED_H +#define MERGED_H + +#include "pq.h" +#include "reftable.h" + +struct reftable_merged_table { +struct reftable_reader **stack; +int stack_len; +uint32_t hash_id; + +uint64_t min; +uint64_t max; +}; + +struct merged_iter { +struct reftable_iterator *stack; +uint32_t hash_id; +int stack_len; +byte typ; +struct merged_iter_pqueue pq; +}; + +void merged_table_clear (struct reftable_merged_table *mt); + +#endif
reftable/pq.c new file +115 -0
115 lines not on this page · raw message
reftable/pq.h new file +34 -0
@@ -0,0 +1,34 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef PQ_H +#define PQ_H + +#include "record.h" + +struct pq_entry { +int index; +struct record rec; +}; + +int pq_less (struct pq_entry a,struct pq_entry b); + +struct merged_iter_pqueue { +struct pq_entry *heap; +int len; +int cap; +}; + +struct pq_entry merged_iter_pqueue_top (struct merged_iter_pqueue pq); +bool merged_iter_pqueue_is_empty (struct merged_iter_pqueue pq); +void merged_iter_pqueue_check (struct merged_iter_pqueue pq); +struct pq_entry merged_iter_pqueue_remove (struct merged_iter_pqueue *pq); +void merged_iter_pqueue_add (struct merged_iter_pqueue *pq,struct pq_entry e); +void merged_iter_pqueue_clear (struct merged_iter_pqueue *pq); + +#endif
reftable/reader.c new file +756 -0
756 lines not on this page · raw message
reftable/reader.h new file +60 -0
60 lines not on this page · raw message
reftable/record.c new file +1098 -0
388 lines not on this page · raw message
reftable/record.h new file +106 -0
106 lines not on this page · raw message
reftable/reftable.h new file +520 -0
520 lines not on this page · raw message
reftable/slice.c new file +219 -0
219 lines not on this page · raw message
reftable/slice.h new file +42 -0
42 lines not on this page · raw message
reftable/stack.c new file +1132 -0
1,132 lines not on this page · raw message
reftable/stack.h new file +42 -0
42 lines not on this page · raw message
reftable/system.h new file +53 -0
53 lines not on this page · raw message
reftable/tree.c new file +67 -0
67 lines not on this page · raw message
reftable/writer.c new file +653 -0
653 lines not on this page · raw message
reftable/writer.h new file +45 -0
45 lines not on this page · raw message
reftable/zlib-compat.c new file +92 -0
92 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
gitgitgadget
Part of the diff is not shown here. View the raw message.
PATCH v8 9/9 Reftable support for git-core
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
For background, see the previous commit introducing the library.
TODO:
* "git show-ref" shows "HEAD"
* Resolve spots marked with XXX
* Test strategy?
Example use: see t/t0031-reftable.sh
Signed-off-by Han-Wen Nienhuys <[email protected]>
Co-authored-by Jeff King <[email protected]>
--- .../technical/repository-version.txt | 7 + Makefile | 24 +- builtin/clone.c | 5 +- builtin/init-db.c | 49 +- cache.h | 4 +- refs.c | 20 +- refs.h | 2 + refs/refs-internal.h | 1 + refs/reftable-backend.c | 1002 +++++++++++++++++ repository.c | 2 + repository.h | 3 + setup.c | 12 +- t/t0031-reftable.sh | 35 + 13 files changed, 1138 insertions(+), 28 deletions(-) create mode 100644 refs/reftable-backend.c create mode 100755 t/t0031-reftable.sh
Documentation/technical/repository-version.txt +7 -0
@@ -100,3 +100,10 @@ If set, by default "git config" reads from both "config" and multiple working directory mode, "config" file is shared while "config.worktree" is per-working directory (i.e., it's in GIT_COMMON_DIR/worktrees/<id>/config.worktree) + +==== `refStorage` + +Specifies the file format for the ref database. Values are `files` +(for the traditional packed + loose ref format) and `reftable` for the +binary reftable format. See https://github.com/google/reftable for +more information.
Makefile +23 -1
@@ -814,6 +814,7 @@ TEST_SHELL_PATH = $(SHELL_PATH) LIB_FILE = libgit.a XDIFF_LIB = xdiff/lib.a VCSSVN_LIB = vcs-svn/lib.a +REFTABLE_LIB = reftable/libreftable.a GENERATED_H += command-list.h
@@ -960,6 +961,7 @@ LIB_OBJS += rebase-interactive.o LIB_OBJS += reflog-walk.o LIB_OBJS += refs.o LIB_OBJS += refs/files-backend.o +LIB_OBJS += refs/reftable-backend.o LIB_OBJS += refs/iterator.o LIB_OBJS += refs/packed-backend.o LIB_OBJS += refs/ref-cache.o
@@ -1164,7 +1166,7 @@ THIRD_PARTY_SOURCES += compat/regex/% THIRD_PARTY_SOURCES += sha1collisiondetection/% THIRD_PARTY_SOURCES += sha1dc/% -GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) +GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) EXTLIBS = GIT_USER_AGENT = git/$(GIT_VERSION)
@@ -2348,11 +2350,28 @@ VCSSVN_OBJS += vcs-svn/fast_export.o VCSSVN_OBJS += vcs-svn/svndiff.o VCSSVN_OBJS += vcs-svn/svndump.o +REFTABLE_OBJS += reftable/basics.o +REFTABLE_OBJS += reftable/block.o +REFTABLE_OBJS += reftable/bytes.o +REFTABLE_OBJS += reftable/file.o +REFTABLE_OBJS += reftable/iter.o +REFTABLE_OBJS += reftable/merged.o +REFTABLE_OBJS += reftable/pq.o +REFTABLE_OBJS += reftable/reader.o +REFTABLE_OBJS += reftable/record.o +REFTABLE_OBJS += reftable/slice.o +REFTABLE_OBJS += reftable/stack.o +REFTABLE_OBJS += reftable/tree.o +REFTABLE_OBJS += reftable/writer.o +REFTABLE_OBJS += reftable/zlib-compat.o + + TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) OBJECTS := $(LIB_OBJS) $(BUILTIN_OBJS) $(PROGRAM_OBJS) $(TEST_OBJS) \ $(XDIFF_OBJS) \ $(VCSSVN_OBJS) \ $(FUZZ_OBJS) \ + $(REFTABLE_OBJS) \ common-main.o \ git.o ifndef NO_CURL
@@ -2489,6 +2508,9 @@ $(XDIFF_LIB): $(XDIFF_OBJS) $(VCSSVN_LIB): $(VCSSVN_OBJS) $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ +$(REFTABLE_LIB): $(REFTABLE_OBJS) + $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ + export DEFAULT_EDITOR DEFAULT_PAGER Documentation/GIT-EXCLUDED-PROGRAMS: FORCE
builtin/clone.c +4 -1
@@ -1109,7 +1109,10 @@ int cmd_clone(int argc, const char **argv, const char *prefix) } } -init_db (git_dir, real_git_dir, option_template,GIT_HASH_UNKNOWN ,INIT_DB_QUIET ); +init_db (git_dir, real_git_dir, option_template, +GIT_HASH_UNKNOWN , +DEFAULT_REF_STORAGE ,/* XXX */ +INIT_DB_QUIET );if (real_git_dir) git_dir = real_git_dir;
builtin/init-db.c +35 -14
125 lines not on this page · raw message
cache.h +3 -1
@@ -628,8 +628,9 @@ int path_inside_repo(const char *prefix, const char *path);int init_db (const char *git_dir,const char *real_git_dir,const char *template_dir,int hash_algo, +const char *ref_storage_format,unsigned int flags); -void initialize_repository_version (int hash_algo); +void initialize_repository_version (int hash_algo,const char *ref_storage_format);void sanitize_stdfds (void );int daemonize (void );
@@ -1043,6 +1044,7 @@ struct repository_format {int is_bare;int hash_algo;char *work_tree; +char *ref_storage;struct string_list unknown_extensions; };
refs.c +12 -8
@@ -20,7 +20,7 @@/* * List of all available backends */ -static struct ref_storage_be *refs_backends = &refs_be_files; +static struct ref_storage_be *refs_backends = &refs_be_reftable;static struct ref_storage_be *find_ref_storage_backend (const char *name) {
@@ -1836,13 +1836,13 @@ static struct ref_store *lookup_ref_store_map(struct hashmap *map,* Create, record, and return a ref_store instance for the specified * gitdir. */ -static struct ref_store *ref_store_init (const char *gitdir, +static struct ref_store *ref_store_init (const char *gitdir,const char *be_name,unsigned int flags) { -const char *be_name ="files" ; -struct ref_storage_be *be =find_ref_storage_backend (be_name); +struct ref_storage_be *be;struct ref_store *refs; + be =find_ref_storage_backend (be_name);if (!be)BUG ("reference backend %s is unknown" , be_name);
@@ -1858,7 +1858,10 @@ struct ref_store *get_main_ref_store(struct repository *r)if (!r->gitdir)BUG ("attempting to get main_ref_store outside of repository" ); - r->refs =ref_store_init (r->gitdir,REF_STORE_ALL_CAPS ); + r->refs =ref_store_init (r->gitdir, + r->ref_storage_format ? r->ref_storage_format : +DEFAULT_REF_STORAGE , +REF_STORE_ALL_CAPS );return r->refs; }
@@ -1913,7 +1916,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule) goto done;/* assume that add_submodule_odb() has been called */ - refs =ref_store_init (submodule_sb.buf, + refs =ref_store_init (submodule_sb.buf,DEFAULT_REF_STORAGE ,/* XXX */ REF_STORE_READ |REF_STORE_ODB );register_ref_store_map (&submodule_ref_stores,"submodule" , refs, submodule);
@@ -1927,6 +1930,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule)struct ref_store *get_worktree_ref_store (const struct worktree *wt) { +const char *format =DEFAULT_REF_STORAGE ;/* XXX */ struct ref_store *refs;const char *id;
@@ -1940,9 +1944,9 @@ struct ref_store *get_worktree_ref_store(const struct worktree *wt)if (wt->id) refs =ref_store_init (git_common_path ("worktrees/%s" , wt->id), -REF_STORE_ALL_CAPS ); + format,REF_STORE_ALL_CAPS );else - refs =ref_store_init (get_git_common_dir (), + refs =ref_store_init (get_git_common_dir (), format,REF_STORE_ALL_CAPS );if (refs)
refs.h +2 -0
@@ -9,6 +9,8 @@ struct string_list;struct string_list_item ;struct worktree ; +#define DEFAULT_REF_STORAGE "files" +/* * Resolve a reference, recursively following symbolic refererences. *
refs/refs-internal.h +1 -0
@@ -661,6 +661,7 @@ struct ref_storage_be { };extern struct ref_storage_be refs_be_files; +extern struct ref_storage_be refs_be_reftable;extern struct ref_storage_be refs_be_packed;/*
refs/reftable-backend.c new file +1002 -0
1,002 lines not on this page · raw message
repository.c +2 -0
@@ -178,6 +178,8 @@ int repo_init(struct repository *repo,if (worktree)repo_set_worktree (repo, worktree); + repo->ref_storage_format =xstrdup_or_null (format.ref_storage); +clear_repository_format (&format);return 0 ;
repository.h +3 -0
@@ -70,6 +70,9 @@ struct repository {/* The store in which the refs are held. */ struct ref_store *refs; +/* The format to use for the ref database. */ +char *ref_storage_format; +/* * Contains path to often used file names. */
setup.c +9 -3
@@ -468,9 +468,11 @@ static int check_repo_format(const char *var, const char *value, void *vdata)if (!value)return config_error_nonbool (var); data->partial_clone =xstrdup (value); - }else if (!strcmp (ext,"worktreeconfig" )) + }else if (!strcmp (ext,"worktreeconfig" )) { data->worktree_config =git_config_bool (var, value); -else + }else if (!strcmp (ext,"refstorage" )) { + data->ref_storage =xstrdup (value); + }else string_list_append (&data->unknown_extensions, ext); }
@@ -559,6 +561,7 @@ void clear_repository_format(struct repository_format *format)string_list_clear (&format->unknown_extensions,0 );free (format->work_tree);free (format->partial_clone); +free (format->ref_storage);init_repository_format (format); }
@@ -1204,8 +1207,11 @@ const char *setup_git_directory_gently(int *nongit_ok) gitdir =DEFAULT_GIT_DIR_ENVIRONMENT ;setup_git_env (gitdir); } -if (startup_info->have_repository) +if (startup_info->have_repository) {repo_set_hash_algo (the_repository, repo_fmt.hash_algo); + the_repository->ref_storage_format = +xstrdup_or_null (repo_fmt.ref_storage); + } }strbuf_release (&dir);
t/t0031-reftable.sh new file +35 -0
@@ -0,0 +1,35 @@ +#!/bin/sh +# +# Copyright (c) 2020 Google LLC +# + +test_description='reftable basics' + +. ./test-lib.sh + +test_expect_success 'basic operation of reftable storage' ' + git init --ref-storage=reftable repo && ( + cd repo && + echo "hello" >world.txt && + git add world.txt && + git commit -m "first post" && + test_write_lines HEAD refs/heads/master >expect && + git show-ref && + git show-ref | cut -f2 -d" " > actual && + test_cmp actual expect && + for count in $(test_seq 1 10) + do + echo "hello" >>world.txt + git commit -m "number ${count}" world.txt || + return 1 + done && + git gc && + nfiles=$(ls -1 .git/reftable | wc -l ) && + test ${nfiles} = "2" && + git reflog refs/heads/master >output && + test_line_count = 11 output && + grep "commit (initial): first post" output && + grep "commit: number 10" output ) +' + +test_done
Signature · 1 line
gitgitgadget
Quoted 1 line · Han-Wen Nienhuys via GitGitGadget
This adds the reftable library, and hooks it up as a ref backend.
Since I queued the topic to 'pu', I needed to apply these fix-up patches on top. The change to the Makefile is about "make clean" which forgot to clean the reftable library. All the rest are ws clean-ups.
There are build breakages reported on Windows, with possible fixes (which IIRC were reported to segfault X-<), but I do not have URL or message-IDs handy.
Thanks.
Makefile | 2 +- builtin/clone.c | 2 +- builtin/init-db.c | 4 ++-- cache.h | 2 +- reftable/reftable.h | 8 ++++---- 5 files changed, 9 insertions(+), 9 deletions(-)
Makefile +1 -1
@@ -3126,7 +3126,7 @@ cocciclean: clean: profile-clean coverage-clean cocciclean $(RM) *.res $(RM) $(OBJECTS) - $(RM) $(LIB_FILE) $(XDIFF_LIB) $(VCSSVN_LIB) + $(RM) $(LIB_FILE) $(XDIFF_LIB) $(VCSSVN_LIB) $(REFTABLE_LIB) $(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) git$X $(RM) $(TEST_PROGRAMS) $(RM) $(FUZZ_PROGRAMS)
builtin/clone.c +1 -1
@@ -1110,7 +1110,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix) }init_db (git_dir, real_git_dir, option_template, -GIT_HASH_UNKNOWN , +GIT_HASH_UNKNOWN ,DEFAULT_REF_STORAGE ,/* XXX */ INIT_DB_QUIET );
builtin/init-db.c +2 -2
@@ -394,7 +394,7 @@ static void validate_hash_algorithm(struct repository_format *repo_fmt, int hashint init_db (const char *git_dir,const char *real_git_dir,const char *template_dir,int hash,const char *ref_storage_format, -unsigned int flags) +unsigned int flags) {int reinit;int exist_ok = flags &INIT_DB_EXIST_OK ;
@@ -433,7 +433,7 @@ int init_db(const char *git_dir, const char *real_git_dir,* is an attempt to reinitialize new repository with an old tool. */ check_repository_format (&repo_fmt); - repo_fmt.ref_storage =xstrdup (ref_storage_format); + repo_fmt.ref_storage =xstrdup (ref_storage_format);validate_hash_algorithm (&repo_fmt, hash);
cache.h +1 -1
@@ -628,7 +628,7 @@ int path_inside_repo(const char *prefix, const char *path);int init_db (const char *git_dir,const char *real_git_dir,const char *template_dir,int hash_algo, -const char *ref_storage_format, +const char *ref_storage_format,unsigned int flags);void initialize_repository_version (int hash_algo,const char *ref_storage_format);
reftable/reftable.h +4 -4
@@ -446,13 +446,13 @@ struct reftable_addition;/* returns a new transaction to add reftables to the given stack. As a side effect, the ref database is locked. -*/ +*/ int reftable_stack_new_addition (struct reftable_addition **dest,struct reftable_stack *st); -/* Adds a reftable to transaction. */ +/* Adds a reftable to transaction. */ int reftable_addition_add (struct reftable_addition *add, -int (*write_table)(struct reftable_writer *wr,void *arg), -void *arg); +int (*write_table)(struct reftable_writer *wr,void *arg), +void *arg);/* Commits the transaction, releasing the lock. */ int reftable_addition_commit (struct reftable_addition *add);
Quoted 4 lines · Junio C Hamano
Quoted 1 line · Han-Wen Nienhuys via GitGitGadget
This adds the reftable library, and hooks it up as a ref backend.
Quoted 8 lines
Since I queued the topic to 'pu', I needed to apply these fix-up patches on top. The change to the
Makefileis about "make clean" which forgot to clean the reftable library. All the rest are ws clean-ups.There are build breakages reported on Windows, with possible fixes (which IIRC were reported to segfault X-<), but I do not have URL or message-IDs handy.
FYI, it's also segfault in Linux even if only the test is applied. IOW, the code to fix breakages on Windows doesn't affect the segfault.
Here is the message: <[email protected]>
Signature · 1 line
Danh
v9 cover Reftable support git-core
In reply to Han-Wen Nienhuys via GitGitGadgetThis adds the reftable library, and hooks it up as a ref backend.
Feedback wanted:
* spots marked with XXX in the Git source code.
* what is a good test strategy? Could we have a CI flavor where we flip the default to reftable, and see how it fares?
v9
* Added spec as technical doc.
* Use 4-byte hash ID as API. Storage format for SHA256 is still pending discussion.
v10
* Serialize 4 byte hash ID to the version-2 format
v11
* use reftable_ namespace prefix.
* transaction API to provide locking.
* reorganized record.{h,c}. More doc in reftable.h
* support log record deletion.
* pluggable malloc/free
v12
* More doc in slice.h, block.h, reader.h, writer.h, basics.h
* Folded in Dscho's patch for struct initialization
* Add "refs/" as default prefix for for_each_[raw]ref
* Fix nullptr deref in log record copying.
* Copy over the iteration prefix; this fixes "warning: bad replace ref
name: e"
* Some unresolved issues with "git gc"; disabled that part of t0031 for now.
Han-Wen Nienhuys (9):
refs.h: clarify reflog iteration order
Iterate over the "refs/" namespace in for_each_[raw]ref create .git/refs in files-backend.c
refs: document how ref_iterator_advance_fn should handle symrefs Add .gitattributes for the reftable/ directory
reftable: define version 2 of the spec to accomodate SHA256
reftable: clarify how empty tables should be written Add reftable library
Reftable support for git-core
Jonathan Nieder (1):
reftable: file format documentation
Documentation/Makefile | 1 + Documentation/technical/reftable.txt | 1080 ++++++++++++++++ .../technical/repository-version.txt | 7 + Makefile | 26 +- builtin/clone.c | 5 +- builtin/init-db.c | 51 +- cache.h | 4 +- refs.c | 26 +- refs.h | 7 +- refs/files-backend.c | 6 + refs/refs-internal.h | 6 + refs/reftable-backend.c | 1021 +++++++++++++++ reftable/.gitattributes | 1 + reftable/LICENSE | 31 + reftable/README.md | 11 + reftable/VERSION | 5 + reftable/basics.c | 189 +++ reftable/basics.h | 53 + reftable/block.c | 425 +++++++ reftable/block.h | 124 ++ reftable/blocksource.h | 22 + reftable/bytes.c | 0 reftable/config.h | 1 + reftable/constants.h | 21 + reftable/dump.c | 97 ++ reftable/file.c | 98 ++ reftable/iter.c | 234 ++++ reftable/iter.h | 60 + reftable/merged.c | 307 +++++ reftable/merged.h | 34 + reftable/pq.c | 115 ++ reftable/pq.h | 34 + reftable/reader.c | 754 +++++++++++ reftable/reader.h | 68 + reftable/record.c | 1119 ++++++++++++++++ reftable/record.h | 117 ++ reftable/reftable.h | 523 ++++++++ reftable/slice.c | 224 ++++ reftable/slice.h | 76 ++ reftable/stack.c | 1132 +++++++++++++++++ reftable/stack.h | 42 + reftable/system.h | 53 + reftable/tree.c | 67 + reftable/tree.h | 34 + reftable/update.sh | 14 + reftable/writer.c | 661 ++++++++++ reftable/writer.h | 60 + reftable/zlib-compat.c | 92 ++ repository.c | 2 + repository.h | 3 + setup.c | 12 +- t/t0031-reftable.sh | 35 + 52 files changed, 9155 insertions(+), 35 deletions(-) create mode 100644 Documentation/technical/reftable.txt create mode 100644 refs/reftable-backend.c create mode 100644 reftable/.gitattributes create mode 100644 reftable/LICENSE create mode 100644 reftable/README.md create mode 100644 reftable/VERSION create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/blocksource.h create mode 100644 reftable/bytes.c create mode 100644 reftable/config.h create mode 100644 reftable/constants.h create mode 100644 reftable/dump.c create mode 100644 reftable/file.c create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/reftable.h create mode 100644 reftable/slice.c create mode 100644 reftable/slice.h create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/system.h create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100644 reftable/update.sh create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h create mode 100644 reftable/zlib-compat.c create mode 100755 t/t0031-reftable.sh
base-commit: 048abe1751e6727bfbacf7b80466d78e04631f94
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-539%2Fhanwen%2Freftable-v9
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-539/hanwen/reftable-v9
Pull-Request: https://github.com/gitgitgadget/git/pull/539
Range-diff vs v8:
1: 05634d26dd5 = 1: b600d0bc6dd refs.h: clarify reflog iteration order
-: ----------- > 2: 89b68145e8f Iterate over the "refs/" namespace in for_each_[raw]ref
2: 8d34e2c5c8b = 3: 91f1efe24ec create .git/refs in files-backend.c
3: d08f823844d = 4: 56f65a2a0d7 refs: document how ref_iterator_advance_fn should handle symrefs
4: ca95b3a371e = 5: b432c1cc2ae Add .gitattributes for the reftable/ directory
5: dfc8b131294 = 6: b1d94be691a reftable: file format documentation
6: 45ab5361750 = 7: 3e418d27f67 reftable: define version 2 of the spec to accomodate SHA256
7: 67ee5962e85 = 8: 6f62be4067b reftable: clarify how empty tables should be written
8: 4f9bdd7312b ! 9: a30001ad1e8 Add reftable library
@@ reftable/README.md (new)
## reftable/VERSION (new) ##
@@
-+commit 920f7413f7a25f308d9a203f584873f6753192ec
++commit b68690acad769d59e80cbb4f79c442ece133e6bd
+Author: Han-Wen Nienhuys <[email protected]>
-+Date: Wed Apr 1 12:41:41 2020 +0200
++Date: Mon Apr 20 21:33:11 2020 +0200
+
-+ C: remove stray debug printfs
++ C: fix search/replace error in dump.c
## reftable/basics.c (new) ##
@@
@@ reftable/basics.c (new)
+ out[1] = (uint8_t)((i)&0xff);
+}
+
-+/*
-+ find smallest index i in [0, sz) at which f(i) is true, assuming
-+ that f is ascending. Return sz if f(i) is false for all indices.
-+*/
+int binsearch(int sz, int (*f)(int k, void *args), void *args)
+{
+ int lo = 0;
@@ reftable/basics.c (new)
+ return len;
+}
+
-+/* parse a newline separated list of names. Empty names are discarded. */
+void parse_names(char *buf, int size, char ***namesp)
+{
+ char **names = NULL;
@@ reftable/basics.h (new)
+#define true 1
+#define false 0
+
++/* Bigendian en/decoding of integers */
++
+void put_be24(byte *out, uint32_t i);
+uint32_t get_be24(byte *in);
+void put_be16(uint8_t *out, uint16_t i);
+
++/*
++ find smallest index i in [0, sz) at which f(i) is true, assuming
++ that f is ascending. Return sz if f(i) is false for all indices.
++*/
+int binsearch(int sz, int (*f)(int k, void *args), void *args);
+
++/*
++ Frees a NULL terminated array of malloced strings. The array itself is also
++ freed.
++ */
+void free_names(char **a);
++
++/* parse a newline separated list of names. Empty names are discarded. */
+void parse_names(char *buf, int size, char ***namesp);
++
++/* compares two NULL-terminated arrays of strings. */
+int names_equal(char **a, char **b);
++
++/* returns the array size of a NULL-terminated array of strings. */
+int names_length(char **names);
+
++/* Allocation routines; they invoke the functions set through
++ * reftable_set_alloc() */
+void *reftable_malloc(size_t sz);
+void *reftable_realloc(void *p, size_t sz);
+void reftable_free(void *p);
@@ reftable/block.c (new)
+ abort();
+}
+
-+int block_writer_register_restart(struct reftable_block_writer *w, int n,
-+ bool restart, struct slice key);
++int block_writer_register_restart(struct block_writer *w, int n, bool restart,
++ struct slice key);
+
-+void block_writer_init(struct reftable_block_writer *bw, byte typ, byte *buf,
++void block_writer_init(struct block_writer *bw, byte typ, byte *buf,
+ uint32_t block_size, uint32_t header_off, int hash_size)
+{
+ bw->buf = buf;
@@ reftable/block.c (new)
+ bw->next = header_off + 4;
+ bw->restart_interval = 16;
+ bw->entries = 0;
++ bw->restart_len = 0;
++ bw->last_key.len = 0;
+}
+
-+byte block_writer_type(struct reftable_block_writer *bw)
++byte block_writer_type(struct block_writer *bw)
+{
+ return bw->buf[bw->header_off];
+}
+
+/* adds the record to the block. Returns -1 if it does not fit, 0 on
+ success */
-+int block_writer_add(struct reftable_block_writer *w, struct record rec)
++int block_writer_add(struct block_writer *w, struct record rec)
+{
+ struct slice empty = { 0 };
+ struct slice last = w->entries % w->restart_interval == 0 ? empty :
@@ reftable/block.c (new)
+ goto err;
+ }
+
-+ reftable_free(slice_yield(&key));
++ slice_clear(&key);
+ return 0;
+
+err:
-+ reftable_free(slice_yield(&key));
++ slice_clear(&key);
+ return -1;
+}
+
-+int block_writer_register_restart(struct reftable_block_writer *w, int n,
-+ bool restart, struct slice key)
++int block_writer_register_restart(struct block_writer *w, int n, bool restart,
++ struct slice key)
+{
+ int rlen = w->restart_len;
+ if (rlen >= MAX_RESTARTS) {
@@ reftable/block.c (new)
+ return 0;
+}
+
-+int block_writer_finish(struct reftable_block_writer *w)
++int block_writer_finish(struct block_writer *w)
+{
+ int i = 0;
+ for (i = 0; i < w->restart_len; i++) {
@@ reftable/block.c (new)
+ }
+
+ if (Z_OK != zresult) {
-+ reftable_free(slice_yield(&compressed));
++ slice_clear(&compressed);
+ return ZLIB_ERROR;
+ }
+
@@ reftable/block.c (new)
+ return w->next;
+}
+
-+byte block_reader_type(struct reftable_block_reader *r)
++byte block_reader_type(struct block_reader *r)
+{
+ return r->block.data[r->header_off];
+}
+
-+int block_reader_init(struct reftable_block_reader *br,
-+ struct reftable_block *block, uint32_t header_off,
-+ uint32_t table_block_size, int hash_size)
++int block_reader_init(struct block_reader *br, struct reftable_block *block,
++ uint32_t header_off, uint32_t table_block_size,
++ int hash_size)
+{
+ uint32_t full_block_size = table_block_size;
+ byte typ = block->data[header_off];
@@ reftable/block.c (new)
+ uncompressed.buf + block_header_skip,
+ &dst_len, block->data + block_header_skip,
+ &src_len)) {
-+ reftable_free(slice_yield(&uncompressed));
++ slice_clear(&uncompressed);
+ return ZLIB_ERROR;
+ }
+
@@ reftable/block.c (new)
+ return 0;
+}
+
-+static uint32_t block_reader_restart_offset(struct reftable_block_reader *br,
-+ int i)
++static uint32_t block_reader_restart_offset(struct block_reader *br, int i)
+{
+ return get_be24(br->restart_bytes + 3 * i);
+}
+
-+void block_reader_start(struct reftable_block_reader *br,
-+ struct reftable_block_iter *it)
++void block_reader_start(struct block_reader *br, struct block_iter *it)
+{
+ it->br = br;
+ slice_resize(&it->last_key, 0);
@@ reftable/block.c (new)
+struct restart_find_args {
+ int error;
+ struct slice key;
-+ struct reftable_block_reader *r;
++ struct block_reader *r;
+};
+
+static int restart_key_less(int idx, void *args)
@@ reftable/block.c (new)
+
+ {
+ int result = slice_compare(a->key, rkey);
-+ reftable_free(slice_yield(&rkey));
++ slice_clear(&rkey);
+ return result;
+ }
+}
+
-+void block_iter_copy_from(struct reftable_block_iter *dest,
-+ struct reftable_block_iter *src)
++void block_iter_copy_from(struct block_iter *dest, struct block_iter *src)
+{
+ dest->br = src->br;
+ dest->next_off = src->next_off;
@@ reftable/block.c (new)
+}
+
+/* return < 0 for error, 0 for OK, > 0 for EOF. */
-+int block_iter_next(struct reftable_block_iter *it, struct record rec)
++int block_iter_next(struct block_iter *it, struct record rec)
+{
+ if (it->next_off >= it->br->block_len) {
+ return 1;
@@ reftable/block.c (new)
+
+ slice_copy(&it->last_key, key);
+ it->next_off += start.len - in.len;
-+ reftable_free(slice_yield(&key));
++ slice_clear(&key);
+ return 0;
+ }
+}
+
-+int block_reader_first_key(struct reftable_block_reader *br, struct slice *key)
++int block_reader_first_key(struct block_reader *br, struct slice *key)
+{
+ struct slice empty = { 0 };
+ int off = br->header_off + 4;
@@ reftable/block.c (new)
+ return 0;
+}
+
-+int block_iter_seek(struct reftable_block_iter *it, struct slice want)
++int block_iter_seek(struct block_iter *it, struct slice want)
+{
+ return block_reader_seek(it->br, it, want);
+}
+
-+void block_iter_close(struct reftable_block_iter *it)
++void block_iter_close(struct block_iter *it)
+{
-+ reftable_free(slice_yield(&it->last_key));
++ slice_clear(&it->last_key);
+}
+
-+int block_reader_seek(struct reftable_block_reader *br,
-+ struct reftable_block_iter *it, struct slice want)
++int block_reader_seek(struct block_reader *br, struct block_iter *it,
++ struct slice want)
+{
+ struct restart_find_args args = {
+ .key = want,
@@ reftable/block.c (new)
+ struct slice key = { 0 };
+ int result = 0;
+ int err = 0;
-+ struct reftable_block_iter next = { 0 };
++ struct block_iter next = { 0 };
+ while (true) {
+ block_iter_copy_from(&next, it);
+
@@ reftable/block.c (new)
+ }
+
+ exit:
-+ reftable_free(slice_yield(&key));
-+ reftable_free(slice_yield(&next.last_key));
-+ record_clear(rec);
-+ reftable_free(record_yield(&rec));
++ slice_clear(&key);
++ slice_clear(&next.last_key);
++ record_destroy(&rec);
+
+ return result;
+ }
+}
+
-+void block_writer_reset(struct reftable_block_writer *bw)
-+{
-+ bw->restart_len = 0;
-+ bw->last_key.len = 0;
-+}
-+
-+void block_writer_clear(struct reftable_block_writer *bw)
++void block_writer_clear(struct block_writer *bw)
+{
+ FREE_AND_NULL(bw->restarts);
-+ reftable_free(slice_yield(&bw->last_key));
++ slice_clear(&bw->last_key);
+ /* the block is not owned. */
+}
@@ reftable/block.h (new)
+#include "record.h"
+#include "reftable.h"
+
-+struct reftable_block_writer {
++/*
++ Writes reftable blocks. The block_writer is reused across blocks to minimize
++ allocation overhead.
++*/
++struct block_writer {
+ byte *buf;
+ uint32_t block_size;
++
++ /* Offset ofof the global header. Nonzero in the first block only. */
+ uint32_t header_off;
++
++ /* How often to restart keys. */
+ int restart_interval;
+ int hash_size;
+
++ /* Offset of next byte to write. */
+ uint32_t next;
+ uint32_t *restarts;
+ uint32_t restart_len;
+ uint32_t restart_cap;
++
+ struct slice last_key;
+ int entries;
+};
+
-+void block_writer_init(struct reftable_block_writer *bw, byte typ, byte *buf,
++/*
++ initializes the blockwriter to write `typ` entries, using `buf` as temporary
++ storage. `buf` is not owned by the block_writer. */
++void block_writer_init(struct block_writer *bw, byte typ, byte *buf,
+ uint32_t block_size, uint32_t header_off, int hash_size);
-+byte block_writer_type(struct reftable_block_writer *bw);
-+int block_writer_add(struct reftable_block_writer *w, struct record rec);
-+int block_writer_finish(struct reftable_block_writer *w);
-+void block_writer_reset(struct reftable_block_writer *bw);
-+void block_writer_clear(struct reftable_block_writer *bw);
+
-+struct reftable_block_reader {
++/*
++ returns the block type (eg. 'r' for ref records.
++*/
++byte block_writer_type(struct block_writer *bw);
++
++/* appends the record, or -1 if it doesn't fit. */
++int block_writer_add(struct block_writer *w, struct record rec);
++
++/* appends the key restarts, and compress the block if necessary. */
++int block_writer_finish(struct block_writer *w);
++
++/* clears out internally allocated block_writer members. */
++void block_writer_clear(struct block_writer *bw);
++
++/* Read a block. */
++struct block_reader {
++ /* offset of the block header; nonzero for the first block in a
++ * reftable. */
+ uint32_t header_off;
++
++ /* the memory block */
+ struct reftable_block block;
+ int hash_size;
+
+ /* size of the data, excluding restart data. */
+ uint32_t block_len;
+ byte *restart_bytes;
-+ uint32_t full_block_size;
+ uint16_t restart_count;
++
++ /* size of the data in the file. For log blocks, this is the compressed
++ * size. */
++ uint32_t full_block_size;
+};
+
-+struct reftable_block_iter {
++/* Iterate over entries in a block */
++struct block_iter {
++ /* offset within the block of the next entry to read. */
+ uint32_t next_off;
-+ struct reftable_block_reader *br;
++ struct block_reader *br;
++
++ /* key for last entry we read. */
+ struct slice last_key;
+};
+
-+int block_reader_init(struct reftable_block_reader *br,
-+ struct reftable_block *bl, uint32_t header_off,
-+ uint32_t table_block_size, int hash_size);
-+void block_reader_start(struct reftable_block_reader *br,
-+ struct reftable_block_iter *it);
-+int block_reader_seek(struct reftable_block_reader *br,
-+ struct reftable_block_iter *it, struct slice want);
-+byte block_reader_type(struct reftable_block_reader *r);
-+int block_reader_first_key(struct reftable_block_reader *br, struct slice *key);
-+
-+void block_iter_copy_from(struct reftable_block_iter *dest,
-+ struct reftable_block_iter *src);
-+int block_iter_next(struct reftable_block_iter *it, struct record rec);
-+int block_iter_seek(struct reftable_block_iter *it, struct slice want);
-+void block_iter_close(struct reftable_block_iter *it);
++/* initializes a block reader */
++int block_reader_init(struct block_reader *br, struct reftable_block *bl,
++ uint32_t header_off, uint32_t table_block_size,
++ int hash_size);
++
++/* Position `it` at start of the block */
++void block_reader_start(struct block_reader *br, struct block_iter *it);
++
++/* Position `it` to the `want` key in the block */
++int block_reader_seek(struct block_reader *br, struct block_iter *it,
++ struct slice want);
++
++/* Returns the block type (eg. 'r' for refs) */
++byte block_reader_type(struct block_reader *r);
++
++/* Decodes the first key in the block */
++int block_reader_first_key(struct block_reader *br, struct slice *key);
++
++void block_iter_copy_from(struct block_iter *dest, struct block_iter *src);
++int block_iter_next(struct block_iter *it, struct record rec);
++
++/* Seek to `want` with in the block pointed to by `it` */
++int block_iter_seek(struct block_iter *it, struct slice want);
++
++/* deallocate memory for `it`. The block reader and its block is left intact. */
++void block_iter_close(struct block_iter *it);
+
++/* size of file header, depending on format version */
+int header_size(int version);
++
++/* size of file footer, depending on format version */
+int footer_size(int version);
+
+#endif
@@ reftable/iter.c (new)
+{
+ struct filtering_ref_iterator *fri =
+ (struct filtering_ref_iterator *)iter_arg;
-+ reftable_free(slice_yield(&fri->oid));
++ slice_clear(&fri->oid);
+ reftable_iterator_destroy(&fri->it);
+}
+
@@ reftable/iter.c (new)
+ struct indexed_table_ref_iter *it = (struct indexed_table_ref_iter *)p;
+ block_iter_close(&it->cur);
+ reader_return_block(it->r, &it->block_reader.block);
-+ reftable_free(slice_yield(&it->oid));
++ slice_clear(&it->oid);
+}
+
+static int indexed_table_ref_iter_next_block(struct indexed_table_ref_iter *it)
@@ reftable/iter.h (new)
+int iterator_next(struct reftable_iterator it, struct record rec);
+bool iterator_is_null(struct reftable_iterator it);
+
++/* iterator that produces only ref records that point to `oid` */
+struct filtering_ref_iterator {
+ bool double_check;
+ struct reftable_reader *r;
@@ reftable/iter.h (new)
+void iterator_from_filtering_ref_iterator(struct reftable_iterator *,
+ struct filtering_ref_iterator *);
+
++/* iterator that produces only ref records that point to `oid`,
++ but using the object index.
++ */
+struct indexed_table_ref_iter {
+ struct reftable_reader *r;
+ struct slice oid;
@@ reftable/iter.h (new)
+ /* Points to the next offset to read. */
+ int offset_idx;
+ int offset_len;
-+ struct reftable_block_reader block_reader;
-+ struct reftable_block_iter cur;
++ struct block_reader block_reader;
++ struct block_iter cur;
+ bool finished;
+};
+
@@ reftable/merged.c (new)
+
+ if (err > 0) {
+ reftable_iterator_destroy(&mi->stack[i]);
-+ record_clear(rec);
-+ reftable_free(record_yield(&rec));
++ record_destroy(&rec);
+ } else {
+ struct pq_entry e = {
+ .rec = rec,
@@ reftable/merged.c (new)
+
+ if (err > 0) {
+ reftable_iterator_destroy(&mi->stack[idx]);
-+ record_clear(rec);
-+ reftable_free(record_yield(&rec));
++ record_destroy(&rec);
+ return 0;
+ }
+
@@ reftable/merged.c (new)
+ return err;
+ }
+
++
++ /*
++ One can also use reftable as datacenter-local storage, where the ref
++ database is maintained in globally consistent database (eg.
++ CockroachDB or Spanner). In this scenario, replication delays together
++ with compaction may cause newer tables to contain older entries. In
++ such a deployment, the loop below must be changed to collect all
++ entries for the same key, and return new the newest one.
++ */
+ record_key(entry.rec, &entry_key);
+ while (!merged_iter_pqueue_is_empty(mi->pq)) {
+ struct pq_entry top = merged_iter_pqueue_top(mi->pq);
@@ reftable/merged.c (new)
+ record_key(top.rec, &k);
+
+ cmp = slice_compare(k, entry_key);
-+ reftable_free(slice_yield(&k));
++ slice_clear(&k);
+
+ if (cmp > 0) {
+ break;
@@ reftable/merged.c (new)
+ record_copy_from(rec, entry.rec, hash_size(mi->hash_id));
+ record_clear(entry.rec);
+ reftable_free(record_yield(&entry.rec));
-+ reftable_free(slice_yield(&entry_key));
++ slice_clear(&entry_key);
+ return 0;
+}
+
@@ reftable/merged.c (new)
+ return err;
+ }
+
-+ merged.stack_len = n, err = merged_iter_init(&merged);
++ merged.stack_len = n;
++ err = merged_iter_init(&merged);
+ if (err < 0) {
+ merged_iter_close(&merged);
+ return err;
@@ reftable/pq.c (new)
+
+ cmp = slice_compare(ak, bk);
+
-+ reftable_free(slice_yield(&ak));
-+ reftable_free(slice_yield(&bk));
++ slice_clear(&ak);
++ slice_clear(&bk);
+
+ if (cmp == 0) {
+ return a.index > b.index;
@@ reftable/reader.c (new)
+ struct reftable_reader *r;
+ byte typ;
+ uint64_t block_off;
-+ struct reftable_block_iter bi;
++ struct block_iter bi;
+ bool finished;
+};
+
@@ reftable/reader.c (new)
+ return result;
+}
+
-+int reader_init_block_reader(struct reftable_reader *r,
-+ struct reftable_block_reader *br,
++int reader_init_block_reader(struct reftable_reader *r, struct block_reader *br,
+ uint64_t next_off, byte want_typ)
+{
+ int32_t guess_block_size = r->block_size ? r->block_size :
@@ reftable/reader.c (new)
+ struct table_iter *src)
+{
+ uint64_t next_block_off = src->block_off + src->bi.br->full_block_size;
-+ struct reftable_block_reader br = { 0 };
++ struct block_reader br = { 0 };
+ int err = 0;
+
+ dest->r = src->r;
@@ reftable/reader.c (new)
+ }
+
+ {
-+ struct reftable_block_reader *brp =
-+ reftable_malloc(sizeof(struct reftable_block_reader));
++ struct block_reader *brp =
++ reftable_malloc(sizeof(struct block_reader));
+ *brp = br;
+
+ dest->finished = false;
@@ reftable/reader.c (new)
+static int reader_table_iter_at(struct reftable_reader *r,
+ struct table_iter *ti, uint64_t off, byte typ)
+{
-+ struct reftable_block_reader br = { 0 };
-+ struct reftable_block_reader *brp = NULL;
++ struct block_reader br = { 0 };
++ struct block_reader *brp = NULL;
+
+ int err = reader_init_block_reader(r, &br, off, typ);
+ if (err != 0) {
+ return err;
+ }
+
-+ brp = reftable_malloc(sizeof(struct reftable_block_reader));
++ brp = reftable_malloc(sizeof(struct block_reader));
+ *brp = br;
+ ti->r = r;
+ ti->typ = block_reader_type(brp);
@@ reftable/reader.c (new)
+
+exit:
+ block_iter_close(&next.bi);
-+ record_clear(rec);
-+ reftable_free(record_yield(&rec));
-+ reftable_free(slice_yield(&want_key));
-+ reftable_free(slice_yield(&got_key));
++ record_destroy(&rec);
++ slice_clear(&want_key);
++ slice_clear(&got_key);
+ return err;
+}
+
@@ reftable/reader.h (new)
+ struct reftable_block *ret);
+void block_source_close(struct reftable_block_source *source);
+
++/* metadata for a block type */
+struct reftable_reader_offsets {
+ bool present;
+ uint64_t offset;
+ uint64_t index_offset;
+};
+
++/* The state for reading a reftable file. */
+struct reftable_reader {
++ /* for convience, associate a name with the instance. */
+ char *name;
+ struct reftable_block_source source;
-+ uint32_t hash_id;
+
-+ // Size of the file, excluding the footer.
++ /* Size of the file, excluding the footer. */
+ uint64_t size;
++
++ /* 'sha1' for SHA1, 's256' for SHA-256 */
++ uint32_t hash_id;
++
+ uint32_t block_size;
+ uint64_t min_update_index;
+ uint64_t max_update_index;
++ /* Length of the OID keys in the 'o' section */
+ int object_id_len;
+ int version;
+
@@ reftable/reader.h (new)
+void reader_close(struct reftable_reader *r);
+const char *reader_name(struct reftable_reader *r);
+void reader_return_block(struct reftable_reader *r, struct reftable_block *p);
-+int reader_init_block_reader(struct reftable_reader *r,
-+ struct reftable_block_reader *br,
++
++/* initialize a block reader to read from `r` */
++int reader_init_block_reader(struct reftable_reader *r, struct block_reader *br,
+ uint64_t next_off, byte want_typ);
+
+#endif
@@ reftable/record.c (new)
+ (const struct reftable_log_record *)src_rec;
+
+ *dst = *src;
-+ dst->ref_name = xstrdup(dst->ref_name);
-+ dst->email = xstrdup(dst->email);
-+ dst->name = xstrdup(dst->name);
-+ dst->message = xstrdup(dst->message);
++ if (dst->ref_name != NULL) {
++ dst->ref_name = xstrdup(dst->ref_name);
++ }
++ if (dst->email != NULL) {
++ dst->email = xstrdup(dst->email);
++ }
++ if (dst->name != NULL) {
++ dst->name = xstrdup(dst->name);
++ }
++ if (dst->message != NULL) {
++ dst->message = xstrdup(dst->message);
++ }
++
+ if (dst->new_hash != NULL) {
+ dst->new_hash = reftable_malloc(hash_size);
+ memcpy(dst->new_hash, src->new_hash, hash_size);
@@ reftable/record.c (new)
+ return start.len - in.len;
+
+error:
-+ reftable_free(slice_yield(&dest));
++ slice_clear(&dest);
+ return FORMAT_ERROR;
+}
+
@@ reftable/record.c (new)
+ return rec;
+}
+
++void *record_yield(struct record *rec)
++{
++ void *p = rec->data;
++ rec->data = NULL;
++ return p;
++}
++
++void record_destroy(struct record *rec)
++{
++ record_clear(*rec);
++ reftable_free(record_yield(rec));
++}
++
+static byte index_record_type(void)
+{
+ return BLOCK_TYPE_INDEX;
@@ reftable/record.c (new)
+static void index_record_clear(void *rec)
+{
+ struct index_record *idx = (struct index_record *)rec;
-+ reftable_free(slice_yield(&idx->last_key));
++ slice_clear(&idx->last_key);
+}
+
+static byte index_record_val_type(const void *rec)
@@ reftable/record.c (new)
+ rec->ops = &reftable_log_record_vtable;
+}
+
-+void *record_yield(struct record *rec)
-+{
-+ void *p = rec->data;
-+ rec->data = NULL;
-+ return p;
-+}
-+
+struct reftable_ref_record *record_as_ref(struct record rec)
+{
+ assert(record_type(rec) == BLOCK_TYPE_REF);
+ return (struct reftable_ref_record *)rec.data;
+}
+
++struct reftable_log_record *record_as_log(struct record rec)
++{
++ assert(record_type(rec) == BLOCK_TYPE_LOG);
++ return (struct reftable_log_record *)rec.data;
++}
++
+static bool hash_equal(byte *a, byte *b, int hash_size)
+{
+ if (a != NULL && b != NULL) {
@@ reftable/record.h (new)
+ struct record_vtable *ops;
+};
+
++/* returns true for recognized block types. Block start with the block type. */
+int is_block_type(byte typ);
+
++/* creates a malloced record of the given type. Dispose with record_destroy */
+struct record new_record(byte typ);
+
+extern struct record_vtable reftable_ref_record_vtable;
+
++/* Encode `key` into `dest`. Sets `restart` to indicate a restart. Returns
++ number of bytes written. */
+int encode_key(bool *restart, struct slice dest, struct slice prev_key,
+ struct slice key, byte extra);
++
++/* Decode into `key` and `extra` from `in` */
+int decode_key(struct slice *key, byte *extra, struct slice last_key,
+ struct slice in);
+
@@ reftable/record.h (new)
+int record_encode(struct record rec, struct slice dest, int hash_size);
+int record_decode(struct record rec, struct slice key, byte extra,
+ struct slice src, int hash_size);
++
++/* zeroes out the embedded record */
+void record_clear(struct record rec);
+
+/* clear out the record, yielding the record data that was encapsulated. */
+void *record_yield(struct record *rec);
+
++/* clear and deallocate embedded record, and zero `rec`. */
++void record_destroy(struct record *rec);
++
+/* initialize generic records from concrete records. The generic record should
+ * be zeroed out. */
-+
+void record_from_obj(struct record *rec, struct obj_record *objrec);
+void record_from_index(struct record *rec, struct index_record *idxrec);
+void record_from_ref(struct record *rec, struct reftable_ref_record *refrec);
+void record_from_log(struct record *rec, struct reftable_log_record *logrec);
+struct reftable_ref_record *record_as_ref(struct record ref);
++struct reftable_log_record *record_as_log(struct record ref);
+
+/* for qsort. */
+int reftable_ref_record_compare_name(const void *a, const void *b);
@@ reftable/reftable.h (new)
+/*
+ returns a new transaction to add reftables to the given stack. As a side
+ effect, the ref database is locked.
-+*/
-+int reftable_stack_new_addition(struct reftable_addition **dest, struct reftable_stack *st);
++*/
++int reftable_stack_new_addition(struct reftable_addition **dest,
++ struct reftable_stack *st);
+
-+/* Adds a reftable to transaction. */
++/* Adds a reftable to transaction. */
+int reftable_addition_add(struct reftable_addition *add,
-+ int (*write_table)(struct reftable_writer *wr, void *arg),
-+ void *arg);
++ int (*write_table)(struct reftable_writer *wr,
++ void *arg),
++ void *arg);
+
+/* Commits the transaction, releasing the lock. */
+int reftable_addition_commit(struct reftable_addition *add);
+
-+/* Release all non-committed data from the transaction; releases the lock if held. */
++/* Release all non-committed data from the transaction; releases the lock if
++ * held. */
+void reftable_addition_close(struct reftable_addition *add);
+
+/* add a new table to the stack. The write_table function must call
@@ reftable/slice.c (new)
+ return p;
+}
+
++void slice_clear(struct slice *s)
++{
++ reftable_free(slice_yield(s));
++}
++
+void slice_copy(struct slice *dest, struct slice src)
+{
+ slice_resize(dest, src.len);
@@ reftable/slice.h (new)
+#include "basics.h"
+#include "reftable.h"
+
++/*
++ provides bounds-checked byte ranges.
++ To use, initialize as "slice x = {0};"
++ */
+struct slice {
-+ byte *buf;
+ int len;
+ int cap;
++ byte *buf;
+};
+
-+void slice_set_string(struct slice *dest, const char *);
-+void slice_append_string(struct slice *dest, const char *);
++void slice_set_string(struct slice *dest, const char *src);
++void slice_append_string(struct slice *dest, const char *src);
++/* Set length to 0, but retain buffer */
++void slice_clear(struct slice *slice);
++
++/* Return a malloced string for `src` */
+char *slice_to_string(struct slice src);
++
++/* Ensure that `buf` is \0 terminated. */
+const char *slice_as_string(struct slice *src);
++
++/* Compare slices */
+bool slice_equal(struct slice a, struct slice b);
++
++/* Return `buf`, clearing out `s` */
+byte *slice_yield(struct slice *s);
++
++/* Copy bytes */
+void slice_copy(struct slice *dest, struct slice src);
++
++/* Advance `buf` by `n`, and decrease length. A copy of the slice
++ should be kept for deallocating the slice. */
+void slice_consume(struct slice *s, int n);
++
++/* Set length of the slice to `l` */
+void slice_resize(struct slice *s, int l);
++
++/* Signed comparison */
+int slice_compare(struct slice a, struct slice b);
-+int slice_write(struct slice *b, byte *data, int sz);
-+int slice_write_void(void *b, byte *data, int sz);
++
++/* Append `data` to the `dest` slice. */
++int slice_write(struct slice *dest, byte *data, int sz);
++
++/* Append `add` to `dest. */
+void slice_append(struct slice *dest, struct slice add);
++
++/* Like slice_write, but suitable for passing to reftable_new_writer
++ */
++int slice_write_void(void *b, byte *data, int sz);
++
++/* Find the longest shared prefix size of `a` and `b` */
+int common_prefix_size(struct slice a, struct slice b);
+
+struct reftable_block_source;
++
++/* Create an in-memory block source for reading reftables */
+void block_source_from_slice(struct reftable_block_source *bs,
+ struct slice *buf);
+
@@ reftable/stack.c (new)
+ slice_append_string(&list_file_name, "/reftables.list");
+
+ p->list_file = slice_to_string(list_file_name);
-+ reftable_free(slice_yield(&list_file_name));
++ slice_clear(&list_file_name);
+ p->reftable_dir = xstrdup(dir);
+ p->config = config;
+
@@ reftable/stack.c (new)
+ }
+ }
+exit:
-+ reftable_free(slice_yield(&table_path));
++ slice_clear(&table_path);
+ {
+ int i = 0;
+ for (i = 0; i < new_tables_len; i++) {
@@ reftable/stack.c (new)
+ }
+ if (add->lock_file_name.len > 0) {
+ unlink(slice_as_string(&add->lock_file_name));
-+ reftable_free(slice_yield(&add->lock_file_name));
++ slice_clear(&add->lock_file_name);
+ }
+
+ free_names(add->names);
@@ reftable/stack.c (new)
+ }
+
+ err = write(add->lock_file_fd, table_list.buf, table_list.len);
-+ free(slice_yield(&table_list));
++ slice_clear(&table_list);
+ if (err < 0) {
+ err = IO_ERROR;
+ goto exit;
@@ reftable/stack.c (new)
+ unlink(slice_as_string(&temp_tab_file_name));
+ }
+
-+ reftable_free(slice_yield(&temp_tab_file_name));
-+ reftable_free(slice_yield(&tab_file_name));
-+ reftable_free(slice_yield(&next_name));
++ slice_clear(&temp_tab_file_name);
++ slice_clear(&tab_file_name);
++ slice_clear(&next_name);
+ reftable_writer_free(wr);
+ return err;
+}
@@ reftable/stack.c (new)
+ }
+ if (err != 0 && temp_tab->len > 0) {
+ unlink(slice_as_string(temp_tab));
-+ reftable_free(slice_yield(temp_tab));
++ slice_clear(temp_tab);
+ }
-+ reftable_free(slice_yield(&next_name));
++ slice_clear(&next_name);
+ return err;
+}
+
@@ reftable/stack.c (new)
+ if (have_lock) {
+ unlink(slice_as_string(&lock_file_name));
+ }
-+ reftable_free(slice_yield(&new_table_name));
-+ reftable_free(slice_yield(&new_table_path));
-+ reftable_free(slice_yield(&ref_list_contents));
-+ reftable_free(slice_yield(&temp_tab_file_name));
-+ reftable_free(slice_yield(&lock_file_name));
++ slice_clear(&new_table_name);
++ slice_clear(&new_table_path);
++ slice_clear(&ref_list_contents);
++ slice_clear(&temp_tab_file_name);
++ slice_clear(&lock_file_name);
+ return err;
+}
+
@@ reftable/tree.h (new)
+#ifndef TREE_H
+#define TREE_H
+
++/* tree_node is a generic binary search tree. */
+struct tree_node {
+ void *key;
+ struct tree_node *left, *right;
+};
+
++/* looks for `key` in `rootp` using `compare` as comparison function. If insert
++ is set, insert the key if it's not found. Else, return NULL.
++*/
+struct tree_node *tree_search(void *key, struct tree_node **rootp,
+ int (*compare)(const void *, const void *),
+ int insert);
++
++/* performs an infix walk of the tree. */
+void infix_walk(struct tree_node *t, void (*action)(void *arg, void *key),
+ void *arg);
++
++/*
++ deallocates the tree nodes recursively. Keys should be deallocated separately
++ by walking over the tree. */
+void tree_free(struct tree_node *t);
+
+#endif
@@ reftable/writer.c (new)
+
+ result = 0;
+exit:
-+ reftable_free(slice_yield(&key));
++ slice_clear(&key);
+ return result;
+}
+
@@ reftable/writer.c (new)
+ assert(err == 0);
+ }
+ for (i = 0; i < idx_len; i++) {
-+ reftable_free(slice_yield(&idx[i].last_key));
++ slice_clear(&idx[i].last_key);
+ }
+ reftable_free(idx);
+ }
@@ reftable/writer.c (new)
+ struct obj_index_tree_node *entry = (struct obj_index_tree_node *)key;
+
+ FREE_AND_NULL(entry->offsets);
-+ reftable_free(slice_yield(&entry->hash));
++ slice_clear(&entry->hash);
+ reftable_free(entry);
+}
+
@@ reftable/writer.c (new)
+{
+ byte footer[72];
+ byte *p = footer;
-+
+ int err = writer_finish_public_section(w);
++ int empty_table = w->next == 0;
+ if (err != 0) {
+ goto exit;
+ }
++ w->pending_padding = 0;
++ if (empty_table) {
++ // Empty tables need a header anyway.
++ byte header[28];
++ int n = writer_write_header(w, header);
++ err = padded_write(w, header, n, 0);
++ if (err < 0) {
++ goto exit;
++ }
++ }
+
+ p += writer_write_header(w, footer);
+ put_be64(p, w->stats.ref_stats.index_offset);
@@ reftable/writer.c (new)
+
+ put_be32(p, crc32(0, footer, p - footer));
+ p += 4;
-+ w->pending_padding = 0;
+
+ err = padded_write(w, footer, footer_size(writer_version(w)), 0);
+ if (err < 0) {
+ goto exit;
+ }
+
-+ if (w->stats.log_stats.entries + w->stats.ref_stats.entries == 0) {
++ if (empty_table) {
+ err = EMPTY_TABLE_ERROR;
+ goto exit;
+ }
@@ reftable/writer.c (new)
+ /* free up memory. */
+ block_writer_clear(&w->block_writer_data);
+ writer_clear_index(w);
-+ reftable_free(slice_yield(&w->last_key));
++ slice_clear(&w->last_key);
+ return err;
+}
+
@@ reftable/writer.c (new)
+{
+ int i = 0;
+ for (i = 0; i < w->index_len; i++) {
-+ reftable_free(slice_yield(&w->index[i].last_key));
++ slice_clear(&w->index[i].last_key);
+ }
+
+ FREE_AND_NULL(w->index);
@@ reftable/writer.c (new)
+
+ w->index_len++;
+ w->next += padding + raw_bytes;
-+ block_writer_reset(&w->block_writer_data);
+ w->block_writer = NULL;
+ return 0;
+}
@@ reftable/writer.h (new)
+ int pending_padding;
+ struct slice last_key;
+
++ /* offset of next block to write. */
+ uint64_t next;
+ uint64_t min_update_index, max_update_index;
+ struct reftable_write_options opts;
+
++ /* memory buffer for writing */
+ byte *block;
-+ struct reftable_block_writer *block_writer;
-+ struct reftable_block_writer block_writer_data;
++
++ /* writer for the current section. NULL or points to
++ * block_writer_data */
++ struct block_writer *block_writer;
++
++ struct block_writer block_writer_data;
++
++ /* pending index records for the current section */
+ struct index_record *index;
+ int index_len;
+ int index_cap;
+
-+ /* tree for use with tsearch */
++ /*
++ tree for use with tsearch; used to populate the 'o' inverse OID
++ map */
+ struct tree_node *obj_index_tree;
+
+ struct reftable_stats stats;
+};
+
++/* finishes a block, and writes it to storage */
+int writer_flush_block(struct reftable_writer *w);
++
++/* deallocates memory related to the index */
+void writer_clear_index(struct reftable_writer *w);
++
++/* finishes writing a 'r' (refs) or 'g' (reflogs) section */
+int writer_finish_public_section(struct reftable_writer *w);
+
+#endif
9: b29c4ecc1c4 ! 10: ad72edbcfd4 Reftable support for git-core
@@ Commit message
TODO:
- * "git show-ref" shows "HEAD"
* Resolve spots marked with XXX
* Test strategy?
Example use: see t/t0031-reftable.sh
Signed-off-by: Han-Wen Nienhuys <[email protected]>
+ Signed-off-by: Johannes Schindelin <[email protected]>
Co-authored-by: Jeff King <[email protected]>
## Documentation/technical/repository-version.txt ##
@@ Makefile: $(XDIFF_LIB): $(XDIFF_OBJS)
export DEFAULT_EDITOR DEFAULT_PAGER
Documentation/GIT-EXCLUDED-PROGRAMS: FORCE
+@@ Makefile: cocciclean:
+ clean: profile-clean coverage-clean cocciclean
+ $(RM) *.res
+ $(RM) $(OBJECTS)
+- $(RM) $(LIB_FILE) $(XDIFF_LIB) $(VCSSVN_LIB)
++ $(RM) $(LIB_FILE) $(XDIFF_LIB) $(VCSSVN_LIB) $(REFTABLE_LIB)
+ $(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) git$X
+ $(RM) $(TEST_PROGRAMS)
+ $(RM) $(FUZZ_PROGRAMS)
## builtin/clone.c ##
@@ builtin/clone.c: int cmd_clone(int argc, const char **argv, const char *prefix)
@@ refs/reftable-backend.c (new)
+static void fill_reftable_log_record(struct reftable_log_record *log)
+{
+ const char *info = git_committer_info(0);
-+ struct ident_split split = {};
++ struct ident_split split = { NULL };
+ int result = split_ident_line(&split, info, strlen(info));
+ int sign = 1;
+ assert(0 == result);
@@ refs/reftable-backend.c (new)
+}
+
+static struct ref_store *git_reftable_ref_store_create(const char *path,
-+ unsigned int store_flags)
++ unsigned int store_flags)
+{
+ struct git_reftable_ref_store *refs = xcalloc(1, sizeof(*refs));
+ struct ref_store *ref_store = (struct ref_store *)refs;
@@ refs/reftable-backend.c (new)
+ struct ref_store *ref_store;
+ unsigned int flags;
+ int err;
-+ char *prefix;
++ const char *prefix;
+};
+
+static int reftable_ref_iterator_advance(struct ref_iterator *ref_iterator)
+{
-+ struct git_reftable_iterator *ri = (struct git_reftable_iterator *)ref_iterator;
++ struct git_reftable_iterator *ri =
++ (struct git_reftable_iterator *)ref_iterator;
+ while (ri->err == 0) {
+ ri->err = reftable_iterator_next_ref(ri->iter, &ri->ref);
+ if (ri->err) {
@@ refs/reftable-backend.c (new)
+static int reftable_ref_iterator_peel(struct ref_iterator *ref_iterator,
+ struct object_id *peeled)
+{
-+ struct git_reftable_iterator *ri = (struct git_reftable_iterator *)ref_iterator;
++ struct git_reftable_iterator *ri =
++ (struct git_reftable_iterator *)ref_iterator;
+ if (ri->ref.target_value != NULL) {
+ hashcpy(peeled->hash, ri->ref.target_value);
+ return 0;
@@ refs/reftable-backend.c (new)
+
+static int reftable_ref_iterator_abort(struct ref_iterator *ref_iterator)
+{
-+ struct git_reftable_iterator *ri = (struct git_reftable_iterator *)ref_iterator;
++ struct git_reftable_iterator *ri =
++ (struct git_reftable_iterator *)ref_iterator;
+ reftable_ref_record_clear(&ri->ref);
+ reftable_iterator_destroy(&ri->iter);
+ return 0;
@@ refs/reftable-backend.c (new)
+ }
+
+ base_ref_iterator_init(&ri->base, &reftable_ref_iterator_vtable, 1);
++ ri->prefix = prefix;
+ ri->base.oid = &ri->oid;
+ ri->flags = flags;
+ ri->ref_store = ref_store;
@@ refs/reftable-backend.c (new)
+static int reftable_check_old_oid(struct ref_store *refs, const char *refname,
+ struct object_id *want_oid)
+{
-+ struct object_id out_oid = {};
++ struct object_id out_oid;
+ int out_flags = 0;
+ const char *resolved = refs_resolve_ref_unsafe(
+ refs, refname, RESOLVE_REF_READING, &out_oid, &out_flags);
@@ refs/reftable-backend.c (new)
+ (struct git_reftable_ref_store *)transaction->ref_store;
+ uint64_t ts = reftable_stack_next_update_index(refs->stack);
+ int err = 0;
-+ struct reftable_log_record *logs = calloc(transaction->nr, sizeof(*logs));
++ struct reftable_log_record *logs =
++ calloc(transaction->nr, sizeof(*logs));
+ struct ref_update **sorted =
+ malloc(transaction->nr * sizeof(struct ref_update *));
+ COPY_ARRAY(sorted, transaction->updates, transaction->nr);
@@ refs/reftable-backend.c (new)
+ log->message = u->msg;
+
+ if (u->flags & REF_HAVE_NEW) {
-+ struct object_id out_oid = {};
++ struct object_id out_oid;
+ int out_flags = 0;
+ /* Memory owned by refs_resolve_ref_unsafe, no need to
+ * free(). */
+ const char *resolved = refs_resolve_ref_unsafe(
+ transaction->ref_store, u->refname, 0, &out_oid,
+ &out_flags);
-+ struct reftable_ref_record ref = {};
++ struct reftable_ref_record ref = { NULL };
+ ref.ref_name =
+ (char *)(resolved ? resolved : u->refname);
+ log->ref_name = ref.ref_name;
@@ refs/reftable-backend.c (new)
+ return refs->err;
+ }
+
-+ err = reftable_stack_add(refs->stack, &write_transaction_table, transaction);
++ err = reftable_stack_add(refs->stack, &write_transaction_table,
++ transaction);
+ if (err < 0) {
+ strbuf_addf(errmsg, "reftable: transaction failure %s",
+ reftable_error_str(err));
@@ refs/reftable-backend.c (new)
+ }
+
+ for (int i = 0; i < arg->refnames->nr; i++) {
-+ struct reftable_log_record log = {};
-+ struct reftable_ref_record current = {};
++ struct reftable_log_record log = { NULL };
++ struct reftable_ref_record current = { NULL };
+ fill_reftable_log_record(&log);
+ log.message = xstrdup(arg->logmsg);
+ log.new_hash = NULL;
@@ refs/reftable-backend.c (new)
+ log.update_index = ts;
+ log.ref_name = (char *)arg->refnames->items[i].string;
+
-+ if (reftable_stack_read_ref(arg->stack, log.ref_name, ¤t) == 0) {
++ if (reftable_stack_read_ref(arg->stack, log.ref_name,
++ ¤t) == 0) {
+ log.old_hash = current.value;
+ }
+ err = reftable_writer_add_log(writer, &log);
@@ refs/reftable-backend.c (new)
+ }
+
+ {
-+ struct reftable_log_record log = {};
-+ struct object_id new_oid = {};
-+ struct object_id old_oid = {};
-+ struct reftable_ref_record current = {};
-+ reftable_stack_read_ref(create->refs->stack, create->refname, ¤t);
++ struct reftable_log_record log = { NULL };
++ struct object_id new_oid;
++ struct object_id old_oid;
++ struct reftable_ref_record current = { NULL };
++ reftable_stack_read_ref(create->refs->stack, create->refname,
++ ¤t);
+
+ fill_reftable_log_record(&log);
+ log.ref_name = current.ref_name;
@@ refs/reftable-backend.c (new)
+ if (refs->err < 0) {
+ return refs->err;
+ }
-+ return reftable_stack_add(refs->stack, &write_create_symref_table, &arg);
++ return reftable_stack_add(refs->stack, &write_create_symref_table,
++ &arg);
+}
+
+struct write_rename_arg {
@@ refs/reftable-backend.c (new)
+{
+ struct write_rename_arg *arg = (struct write_rename_arg *)argv;
+ uint64_t ts = reftable_stack_next_update_index(arg->stack);
-+ struct reftable_ref_record ref = {};
++ struct reftable_ref_record ref = { NULL };
+ int err = reftable_stack_read_ref(arg->stack, arg->oldname, &ref);
+
+ if (err) {
@@ refs/reftable-backend.c (new)
+ ref.update_index = ts;
+
+ {
-+ struct reftable_ref_record todo[2] = {};
++ struct reftable_ref_record todo[2] = { { NULL } };
+ todo[0].ref_name = (char *)arg->oldname;
+ todo[0].update_index = ts;
+ /* leave todo[0] empty */
@@ refs/reftable-backend.c (new)
+ }
+
+ if (ref.value != NULL) {
-+ struct reftable_log_record todo[2] = {};
++ struct reftable_log_record todo[2] = { { NULL } };
+ fill_reftable_log_record(&todo[0]);
+ fill_reftable_log_record(&todo[1]);
+
@@ refs/reftable-backend.c (new)
+ return ITER_ERROR;
+ }
+
++ if (reftable_log_record_is_deletion(&ri->log)) {
++ /* XXX - Why does the reftable_stack filter these? */
++ continue;
++ }
+ ri->base.refname = ri->log.ref_name;
+ if (ri->last_name != NULL &&
+ !strcmp(ri->log.ref_name, ri->last_name)) {
++ /* we want the refnames that we have reflogs for, so we
++ * skip if we've already produced this name. This could
++ * be faster by seeking directly to
++ * reflog@update_index==0.
++ */
+ continue;
+ }
+
@@ refs/reftable-backend.c (new)
+ struct git_reftable_ref_store *refs =
+ (struct git_reftable_ref_store *)ref_store;
+
-+ struct reftable_merged_table *mt = reftable_stack_merged_table(refs->stack);
++ struct reftable_merged_table *mt =
++ reftable_stack_merged_table(refs->stack);
+ int err = reftable_merged_table_seek_log(mt, &ri->iter, "");
+ if (err < 0) {
+ free(ri);
@@ refs/reftable-backend.c (new)
+ const char *refname,
+ each_reflog_ent_fn fn, void *cb_data)
+{
-+ struct reftable_iterator it = {};
++ struct reftable_iterator it = { NULL };
+ struct git_reftable_ref_store *refs =
+ (struct git_reftable_ref_store *)ref_store;
+ struct reftable_merged_table *mt = NULL;
+ int err = 0;
-+ struct reftable_log_record log = {};
++ struct reftable_log_record log = { NULL };
+
+ if (refs->err < 0) {
+ return refs->err;
@@ refs/reftable-backend.c (new)
+ }
+
+ {
-+ struct object_id old_oid = {};
-+ struct object_id new_oid = {};
++ struct object_id old_oid;
++ struct object_id new_oid;
+ const char *full_committer = "";
+
+ hashcpy(old_oid.hash, log.old_hash);
@@ refs/reftable-backend.c (new)
+ const char *refname,
+ each_reflog_ent_fn fn, void *cb_data)
+{
-+ struct reftable_iterator it = {};
++ struct reftable_iterator it = { NULL };
+ struct git_reftable_ref_store *refs =
+ (struct git_reftable_ref_store *)ref_store;
+ struct reftable_merged_table *mt = NULL;
@@ refs/reftable-backend.c (new)
+ err = reftable_merged_table_seek_log(mt, &it, refname);
+
+ while (err == 0) {
-+ struct reftable_log_record log = {};
++ struct reftable_log_record log = { NULL };
+ err = reftable_iterator_next_log(it, &log);
+ if (err != 0) {
+ break;
@@ refs/reftable-backend.c (new)
+
+ for (int i = len; i--;) {
+ struct reftable_log_record *log = &logs[i];
-+ struct object_id old_oid = {};
-+ struct object_id new_oid = {};
++ struct object_id old_oid;
++ struct object_id new_oid;
+ const char *full_committer = "";
+
+ hashcpy(old_oid.hash, log->old_hash);
@@ refs/reftable-backend.c (new)
+ struct reflog_expiry_arg arg = {
+ .refs = refs,
+ };
-+ struct reftable_log_record log = {};
-+ struct reftable_iterator it = {};
++ struct reftable_log_record log = { NULL };
++ struct reftable_iterator it = { NULL };
+ int err = 0;
+ if (refs->err < 0) {
+ return refs->err;
@@ refs/reftable-backend.c (new)
+ }
+
+ while (1) {
-+ struct object_id ooid = {};
-+ struct object_id noid = {};
++ struct object_id ooid;
++ struct object_id noid;
+
+ int err = reftable_iterator_next_log(it, &log);
+ if (err < 0) {
@@ refs/reftable-backend.c (new)
+{
+ struct git_reftable_ref_store *refs =
+ (struct git_reftable_ref_store *)ref_store;
-+ struct reftable_ref_record ref = {};
++ struct reftable_ref_record ref = { NULL };
+ int err = 0;
+ if (refs->err < 0) {
+ return refs->err;
@@ t/t0031-reftable.sh (new)
+
+. ./test-lib.sh
+
++# XXX - fix GC
+test_expect_success 'basic operation of reftable storage' '
-+ git init --ref-storage=reftable repo && (
-+ cd repo &&
-+ echo "hello" >world.txt &&
-+ git add world.txt &&
-+ git commit -m "first post" &&
-+ test_write_lines HEAD refs/heads/master >expect &&
++ rm -rf .git &&
++ git init --ref-storage=reftable &&
++ mv .git/hooks .git/hooks-disabled &&
++ test_commit file &&
++ test_write_lines refs/heads/master refs/tags/file >expect &&
+ git show-ref &&
+ git show-ref | cut -f2 -d" " > actual &&
+ test_cmp actual expect &&
+ for count in $(test_seq 1 10)
+ do
-+ echo "hello" >>world.txt
-+ git commit -m "number ${count}" world.txt ||
-+ return 1
++ test_commit "number $count" file.t $count number-$count ||
++ return 1
+ done &&
++(true || (test_pause &&
+ git gc &&
-+ nfiles=$(ls -1 .git/reftable | wc -l ) &&
-+ test ${nfiles} = "2" &&
-+ git reflog refs/heads/master >output &&
++ ls -1 .git/reftable >table-files &&
++ test_line_count = 2 table-files &&
++ git reflog refs/heads/master >output &&
+ test_line_count = 11 output &&
+ grep "commit (initial): first post" output &&
-+ grep "commit: number 10" output )
++ grep "commit: number 10" output ))
+'
+
+test_doneSignature · 1 line
gitgitgadget
PATCH v9 1/10 refs.h: clarify reflog iteration order
From: Han-Wen Nienhuys <[email protected]>
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- refs.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
refs.h +4 -1
@@ -444,18 +444,21 @@ int delete_refs(const char *msg, struct string_list *refnames,int refs_delete_reflog (struct ref_store *refs,const char *refname);int delete_reflog (const char *refname); -/* iterate over reflog entries */ +/* Iterate over reflog entries. */ typedef int each_reflog_ent_fn (struct object_id *old_oid,struct object_id *new_oid,const char *committer,timestamp_t timestamp,int tz,const char *msg,void *cb_data); +/* Iterate in over reflog entries, oldest entry first. */ int refs_for_each_reflog_ent (struct ref_store *refs,const char *refname,each_reflog_ent_fn fn,void *cb_data);int refs_for_each_reflog_ent_reverse (struct ref_store *refs,const char *refname,each_reflog_ent_fn fn,void *cb_data); + +/* Call a function for each reflog entry, oldest entry first. */ int for_each_reflog_ent (const char *refname,each_reflog_ent_fn fn,void *cb_data);int for_each_reflog_ent_reverse (const char *refname,each_reflog_ent_fn fn,void *cb_data);
Signature · 1 line
gitgitgadget
PATCH v9 2/10 Iterate over the "refs/" namespace in for_each_[raw]ref
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This happens implicitly in the files/packed ref backend; making it explicit simplifies adding alternate ref storage backends, such as reftable.
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- refs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
refs.c +3 -3
@@ -1569,7 +1569,7 @@ static int do_for_each_ref(struct ref_store *refs, const char *prefix,int refs_for_each_ref (struct ref_store *refs,each_ref_fn fn,void *cb_data) { -return do_for_each_ref (refs,"" , fn,0 ,0 , cb_data); +return do_for_each_ref (refs,"refs/" , fn,0 ,0 , cb_data); }int for_each_ref (each_ref_fn fn,void *cb_data)
@@ -1629,8 +1629,8 @@ int for_each_namespaced_ref(each_ref_fn fn, void *cb_data)int refs_for_each_rawref (struct ref_store *refs,each_ref_fn fn,void *cb_data) { -return do_for_each_ref (refs,"" , fn,0 , -DO_FOR_EACH_INCLUDE_BROKEN , cb_data); +return do_for_each_ref (refs,"refs/" , fn,0 ,DO_FOR_EACH_INCLUDE_BROKEN , + cb_data); }int for_each_rawref (each_ref_fn fn,void *cb_data)
Signature · 1 line
gitgitgadget
PATCH v9 3/10 create .git/refs in files-backend.c
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This prepares for supporting the reftable format, which will want create its own file system layout in .git
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- builtin/init-db.c | 2 -- refs/files-backend.c | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-)
builtin/init-db.c +0 -2
@@ -251,8 +251,6 @@ static int create_default_files(const char *template_path,* We need to create a "refs" dir in any case so that older * versions of git can tell that this is a repository. */ -safe_create_dir (git_path ("refs" ),1 ); -adjust_shared_perm (git_path ("refs" ));if (refs_init_db (&err))die ("failed to set up refs db: %s" , err.buf);
refs/files-backend.c +6 -0
@@ -3157,9 +3157,15 @@ static int files_init_db(struct ref_store *ref_store, struct strbuf *err)files_downcast (ref_store,REF_STORE_WRITE ,"init_db" );struct strbuf sb =STRBUF_INIT ; +files_ref_path (refs, &sb,"refs" ); +safe_create_dir (sb.buf,1 ); +/* adjust permissions even if directory already exists. */ +adjust_shared_perm (sb.buf); +/* * Create .git/refs/{heads,tags} */ +strbuf_reset (&sb);files_ref_path (refs, &sb,"refs/heads" );safe_create_dir (sb.buf,1 );
Signature · 1 line
gitgitgadget
PATCH v9 4/10 refs: document how ref_iterator_advance_fn should handle symrefs
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- refs/refs-internal.h | 5 +++++ 1 file changed, 5 insertions(+)
refs/refs-internal.h +5 -0
@@ -438,6 +438,11 @@ void base_ref_iterator_free(struct ref_iterator *iter);/* Virtual function declarations for ref_iterators: */ +/* +* backend-specific implementation of ref_iterator_advance. +* For symrefs, the function should set REF_ISSYMREF, and it should also +* dereference the symref to provide the OID referent. +*/ typedef int ref_iterator_advance_fn (struct ref_iterator *ref_iterator); typedef int ref_iterator_peel_fn(struct ref_iterator *ref_iterator,
Signature · 1 line
gitgitgadget
PATCH v9 5/10 Add .gitattributes for the reftable/ directory
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- reftable/.gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 reftable/.gitattributes
reftable/.gitattributes new file +1 -0
@@ -0,0 +1 @@ +/zlib-compat.c whitespace=-indent-with-non-tab,-trailing-space
Signature · 1 line
gitgitgadget
PATCH v9 6/10 reftable: file format documentation
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Jonathan Nieder <[email protected]>
Shawn Pearce explains:
Some repositories contain a lot of references (e.g. android at 866k, rails at 31k). The reftable format provides:
- Near constant time lookup for any single reference, even when the repository is cold and not in process or kernel cache.
- Near constant time verification a SHA-1 is referred to by at least one reference (for allow-tip-sha1-in-want).
- Efficient lookup of an entire namespace, such as refs/tags/.
- Support atomic push O(size_of_update) operations.
- Combine reflog storage with ref storage.
This file format spec was originally written in July, 2017 by Shawn Pearce. Some refinements since then were made by Shawn and by Han-Wen Nienhuys based on experiences implementing and experimenting with the format. (All of this was in the context of our work at Google and Google is happy to contribute the result to the Git project.)
Imported from JGit[1]'s current version (c217d33ff, "Documentation/technical/reftable: improve repo layout", 2020-02-04) of Documentation/technical/reftable.md and converted to asciidoc by running
pandoc -t asciidoc -f markdown reftable.md >reftable.txt
using pandoc 2.2.1. The result required the following additional minor changes:
- removed the [TOC] directive to add a table of contents, since asciidoc does not support it
- replaced git-scm.com/docs links with linkgit: directives that link to other pages within Git's documentation
[1] https://eclipse.googlesource.com/jgit/jgit
Signed-off-by Jonathan Nieder <[email protected]>
--- Documentation/Makefile | 1 + Documentation/technical/reftable.txt | 1067 ++++++++++++++++++++++++++ 2 files changed, 1068 insertions(+) create mode 100644 Documentation/technical/reftable.txt
Documentation/Makefile +1 -0
@@ -92,6 +92,7 @@ TECH_DOCS += technical/protocol-capabilities TECH_DOCS += technical/protocol-common TECH_DOCS += technical/protocol-v2 TECH_DOCS += technical/racy-git +TECH_DOCS += technical/reftable TECH_DOCS += technical/send-pack-pipeline TECH_DOCS += technical/shallow TECH_DOCS += technical/signature-format
Documentation/technical/reftable.txt new file +1067 -0
1,067 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v9 7/10 reftable: define version 2 of the spec to accomodate SHA256
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- Documentation/technical/reftable.txt | 50 ++++++++++++++++------------ 1 file changed, 28 insertions(+), 22 deletions(-)
Documentation/technical/reftable.txt +28 -22
@@ -193,8 +193,8 @@ and non-aligned files. Very small files (e.g. 1 only ref block) may omit `padding` and the ref index to reduce total file size. -Header -^^^^^^ +Header (version 1) +^^^^^^^^^^^^^^^^^^ A 24-byte header appears at the beginning of the file:
@@ -215,6 +215,24 @@ used in a stack for link:#Update-transactions[transactions], these fields can order the files such that the prior file’s `max_update_index + 1` is the next file’s `min_update_index`. +Header (version 2) +^^^^^^^^^^^^^^^^^^ + +A 28-byte header appears at the beginning of the file: + +.... +'REFT' +uint8( version_number = 1 ) +uint24( block_size ) +uint64( min_update_index ) +uint64( max_update_index ) +uint32( hash_id ) +.... + +The header is identical to `version_number=1`, with the 4-byte hash ID +("sha1" for SHA1 and "s256" for SHA-256) append to the header. + + First ref block ^^^^^^^^^^^^^^^
@@ -671,14 +689,8 @@ Footer After the last block of the file, a file footer is written. It begins like the file header, but is extended with additional data. -A 68-byte footer appears at the end: - .... - 'REFT' - uint8( version_number = 1 ) - uint24( block_size ) - uint64( min_update_index ) - uint64( max_update_index ) + HEADER uint64( ref_index_position ) uint64( (obj_position << 5) | obj_id_len )
@@ -701,12 +713,16 @@ obj blocks. * `obj_index_position`: byte position for the start of the obj index. * `log_index_position`: byte position for the start of the log index. +The size of the footer is 68 bytes for version 1, and 72 bytes for +version 2. + Reading the footer ++++++++++++++++++ -Readers must seek to `file_length - 68` to access the footer. A trusted -external source (such as `stat(2)`) is necessary to obtain -`file_length`. When reading the footer, readers must verify: +Readers must first read the file start to determine the version +number. Then they seek to `file_length - FOOTER_LENGTH` to access the +footer. A trusted external source (such as `stat(2)`) is necessary to +obtain `file_length`. When reading the footer, readers must verify: * 4-byte magic is correct * 1-byte version number is recognized
@@ -1055,13 +1071,3 @@ impossible. A common format that can be supported by all major Git implementations (git-core, JGit, libgit2) is strongly preferred. - -Future -~~~~~~ - -Longer hashes -^^^^^^^^^^^^^ - -Version will bump (e.g. 2) to indicate `value` uses a different object -id length other than 20. The length could be stored in an expanded file -header, or hardcoded as part of the version.
Signature · 1 line
gitgitgadget
PATCH v9 8/10 reftable: clarify how empty tables should be written
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The format allows for some ambiguity, as a lone footer also starts with a valid file header. However, the current JGit code will barf on this. This commit codifies this behavior into the standard.
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- Documentation/technical/reftable.txt | 7 +++++++ 1 file changed, 7 insertions(+)
Documentation/technical/reftable.txt +7 -0
@@ -731,6 +731,13 @@ version) Once verified, the other fields of the footer can be accessed. +Empty tables +++++++++++++ + +A reftable may be empty. In this case, the file starts with a header +and is immediately followed by a footer. + + Varint encoding ^^^^^^^^^^^^^^^
Signature · 1 line
gitgitgadget
PATCH v9 9/10 Add reftable library
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Reftable is a format for storing the ref database. Its rationale and specification is in the preceding commit.
This imports the upstream library as one big commit. For understanding the code, it is suggested to read the code in the following order:
* The specification under Documentation/technical/reftable.txt
* reftable.h - the public API
* block.{c,h} - reading and writing blocks.
* writer.{c,h} - writing a complete reftable file.
* merged.{c,h} and pq.{c,h} - reading a stack of reftables
* stack.{c,h} - writing and compacting stacks of reftable on the filesystem.
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- reftable/LICENSE | 31 ++ reftable/README.md | 11 + reftable/VERSION | 5 + reftable/basics.c | 189 +++++++ reftable/basics.h | 53 ++ reftable/block.c | 425 +++++++++++++++ reftable/block.h | 124 +++++ reftable/blocksource.h | 22 + reftable/bytes.c | 0 reftable/config.h | 1 + reftable/constants.h | 21 + reftable/dump.c | 97 ++++ reftable/file.c | 98 ++++ reftable/iter.c | 234 +++++++++ reftable/iter.h | 60 +++ reftable/merged.c | 307 +++++++++++ reftable/merged.h | 34 ++ reftable/pq.c | 115 ++++ reftable/pq.h | 34 ++ reftable/reader.c | 754 ++++++++++++++++++++++++++ reftable/reader.h | 68 +++ reftable/record.c | 1119 +++++++++++++++++++++++++++++++++++++++ reftable/record.h | 117 +++++ reftable/reftable.h | 523 +++++++++++++++++++ reftable/slice.c | 224 ++++++++ reftable/slice.h | 76 +++ reftable/stack.c | 1132 ++++++++++++++++++++++++++++++++++++++++ reftable/stack.h | 42 ++ reftable/system.h | 53 ++ reftable/tree.c | 67 +++ reftable/tree.h | 34 ++ reftable/update.sh | 14 + reftable/writer.c | 661 +++++++++++++++++++++++ reftable/writer.h | 60 +++ reftable/zlib-compat.c | 92 ++++ 35 files changed, 6897 insertions(+) create mode 100644 reftable/LICENSE create mode 100644 reftable/README.md create mode 100644 reftable/VERSION create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/blocksource.h create mode 100644 reftable/bytes.c create mode 100644 reftable/config.h create mode 100644 reftable/constants.h create mode 100644 reftable/dump.c create mode 100644 reftable/file.c create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/reftable.h create mode 100644 reftable/slice.c create mode 100644 reftable/slice.h create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/system.h create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100644 reftable/update.sh create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h create mode 100644 reftable/zlib-compat.c
reftable/LICENSE new file +31 -0
@@ -0,0 +1,31 @@ +BSD License + +Copyright (c) 2020, Google LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of Google LLC nor the names of its contributors may +be used to endorse or promote products derived from this software +without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
reftable/README.md new file +11 -0
@@ -0,0 +1,11 @@ + +The source code in this directory comes from https://github.com/google/reftable. + +The VERSION file keeps track of the current version of the reftable library. + +To update the library, do: + + sh reftable/update.sh + +Bugfixes should be accompanied by a test and applied to upstream project at +https://github.com/google/reftable.
reftable/VERSION new file +5 -0
@@ -0,0 +1,5 @@ +commit b68690acad769d59e80cbb4f79c442ece133e6bd +Author: Han-Wen Nienhuys <[email protected]> +Date: Mon Apr 20 21:33:11 2020 +0200 + + C: fix search/replace error in dump.c
reftable/basics.c new file +189 -0
189 lines not on this page · raw message
reftable/basics.h new file +53 -0
53 lines not on this page · raw message
reftable/block.c new file +425 -0
425 lines not on this page · raw message
reftable/block.h new file +124 -0
124 lines not on this page · raw message
reftable/blocksource.h new file +22 -0
@@ -0,0 +1,22 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef BLOCKSOURCE_H +#define BLOCKSOURCE_H + +#include "reftable.h" + +uint64_t block_source_size (struct reftable_block_source source); +int block_source_read_block (struct reftable_block_source source, +struct reftable_block *dest,uint64_t off, +uint32_t size); +void block_source_return_block (struct reftable_block_source source, +struct reftable_block *ret); +void block_source_close (struct reftable_block_source source); + +#endif
reftable/bytes.c new file +0 -0
reftable/config.h new file +1 -0
@@ -0,0 +1 @@ +/* empty */
reftable/constants.h new file +21 -0
@@ -0,0 +1,21 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef CONSTANTS_H +#define CONSTANTS_H + +#define BLOCK_TYPE_LOG 'g' +#define BLOCK_TYPE_INDEX 'i' +#define BLOCK_TYPE_REF 'r' +#define BLOCK_TYPE_OBJ 'o' +#define BLOCK_TYPE_ANY 0 + +#define MAX_RESTARTS ((1 << 16) - 1) +#define DEFAULT_BLOCK_SIZE 4096 + +#endif
reftable/dump.c new file +97 -0
97 lines not on this page · raw message
reftable/file.c new file +98 -0
98 lines not on this page · raw message
reftable/iter.c new file +234 -0
234 lines not on this page · raw message
reftable/iter.h new file +60 -0
60 lines not on this page · raw message
reftable/merged.c new file +307 -0
307 lines not on this page · raw message
reftable/merged.h new file +34 -0
@@ -0,0 +1,34 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef MERGED_H +#define MERGED_H + +#include "pq.h" +#include "reftable.h" + +struct reftable_merged_table { +struct reftable_reader **stack; +int stack_len; +uint32_t hash_id; + +uint64_t min; +uint64_t max; +}; + +struct merged_iter { +struct reftable_iterator *stack; +uint32_t hash_id; +int stack_len; +byte typ; +struct merged_iter_pqueue pq; +}; + +void merged_table_clear (struct reftable_merged_table *mt); + +#endif
reftable/pq.c new file +115 -0
115 lines not on this page · raw message
reftable/pq.h new file +34 -0
@@ -0,0 +1,34 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef PQ_H +#define PQ_H + +#include "record.h" + +struct pq_entry { +int index; +struct record rec; +}; + +int pq_less (struct pq_entry a,struct pq_entry b); + +struct merged_iter_pqueue { +struct pq_entry *heap; +int len; +int cap; +}; + +struct pq_entry merged_iter_pqueue_top (struct merged_iter_pqueue pq); +bool merged_iter_pqueue_is_empty (struct merged_iter_pqueue pq); +void merged_iter_pqueue_check (struct merged_iter_pqueue pq); +struct pq_entry merged_iter_pqueue_remove (struct merged_iter_pqueue *pq); +void merged_iter_pqueue_add (struct merged_iter_pqueue *pq,struct pq_entry e); +void merged_iter_pqueue_clear (struct merged_iter_pqueue *pq); + +#endif
reftable/reader.c new file +754 -0
754 lines not on this page · raw message
reftable/reader.h new file +68 -0
68 lines not on this page · raw message
reftable/record.c new file +1119 -0
317 lines not on this page · raw message
reftable/record.h new file +117 -0
117 lines not on this page · raw message
reftable/reftable.h new file +523 -0
523 lines not on this page · raw message
reftable/slice.c new file +224 -0
224 lines not on this page · raw message
reftable/slice.h new file +76 -0
76 lines not on this page · raw message
reftable/stack.c new file +1132 -0
1,132 lines not on this page · raw message
reftable/stack.h new file +42 -0
42 lines not on this page · raw message
reftable/system.h new file +53 -0
53 lines not on this page · raw message
reftable/tree.c new file +67 -0
67 lines not on this page · raw message
reftable/writer.c new file +661 -0
661 lines not on this page · raw message
reftable/writer.h new file +60 -0
60 lines not on this page · raw message
reftable/zlib-compat.c new file +92 -0
92 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
gitgitgadget
Part of the diff is not shown here. View the raw message.
Re: [PATCH v9 09/10] Add reftable library
reftable/update.sh
Quoted 2 lines · Han-Wen Nienhuys via GitGitGadget
reftable/update.sh new file +0
-0
Shouldn't this be executable, even though the readme tells the reader to run "sh update.sh"?
reftable/update.sh line 3
Quoted 7 lines
index 00000000000..0fd90f89675
reftable/update.sh new file +3
-0@@ -0,0 +1,14 @@ +#!/bin/sh + +set -eux
Use of -e and -u are understandable but -x is a bit unusual unless one is debugging...
reftable/update.sh line 6
Quoted 2 lines
reftable/update.sh new file +2
-0+((cd reftable-repo && git fetch origin && git checkout origin/master ) || +git clone https://github.com/google/reftable reftable-repo) && \
I think you can discard most of the '\' at the end of the line, as the shell knows, when you stop a line with &&, you haven't finished talking to it yet. That would make the result slightly less noisy to follow. We don't rely on "set -e" in this project, but this is a borrowed file from our point of view---as long as we are using -e, wouldn't it make more sense to take full advantage of it and do without &&?
reftable/update.sh line 12
Quoted 7 lines
reftable/update.sh new file +6
-0+cp reftable-repo/c/*.[ch] reftable/ && \ +cp reftable-repo/c/include/*.[ch] reftable/ && \ +cp reftable-repo/LICENSE reftable/ && +git --git-dir reftable-repo/.git show --no-patch origin/master \ +> reftable/VERSION && \ +sed -i~ 's|if REFTABLE_IN_GITCORE|if 1 /* REFTABLE_IN_GITCORE */|' reftable/system.h
"sed -i" is probably GNUism that does not port well.
Quoted 1 line
+rm
reftable/*_test.creftable/test_framework.* reftable/compat.*
I presume that this is because we copied everything that match c/*.[ch] and is not about removing what used to exist in reftable/ directory locally? Not complaining, but checking if I am reading the intention of the code correctly.
Quoted 1 line
+git add reftable/*.[ch]
As we flattened when copying from c/include/*.[ch], this "git add" can also be flat, which makes sense. Don't you need to also add the LICENSE file in there?
Two comments on the design:
- The list of "what to copy in" largely depends on the upstream reftable library; hardcoding the patterns here would make it harder to reorganize the files for the upstream project. I wonder if it makes more sense to reserve a single path in the reftable-repo and hardcode the knowledge of what to copy out there? That would mean the resulting update.sh here can become
(git -C reftable-repo pull --ff-only && git clone ...) && reftable-repo/copy-out-to-git-core.sh reftable/
and the copy-out-to-git-core.sh script that comes with the upstream reftable library is the only one that needs to know where the license and C source files are in the same version as itself, and how to produce $1/VERSION file using the commit log message of the commit it comes from.
- The above procedure (with or without the "who should know what to copy" change) would cope well with modified and new files in the reftable upstream, but does not remove a library file that used to exist (and copied to the git-core project) that no longer is needed and shipped. It appears that there has to be some way to remove them.
Thanks.
Re: [PATCH v9 09/10] Add reftable library
Quoted 2 lines · Junio C Hamano
Shouldn't this be executable, even though the readme tells the reader to run "sh
update.sh"?
Done.
(should I post an updated version of the patch series for these minor changes, or is it OK to fold them with the next major update?)
Quoted 4 lines
Quoted 1 line
+set -eux
Use of -e and -u are understandable but -x is a bit unusual unless one is debugging...
Done.
Quoted 10 lines
reftable/update.sh line 6
Quoted 2 lines
reftable/update.sh new file +2
-0+((cd reftable-repo && git fetch origin && git checkout origin/master ) || +git clone https://github.com/google/reftable reftable-repo) && \I think you can discard most of the '\' at the end of the line, as the shell knows, when you stop a line with &&, you haven't finished talking to it yet. That would make the result slightly less noisy to follow. We don't rely on "set -e" in this project, but this is a borrowed file from our point of view---as long as we are using -e, wouldn't it make more sense to take full advantage of it and do without &&?
Done.
Quoted 9 lines
reftable/update.sh line 12
Quoted 7 lines
reftable/update.sh new file +6
-0+cp reftable-repo/c/*.[ch] reftable/ && \ +cp reftable-repo/c/include/*.[ch] reftable/ && \ +cp reftable-repo/LICENSE reftable/ && +git --git-dir reftable-repo/.git show --no-patch origin/master \ +> reftable/VERSION && \ +sed -i~ 's|if REFTABLE_IN_GITCORE|if 1 /* REFTABLE_IN_GITCORE */|' reftable/system.h"sed -i" is probably GNUism that does not port well.
done.
Quoted 6 lines
Quoted 1 line
+rm
reftable/*_test.creftable/test_framework.* reftable/compat.*I presume that this is because we copied everything that match c/*.[ch] and is not about removing what used to exist in reftable/ directory locally? Not complaining, but checking if I am reading the intention of the code correctly.
correct.
Quoted 5 lines
Quoted 1 line
+git add reftable/*.[ch]
As we flattened when copying from c/include/*.[ch], this "git add" can also be flat, which makes sense. Don't you need to also add the LICENSE file in there?
Done.
Quoted 18 lines
Two comments on the design:
- The list of "what to copy in" largely depends on the upstream reftable library; hardcoding the patterns here would make it harder to reorganize the files for the upstream project. I wonder if it makes more sense to reserve a single path in the reftable-repo and hardcode the knowledge of what to copy out there? That would mean the resulting
update.shhere can become(git -C reftable-repo pull --ff-only && git clone ...) && reftable-repo/copy-out-to-git-core.sh reftable/and the
copy-out-to-git-core.shscript that comes with the upstream reftable library is the only one that needs to know where the license and C source files are in the same version as itself, and how to produce $1/VERSION file using the commit log message of the commit it comes from.
Quoted 6 lines
- The above procedure (with or without the "who should know what to copy" change) would cope well with modified and new files in the reftable upstream, but does not remove a library file that used to exist (and copied to the git-core project) that no longer is needed and shipped. It appears that there has to be some way to remove them.
You are correct, but once this is working to the extent that it's accepted into git-core, I expect imports to be infrequent, and I think it's not worth the effort to automate this further. If files are renamed, the result will likely not compile, so this mistake will be easy to catch.
Signature · 11 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Re: [PATCH v9 09/10] Add reftable library
In reply to Junio C HamanoHi,
Quoted 7 lines · Junio C Hamano
reftable/update.sh
Quoted 2 lines · Han-Wen Nienhuys via GitGitGadget
reftable/update.sh new file +0
-0Shouldn't this be executable, even though the readme tells the reader to run "sh
update.sh"?
Do we really want to have such a shell script? To be honest, I'd rather do a subtree merge. In my mind, this should make it easier to maintain this code elsewhere, yet debug and fix it in git.git (because there is no doubt whatsoever in my mind that the vast majority of bug fixes in this code will come in via git.git, which _does_ make me wonder whether it makes sense to keep the reftable library separate, as if it was truly an independent part of Git: it is absolutely not).
Of course, this would make it even more desirable to do away with code that reads like Java (and is probably a silent, defiant protest against Git's requirement to have no declarations after statements), e.g.
int block_iter_next(struct block_iter *it, struct record rec)
{
if (it->next_off >= it->br->block_len) {
return 1;
}
{
struct slice in = {
.buf = it->br->block.data + it->next_off,
.len = it->br->block_len - it->next_off,
};
struct slice start = in;
struct slice key = { 0 };
byte extra;
int n = decode_key(&key, &extra, it->last_key, in);
if (n < 0) {
return -1;
}
[...]
}
}The fact that this code uses a coding style that is very different from Git's, on purpose, makes it quite awkward to even so much as debug, say, the segmentation fault that was caused by my rewrite of the test script (which _also_ looks _very_ different from the existing test cases, adding to the awkwardness):
test_expect_success 'basic operation of reftable storage' ' rm -rf .git && git init --ref-storage=reftable && mv .git/hooks .git/hooks-disabled && test_commit file && test_write_lines HEAD refs/heads/master refs/tags/file >expect && git show-ref && git show-ref | cut -f2 -d" " > actual && test_cmp actual expect && for count in $(test_seq 1 10) do test_commit "number $count" file.t $count number-$count || return 1 done && git gc && ls -1 .git/reftable >table-files && test_line_count = 2 table-files && git reflog refs/heads/master >output && test_line_count = 11 output && grep "commit (initial): first post" output && grep "commit: number 10" output '
The fact that this rather trivial usage can still cause a segmentation fault makes me believe that this rationale for the purposefully-different coding style implies a fundamental fallacy (see
https://github.com/git-for-windows/git/commit/00eb1fd4496ef763f840878fb5249507ae83af43#commitcomment-38614650):
Each time you use an unitialized variable, you put another mental burden on the reader to check that it is actually OK in that place, and the Git source code (which puts the declaration and use) far apart makes this harder to verify.
The major challenge with writing C code these days, especially when you come from a background of Java or other languages that take care of memory management "for you" is that you _do_ have to manage memory in C, explicitly. And if you already start not paying attention to the initialization of local variables, you go down the path where all of a sudden you end up with a segmentation fault and have no idea where it is coming from, and the most likely explanation is that the memory was managed incorrectly.
So I would recommend to actually not get sloppy with confusing declaration and initialization and initializing unnecessarily and trying to get cute with introducing code blocks "just to introduce a narrower scope for local variables".
Instead, I would suggest to stick with the style Git developed; We have gotten relatively good with keeping our memory management straight that way.
Ciao,
Dscho
PATCH v9 10/10 Reftable support for git-core
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
For background, see the previous commit introducing the library.
TODO:
* Resolve spots marked with XXX
* Test strategy?
Example use: see t/t0031-reftable.sh
Signed-off-by Han-Wen Nienhuys <[email protected]>
Signed-off-by Johannes Schindelin <[email protected]>
Co-authored-by Jeff King <[email protected]>
--- .../technical/repository-version.txt | 7 + Makefile | 26 +- builtin/clone.c | 5 +- builtin/init-db.c | 49 +- cache.h | 4 +- refs.c | 20 +- refs.h | 2 + refs/refs-internal.h | 1 + refs/reftable-backend.c | 1021 +++++++++++++++++ repository.c | 2 + repository.h | 3 + setup.c | 12 +- t/t0031-reftable.sh | 35 + 13 files changed, 1158 insertions(+), 29 deletions(-) create mode 100644 refs/reftable-backend.c create mode 100755 t/t0031-reftable.sh
Documentation/technical/repository-version.txt +7 -0
@@ -100,3 +100,10 @@ If set, by default "git config" reads from both "config" and multiple working directory mode, "config" file is shared while "config.worktree" is per-working directory (i.e., it's in GIT_COMMON_DIR/worktrees/<id>/config.worktree) + +==== `refStorage` + +Specifies the file format for the ref database. Values are `files` +(for the traditional packed + loose ref format) and `reftable` for the +binary reftable format. See https://github.com/google/reftable for +more information.
Makefile +24 -2
@@ -814,6 +814,7 @@ TEST_SHELL_PATH = $(SHELL_PATH) LIB_FILE = libgit.a XDIFF_LIB = xdiff/lib.a VCSSVN_LIB = vcs-svn/lib.a +REFTABLE_LIB = reftable/libreftable.a GENERATED_H += command-list.h
@@ -960,6 +961,7 @@ LIB_OBJS += rebase-interactive.o LIB_OBJS += reflog-walk.o LIB_OBJS += refs.o LIB_OBJS += refs/files-backend.o +LIB_OBJS += refs/reftable-backend.o LIB_OBJS += refs/iterator.o LIB_OBJS += refs/packed-backend.o LIB_OBJS += refs/ref-cache.o
@@ -1164,7 +1166,7 @@ THIRD_PARTY_SOURCES += compat/regex/% THIRD_PARTY_SOURCES += sha1collisiondetection/% THIRD_PARTY_SOURCES += sha1dc/% -GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) +GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) EXTLIBS = GIT_USER_AGENT = git/$(GIT_VERSION)
@@ -2348,11 +2350,28 @@ VCSSVN_OBJS += vcs-svn/fast_export.o VCSSVN_OBJS += vcs-svn/svndiff.o VCSSVN_OBJS += vcs-svn/svndump.o +REFTABLE_OBJS += reftable/basics.o +REFTABLE_OBJS += reftable/block.o +REFTABLE_OBJS += reftable/bytes.o +REFTABLE_OBJS += reftable/file.o +REFTABLE_OBJS += reftable/iter.o +REFTABLE_OBJS += reftable/merged.o +REFTABLE_OBJS += reftable/pq.o +REFTABLE_OBJS += reftable/reader.o +REFTABLE_OBJS += reftable/record.o +REFTABLE_OBJS += reftable/slice.o +REFTABLE_OBJS += reftable/stack.o +REFTABLE_OBJS += reftable/tree.o +REFTABLE_OBJS += reftable/writer.o +REFTABLE_OBJS += reftable/zlib-compat.o + + TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) OBJECTS := $(LIB_OBJS) $(BUILTIN_OBJS) $(PROGRAM_OBJS) $(TEST_OBJS) \ $(XDIFF_OBJS) \ $(VCSSVN_OBJS) \ $(FUZZ_OBJS) \ + $(REFTABLE_OBJS) \ common-main.o \ git.o ifndef NO_CURL
@@ -2489,6 +2508,9 @@ $(XDIFF_LIB): $(XDIFF_OBJS) $(VCSSVN_LIB): $(VCSSVN_OBJS) $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ +$(REFTABLE_LIB): $(REFTABLE_OBJS) + $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ + export DEFAULT_EDITOR DEFAULT_PAGER Documentation/GIT-EXCLUDED-PROGRAMS: FORCE
@@ -3104,7 +3126,7 @@ cocciclean: clean: profile-clean coverage-clean cocciclean $(RM) *.res $(RM) $(OBJECTS) - $(RM) $(LIB_FILE) $(XDIFF_LIB) $(VCSSVN_LIB) + $(RM) $(LIB_FILE) $(XDIFF_LIB) $(VCSSVN_LIB) $(REFTABLE_LIB) $(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) git$X $(RM) $(TEST_PROGRAMS) $(RM) $(FUZZ_PROGRAMS)
builtin/clone.c +4 -1
@@ -1109,7 +1109,10 @@ int cmd_clone(int argc, const char **argv, const char *prefix) } } -init_db (git_dir, real_git_dir, option_template,GIT_HASH_UNKNOWN ,INIT_DB_QUIET ); +init_db (git_dir, real_git_dir, option_template, +GIT_HASH_UNKNOWN , +DEFAULT_REF_STORAGE ,/* XXX */ +INIT_DB_QUIET );if (real_git_dir) git_dir = real_git_dir;
builtin/init-db.c +35 -14
125 lines not on this page · raw message
cache.h +3 -1
@@ -628,8 +628,9 @@ int path_inside_repo(const char *prefix, const char *path);int init_db (const char *git_dir,const char *real_git_dir,const char *template_dir,int hash_algo, +const char *ref_storage_format,unsigned int flags); -void initialize_repository_version (int hash_algo); +void initialize_repository_version (int hash_algo,const char *ref_storage_format);void sanitize_stdfds (void );int daemonize (void );
@@ -1043,6 +1044,7 @@ struct repository_format {int is_bare;int hash_algo;char *work_tree; +char *ref_storage;struct string_list unknown_extensions; };
refs.c +12 -8
@@ -20,7 +20,7 @@/* * List of all available backends */ -static struct ref_storage_be *refs_backends = &refs_be_files; +static struct ref_storage_be *refs_backends = &refs_be_reftable;static struct ref_storage_be *find_ref_storage_backend (const char *name) {
@@ -1836,13 +1836,13 @@ static struct ref_store *lookup_ref_store_map(struct hashmap *map,* Create, record, and return a ref_store instance for the specified * gitdir. */ -static struct ref_store *ref_store_init (const char *gitdir, +static struct ref_store *ref_store_init (const char *gitdir,const char *be_name,unsigned int flags) { -const char *be_name ="files" ; -struct ref_storage_be *be =find_ref_storage_backend (be_name); +struct ref_storage_be *be;struct ref_store *refs; + be =find_ref_storage_backend (be_name);if (!be)BUG ("reference backend %s is unknown" , be_name);
@@ -1858,7 +1858,10 @@ struct ref_store *get_main_ref_store(struct repository *r)if (!r->gitdir)BUG ("attempting to get main_ref_store outside of repository" ); - r->refs =ref_store_init (r->gitdir,REF_STORE_ALL_CAPS ); + r->refs =ref_store_init (r->gitdir, + r->ref_storage_format ? r->ref_storage_format : +DEFAULT_REF_STORAGE , +REF_STORE_ALL_CAPS );return r->refs; }
@@ -1913,7 +1916,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule) goto done;/* assume that add_submodule_odb() has been called */ - refs =ref_store_init (submodule_sb.buf, + refs =ref_store_init (submodule_sb.buf,DEFAULT_REF_STORAGE ,/* XXX */ REF_STORE_READ |REF_STORE_ODB );register_ref_store_map (&submodule_ref_stores,"submodule" , refs, submodule);
@@ -1927,6 +1930,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule)struct ref_store *get_worktree_ref_store (const struct worktree *wt) { +const char *format =DEFAULT_REF_STORAGE ;/* XXX */ struct ref_store *refs;const char *id;
@@ -1940,9 +1944,9 @@ struct ref_store *get_worktree_ref_store(const struct worktree *wt)if (wt->id) refs =ref_store_init (git_common_path ("worktrees/%s" , wt->id), -REF_STORE_ALL_CAPS ); + format,REF_STORE_ALL_CAPS );else - refs =ref_store_init (get_git_common_dir (), + refs =ref_store_init (get_git_common_dir (), format,REF_STORE_ALL_CAPS );if (refs)
refs.h +2 -0
@@ -9,6 +9,8 @@ struct string_list;struct string_list_item ;struct worktree ; +#define DEFAULT_REF_STORAGE "files" +/* * Resolve a reference, recursively following symbolic refererences. *
refs/refs-internal.h +1 -0
@@ -661,6 +661,7 @@ struct ref_storage_be { };extern struct ref_storage_be refs_be_files; +extern struct ref_storage_be refs_be_reftable;extern struct ref_storage_be refs_be_packed;/*
refs/reftable-backend.c new file +1021 -0
1,021 lines not on this page · raw message
repository.c +2 -0
@@ -178,6 +178,8 @@ int repo_init(struct repository *repo,if (worktree)repo_set_worktree (repo, worktree); + repo->ref_storage_format =xstrdup_or_null (format.ref_storage); +clear_repository_format (&format);return 0 ;
repository.h +3 -0
@@ -70,6 +70,9 @@ struct repository {/* The store in which the refs are held. */ struct ref_store *refs; +/* The format to use for the ref database. */ +char *ref_storage_format; +/* * Contains path to often used file names. */
setup.c +9 -3
@@ -468,9 +468,11 @@ static int check_repo_format(const char *var, const char *value, void *vdata)if (!value)return config_error_nonbool (var); data->partial_clone =xstrdup (value); - }else if (!strcmp (ext,"worktreeconfig" )) + }else if (!strcmp (ext,"worktreeconfig" )) { data->worktree_config =git_config_bool (var, value); -else + }else if (!strcmp (ext,"refstorage" )) { + data->ref_storage =xstrdup (value); + }else string_list_append (&data->unknown_extensions, ext); }
@@ -559,6 +561,7 @@ void clear_repository_format(struct repository_format *format)string_list_clear (&format->unknown_extensions,0 );free (format->work_tree);free (format->partial_clone); +free (format->ref_storage);init_repository_format (format); }
@@ -1204,8 +1207,11 @@ const char *setup_git_directory_gently(int *nongit_ok) gitdir =DEFAULT_GIT_DIR_ENVIRONMENT ;setup_git_env (gitdir); } -if (startup_info->have_repository) +if (startup_info->have_repository) {repo_set_hash_algo (the_repository, repo_fmt.hash_algo); + the_repository->ref_storage_format = +xstrdup_or_null (repo_fmt.ref_storage); + } }strbuf_release (&dir);
t/t0031-reftable.sh new file +35 -0
@@ -0,0 +1,35 @@ +#!/bin/sh +# +# Copyright (c) 2020 Google LLC +# + +test_description='reftable basics' + +. ./test-lib.sh + +# XXX - fix GC +test_expect_success 'basic operation of reftable storage' ' + rm -rf .git && + git init --ref-storage=reftable && + mv .git/hooks .git/hooks-disabled && + test_commit file && + test_write_lines refs/heads/master refs/tags/file >expect && + git show-ref && + git show-ref | cut -f2 -d" " > actual && + test_cmp actual expect && + for count in $(test_seq 1 10) + do + test_commit "number $count" file.t $count number-$count || + return 1 + done && +(true || (test_pause && + git gc && + ls -1 .git/reftable >table-files && + test_line_count = 2 table-files && + git reflog refs/heads/master >output && + test_line_count = 11 output && + grep "commit (initial): first post" output && + grep "commit: number 10" output )) +' + +test_done
Signature · 1 line
gitgitgadget
Quoted 2 lines · Han-Wen Nienhuys via GitGitGadget
* what is a good test strategy? Could we have a CI flavor where we flip the default to reftable, and see how it fares?
I do not know if the current tests are prepared for it yet, e.g. there may be places that say "cat .git/refs/heads/master" and do something to its contents. But I think that it is a good goal to shoot for to make sure we can run all the tests with reftable enabled.
Thanks.
Quoted 11 lines · Junio C Hamano
Quoted 2 lines · Han-Wen Nienhuys via GitGitGadget
* what is a good test strategy? Could we have a CI flavor where we flip the default to reftable, and see how it fares?
I do not know if the current tests are prepared for it yet, e.g. there may be places that say "cat .git/refs/heads/master" and do something to its contents. But I think that it is a good goal to shoot for to make sure we can run all the tests with reftable enabled.
Obviously, it would be nice if all tests pass, but that doesn't seem very realistic.
hanwen@chiquinho:~/vc/git/t$ grep '^ok' log | wc 18457 154017 896360 hanwen@chiquinho:~/vc/git/t$ grep '^not ok' log | wc 3416 39825 228427
I expect that the bulk of this number reflects a (hopefully) small number of bugs in the reftable glue, but there are many tests that do things that are just not compatible with a more abstract ref database. For example,
not ok 10 - check rev-list
#
# echo $SHA >"$REAL/HEAD" && # test "$SHA" = "$(git rev-list HEAD)" #
What is the right way to approach this? Should the test use
git update-ref HEAD $SHA
instead of writing to the loose ref? Or should we skip it in case of reftable? Similarly
git update-ref $m $A && test $A = $(cat .git/$m)
My suggestion to leave these tests as is, but skip them if they can't work for reftable. However, this needs a new primitive in the test library; what should that primitive look like? Maybe a function, eg.
test_expect_succes_skip_reftable
or similar?
Signature · 7 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Quoted 8 lines · Han-Wen Nienhuys
For example,
not ok 10 - check rev-list
## echo $SHA >"$REAL/HEAD" && # test "$SHA" = "$(git rev-list HEAD)" #
Quoted 5 lines
What is the right way to approach this? Should the test use
git update-ref HEAD $SHA
instead of writing to the loose ref?
Preferred.
I didn't bother checking the context, but if the test is checking "the history leading to $SHA has only one commit, i.e. $SHA, and rev-list can handle that correctly", certainly that would be a preferred rewrite, rather than skipping the check for reftable, which may risk not noticing that HEAD is broken with reftable.
Quoted 5 lines
Or should we skip it in case of
reftable? Similarlygit update-ref $m $A && test $A = $(cat .git/$m)
Ditto.
We have "git show-ref -s --verify" for that; "cat .git/$m" certainly is wrong.
Thanks.
Quoted 16 lines · Junio C Hamano
Quoted 8 lines · Han-Wen Nienhuys
For example,
not ok 10 - check rev-list
## echo $SHA >"$REAL/HEAD" && # test "$SHA" = "$(git rev-list HEAD)" #Quoted 5 lines
What is the right way to approach this? Should the test use
git update-ref HEAD $SHA
instead of writing to the loose ref?
Quoted 7 lines
Preferred.
I didn't bother checking the context, but if the test is checking "the history leading to $SHA has only one commit, i.e. $SHA, and rev-list can handle that correctly", certainly that would be a preferred rewrite, rather than skipping the check for reftable, which may risk not noticing that HEAD is broken with reftable.
Now I have. The test is about various low-level machineries we have work correctly even if .git is *not* a directory but is a "gitfile:
$other location" (which is an underlying mechanism for multiple worktree support etc.), and it is making sure "git rev-list" understands HEAD in such a repository that uses the gitfile mechanism.
If I didn't know it, I might have said that "if we are interested in seeing $SHA is a root commit, we should check it more directly, perhaps by making sure 'cat-file commit $SHA' does not say 'parent' and that won't need to write to .git/HEAD at all", but the point of the test is to ensure 'rev-list' works correctly in such a repository, I think "update-ref HEAD $SHA" would be the right "fix" for the test.
Thanks.
Hi Junio & Han-Wen,
Quoted 32 lines · Junio C Hamano
Quoted 16 lines · Junio C Hamano
Quoted 8 lines · Han-Wen Nienhuys
For example,
not ok 10 - check rev-list
## echo $SHA >"$REAL/HEAD" && # test "$SHA" = "$(git rev-list HEAD)" #Quoted 5 lines
What is the right way to approach this? Should the test use
git update-ref HEAD $SHA
instead of writing to the loose ref?
Quoted 7 lines
Preferred.
I didn't bother checking the context, but if the test is checking "the history leading to $SHA has only one commit, i.e. $SHA, and rev-list can handle that correctly", certainly that would be a preferred rewrite, rather than skipping the check for reftable, which may risk not noticing that HEAD is broken with reftable.
Now I have. The test is about various low-level machineries we have work correctly even if .git is *not* a directory but is a "gitfile:
$other location" (which is an underlying mechanism for multiple worktree support etc.), and it is making sure "git rev-list" understands HEAD in such a repository that uses the gitfile mechanism.
Quoted 7 lines
If I didn't know it, I might have said that "if we are interested in seeing $SHA is a root commit, we should check it more directly, perhaps by making sure 'cat-file commit $SHA' does not say 'parent' and that won't need to write to .git/HEAD at all", but the point of the test is to ensure 'rev-list' works correctly in such a repository, I think "update-ref HEAD $SHA" would be the right "fix" for the test.
I believe the common strategy for this kind of thing is to
- introduce a GIT_TEST_* variable that is used to change the default (follow e.g. GIT_TEST_COMMIT_GRAPH as an example)
- either add a prereq that guards the test cases that cannot handle that knob (!REFTABLE in this case), or follow GIT_TEST_COMMIT_GRAPH's example again and _force_ its value to 0 for those test cases.
Ciao,
Dscho
v10 cover Reftable support git-core
In reply to Han-Wen Nienhuys via GitGitGadgetThis adds the reftable library, and hooks it up as a ref backend.
Includes testing support, to test: make -C t/ GIT_TEST_REFTABLE=1
Summary 18816 tests pass 2994 tests fail
Some issues:
* worktree support looks broken
* no detection for file/dir conflicts. You can have branch {a, a/b} simultaneously.
* submodules (eg. t1013)
* many tests inspect .git/logs/ directly.
v13
* Avoid exposing tombstones from reftable_stack to git
* Write peeled tags
* Fix GC issue: use git-refpack in test
* Fix unborn branches: must propagate errno from read_raw_ref.
* add GIT_TEST_REFTABLE environment to control default.
* add REFTABLE test prerequisite.
* fix & test update-ref -d.
* print error message for failed update-ref transactions.
Han-Wen Nienhuys (11):
refs.h: clarify reflog iteration order
Iterate over the "refs/" namespace in for_each_[raw]ref create .git/refs in files-backend.c
refs: document how ref_iterator_advance_fn should handle symrefs Add .gitattributes for the reftable/ directory
reftable: define version 2 of the spec to accomodate SHA256
reftable: clarify how empty tables should be written Add reftable library
Reftable support for git-core
Add some reftable testing infrastructure
t: use update-ref and show-ref to reading/writing refs
Jonathan Nieder (1):
reftable: file format documentation
Documentation/Makefile | 1 + Documentation/technical/reftable.txt | 1080 ++++++++++++++++ .../technical/repository-version.txt | 7 + Makefile | 26 +- builtin/clone.c | 3 +- builtin/init-db.c | 63 +- cache.h | 6 +- refs.c | 33 +- refs.h | 8 +- refs/files-backend.c | 6 + refs/refs-internal.h | 6 + refs/reftable-backend.c | 1045 +++++++++++++++ reftable/.gitattributes | 1 + reftable/LICENSE | 31 + reftable/README.md | 11 + reftable/VERSION | 5 + reftable/basics.c | 209 +++ reftable/basics.h | 53 + reftable/block.c | 425 ++++++ reftable/block.h | 124 ++ reftable/blocksource.h | 22 + reftable/bytes.c | 0 reftable/config.h | 1 + reftable/constants.h | 21 + reftable/dump.c | 97 ++ reftable/file.c | 98 ++ reftable/iter.c | 234 ++++ reftable/iter.h | 60 + reftable/merged.c | 327 +++++ reftable/merged.h | 36 + reftable/pq.c | 115 ++ reftable/pq.h | 34 + reftable/reader.c | 754 +++++++++++ reftable/reader.h | 68 + reftable/record.c | 1126 ++++++++++++++++ reftable/record.h | 121 ++ reftable/reftable.h | 527 ++++++++ reftable/slice.c | 224 ++++ reftable/slice.h | 76 ++ reftable/stack.c | 1151 +++++++++++++++++ reftable/stack.h | 44 + reftable/system.h | 53 + reftable/tree.c | 67 + reftable/tree.h | 34 + reftable/update.sh | 23 + reftable/writer.c | 661 ++++++++++ reftable/writer.h | 60 + reftable/zlib-compat.c | 92 ++ repository.c | 2 + repository.h | 3 + setup.c | 12 +- t/t0002-gitfile.sh | 2 +- t/t0031-reftable.sh | 99 ++ t/t1400-update-ref.sh | 32 +- t/t1506-rev-parse-diagnosis.sh | 2 +- t/t3210-pack-refs.sh | 6 + t/t6050-replace.sh | 2 +- t/t9020-remote-svn.sh | 4 +- t/test-lib.sh | 5 + 59 files changed, 9378 insertions(+), 60 deletions(-) create mode 100644 Documentation/technical/reftable.txt create mode 100644 refs/reftable-backend.c create mode 100644 reftable/.gitattributes create mode 100644 reftable/LICENSE create mode 100644 reftable/README.md create mode 100644 reftable/VERSION create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/blocksource.h create mode 100644 reftable/bytes.c create mode 100644 reftable/config.h create mode 100644 reftable/constants.h create mode 100644 reftable/dump.c create mode 100644 reftable/file.c create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/reftable.h create mode 100644 reftable/slice.c create mode 100644 reftable/slice.h create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/system.h create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100755 reftable/update.sh create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h create mode 100644 reftable/zlib-compat.c create mode 100755 t/t0031-reftable.sh
base-commit: e870325ee8575d5c3d7afe0ba2c9be072c692b65
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-539%2Fhanwen%2Freftable-v10
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-539/hanwen/reftable-v10
Pull-Request: https://github.com/gitgitgadget/git/pull/539
Range-diff vs v9:
1: b600d0bc6dd = 1: d3d8ed2032c refs.h: clarify reflog iteration order
2: 89b68145e8f = 2: 45fd65f72e0 Iterate over the "refs/" namespace in for_each_[raw]ref
3: 91f1efe24ec = 3: bc89bcd9c8c create .git/refs in files-backend.c
4: 56f65a2a0d7 = 4: fd67cdff0cd refs: document how ref_iterator_advance_fn should handle symrefs
5: b432c1cc2ae = 5: 5373828f854 Add .gitattributes for the reftable/ directory
6: b1d94be691a = 6: 8eeed29ebbf reftable: file format documentation
7: 3e418d27f67 = 7: 5ebc272e23d reftable: define version 2 of the spec to accomodate SHA256
8: 6f62be4067b = 8: 95aae041613 reftable: clarify how empty tables should be written
9: a30001ad1e8 ! 9: 59209a5ad39 Add reftable library
@@ Commit message
* reftable.h - the public API
+ * record.{c,h} - reading and writing records
+
* block.{c,h} - reading and writing blocks.
* writer.{c,h} - writing a complete reftable file.
@@ reftable/README.md (new)
## reftable/VERSION (new) ##
@@
-+commit b68690acad769d59e80cbb4f79c442ece133e6bd
++commit a6d6b31bea256044621d789df64a8159647f21bc
+Author: Han-Wen Nienhuys <[email protected]>
-+Date: Mon Apr 20 21:33:11 2020 +0200
++Date: Mon Apr 27 20:46:21 2020 +0200
+
-+ C: fix search/replace error in dump.c
++ C: prefix error codes with REFTABLE_
## reftable/basics.c (new) ##
@@
@@ reftable/basics.c (new)
+const char *reftable_error_str(int err)
+{
+ switch (err) {
-+ case IO_ERROR:
++ case REFTABLE_IO_ERROR:
+ return "I/O error";
-+ case FORMAT_ERROR:
-+ return "FORMAT_ERROR";
-+ case NOT_EXIST_ERROR:
-+ return "NOT_EXIST_ERROR";
-+ case LOCK_ERROR:
-+ return "LOCK_ERROR";
-+ case API_ERROR:
-+ return "API_ERROR";
-+ case ZLIB_ERROR:
-+ return "ZLIB_ERROR";
++ case REFTABLE_FORMAT_ERROR:
++ return "corrupt reftable file";
++ case REFTABLE_NOT_EXIST_ERROR:
++ return "file does not exist";
++ case REFTABLE_LOCK_ERROR:
++ return "data is outdated";
++ case REFTABLE_API_ERROR:
++ return "misuse of the reftable API";
++ case REFTABLE_ZLIB_ERROR:
++ return "zlib failure";
+ case -1:
+ return "general error";
+ default:
@@ reftable/basics.c (new)
+ }
+}
+
++int reftable_error_to_errno(int err) {
++ switch (err) {
++ case REFTABLE_IO_ERROR:
++ return EIO;
++ case REFTABLE_FORMAT_ERROR:
++ return EFAULT;
++ case REFTABLE_NOT_EXIST_ERROR:
++ return ENOENT;
++ case REFTABLE_LOCK_ERROR:
++ return EBUSY;
++ case REFTABLE_API_ERROR:
++ return EINVAL;
++ case REFTABLE_ZLIB_ERROR:
++ return EDOM;
++ default:
++ return ERANGE;
++ }
++}
++
++
+void *(*reftable_malloc_ptr)(size_t sz) = &malloc;
+void *(*reftable_realloc_ptr)(void *, size_t) = &realloc;
+void (*reftable_free_ptr)(void *) = &free;
@@ reftable/block.c (new)
+
+ if (Z_OK != zresult) {
+ slice_clear(&compressed);
-+ return ZLIB_ERROR;
++ return REFTABLE_ZLIB_ERROR;
+ }
+
+ memcpy(w->buf + block_header_skip, compressed.buf,
@@ reftable/block.c (new)
+ uint32_t sz = get_be24(block->data + header_off + 1);
+
+ if (!is_block_type(typ)) {
-+ return FORMAT_ERROR;
++ return REFTABLE_FORMAT_ERROR;
+ }
+
+ if (typ == BLOCK_TYPE_LOG) {
@@ reftable/block.c (new)
+ &dst_len, block->data + block_header_skip,
+ &src_len)) {
+ slice_clear(&uncompressed);
-+ return ZLIB_ERROR;
++ return REFTABLE_ZLIB_ERROR;
+ }
+
+ block_source_return_block(block->source, block);
@@ reftable/file.c (new)
+ int fd = open(name, O_RDONLY);
+ if (fd < 0) {
+ if (errno == ENOENT) {
-+ return NOT_EXIST_ERROR;
++ return REFTABLE_NOT_EXIST_ERROR;
+ }
+ return -1;
+ }
@@ reftable/iter.c (new)
+ }
+ if (err > 0) {
+ /* indexed block does not exist. */
-+ return FORMAT_ERROR;
++ return REFTABLE_FORMAT_ERROR;
+ }
+ }
+ block_reader_start(&it->block_reader, &it->cur);
@@ reftable/merged.c (new)
+ return 0;
+}
+
-+static int merged_iter_next(struct merged_iter *mi, struct record rec)
++static int merged_iter_next_entry(struct merged_iter *mi, struct record rec)
+{
+ struct slice entry_key = { 0 };
-+ struct pq_entry entry = merged_iter_pqueue_remove(&mi->pq);
-+ int err = merged_iter_advance_subiter(mi, entry.index);
++ struct pq_entry entry = { 0 };
++ int err = 0;
++
++ if (merged_iter_pqueue_is_empty(mi->pq)) {
++ return 1;
++ }
++
++ entry = merged_iter_pqueue_remove(&mi->pq);
++ err = merged_iter_advance_subiter(mi, entry.index);
+ if (err < 0) {
+ return err;
+ }
+
-+
-+ /*
-+ One can also use reftable as datacenter-local storage, where the ref
-+ database is maintained in globally consistent database (eg.
-+ CockroachDB or Spanner). In this scenario, replication delays together
-+ with compaction may cause newer tables to contain older entries. In
-+ such a deployment, the loop below must be changed to collect all
-+ entries for the same key, and return new the newest one.
-+ */
++ /*
++ One can also use reftable as datacenter-local storage, where the ref
++ database is maintained in globally consistent database (eg.
++ CockroachDB or Spanner). In this scenario, replication delays together
++ with compaction may cause newer tables to contain older entries. In
++ such a deployment, the loop below must be changed to collect all
++ entries for the same key, and return new the newest one.
++ */
+ record_key(entry.rec, &entry_key);
+ while (!merged_iter_pqueue_is_empty(mi->pq)) {
+ struct pq_entry top = merged_iter_pqueue_top(mi->pq);
@@ reftable/merged.c (new)
+ return 0;
+}
+
++static int merged_iter_next(struct merged_iter *mi, struct record rec)
++{
++ while (true) {
++ int err = merged_iter_next_entry(mi, rec);
++ if (err == 0 && mi->suppress_deletions &&
++ record_is_deletion(rec)) {
++ continue;
++ }
++
++ return err;
++ }
++}
++
+static int merged_iter_next_void(void *p, struct record rec)
+{
+ struct merged_iter *mi = (struct merged_iter *)p;
@@ reftable/merged.c (new)
+ for (i = 0; i < n; i++) {
+ struct reftable_reader *r = stack[i];
+ if (r->hash_id != hash_id) {
-+ return FORMAT_ERROR;
++ return REFTABLE_FORMAT_ERROR;
+ }
+ if (i > 0 && last_max >= reftable_reader_min_update_index(r)) {
-+ return FORMAT_ERROR;
++ return REFTABLE_FORMAT_ERROR;
+ }
+ if (i == 0) {
+ first_min = reftable_reader_min_update_index(r);
@@ reftable/merged.c (new)
+ .stack = iters,
+ .typ = record_type(rec),
+ .hash_id = mt->hash_id,
++ .suppress_deletions = mt->suppress_deletions,
+ };
+ int n = 0;
+ int err = 0;
@@ reftable/merged.h (new)
+ struct reftable_reader **stack;
+ int stack_len;
+ uint32_t hash_id;
++ bool suppress_deletions;
+
+ uint64_t min;
+ uint64_t max;
@@ reftable/merged.h (new)
+ uint32_t hash_id;
+ int stack_len;
+ byte typ;
++ bool suppress_deletions;
+ struct merged_iter_pqueue pq;
+};
+
@@ reftable/reader.c (new)
+ byte *f = footer;
+ int err = 0;
+ if (memcmp(f, "REFT", 4)) {
-+ err = FORMAT_ERROR;
++ err = REFTABLE_FORMAT_ERROR;
+ goto exit;
+ }
+ f += 4;
+
+ if (memcmp(footer, header, header_size(r->version))) {
-+ err = FORMAT_ERROR;
++ err = REFTABLE_FORMAT_ERROR;
+ goto exit;
+ }
+
@@ reftable/reader.c (new)
+ case SHA256_ID:
+ break;
+ default:
-+ err = FORMAT_ERROR;
++ err = REFTABLE_FORMAT_ERROR;
+ goto exit;
+ }
+ f += 4;
@@ reftable/reader.c (new)
+ uint32_t file_crc = get_be32(f);
+ f += 4;
+ if (computed_crc != file_crc) {
-+ err = FORMAT_ERROR;
++ err = REFTABLE_FORMAT_ERROR;
+ goto exit;
+ }
+ }
@@ reftable/reader.c (new)
+ /* Need +1 to read type of first block. */
+ err = block_source_read_block(source, &header, 0, header_size(2) + 1);
+ if (err != header_size(2) + 1) {
-+ err = IO_ERROR;
++ err = REFTABLE_IO_ERROR;
+ goto exit;
+ }
+
+ if (memcmp(header.data, "REFT", 4)) {
-+ err = FORMAT_ERROR;
++ err = REFTABLE_FORMAT_ERROR;
+ goto exit;
+ }
+ r->version = header.data[4];
+ if (r->version != 1 && r->version != 2) {
-+ err = FORMAT_ERROR;
++ err = REFTABLE_FORMAT_ERROR;
+ goto exit;
+ }
+
@@ reftable/reader.c (new)
+ err = block_source_read_block(source, &footer, r->size,
+ footer_size(r->version));
+ if (err != footer_size(r->version)) {
-+ err = IO_ERROR;
++ err = REFTABLE_IO_ERROR;
+ goto exit;
+ }
+
@@ reftable/reader.c (new)
+static int table_iter_next(struct table_iter *ti, struct record rec)
+{
+ if (record_type(rec) != ti->typ) {
-+ return API_ERROR;
++ return REFTABLE_API_ERROR;
+ }
+
+ while (true) {
@@ reftable/reader.c (new)
+ }
+
+ if (next.typ != BLOCK_TYPE_INDEX) {
-+ err = FORMAT_ERROR;
++ err = REFTABLE_FORMAT_ERROR;
+ break;
+ }
+
@@ reftable/record.c (new)
+ return start_len - in.len;
+}
+
-+static byte reftable_ref_record_type(void)
-+{
-+ return BLOCK_TYPE_REF;
-+}
-+
+static void reftable_ref_record_key(const void *r, struct slice *dest)
+{
+ const struct reftable_ref_record *rec =
@@ reftable/record.c (new)
+ }
+}
+
-+void reftable_ref_record_print(struct reftable_ref_record *ref, int hash_size)
++void reftable_ref_record_print(struct reftable_ref_record *ref,
++ uint32_t hash_id)
+{
+ char hex[SHA256_SIZE + 1] = { 0 };
-+
+ printf("ref{%s(%" PRIu64 ") ", ref->ref_name, ref->update_index);
+ if (ref->value != NULL) {
-+ hex_format(hex, ref->value, hash_size);
++ hex_format(hex, ref->value, hash_size(hash_id));
+ printf("%s", hex);
+ }
+ if (ref->target_value != NULL) {
-+ hex_format(hex, ref->target_value, hash_size);
++ hex_format(hex, ref->target_value, hash_size(hash_id));
+ printf(" (T %s)", hex);
+ }
+ if (ref->target != NULL) {
@@ reftable/record.c (new)
+ return start.len - in.len;
+}
+
++static bool reftable_ref_record_is_deletion_void(const void *p)
++{
++ return reftable_ref_record_is_deletion(
++ (const struct reftable_ref_record *)p);
++}
++
+struct record_vtable reftable_ref_record_vtable = {
+ .key = &reftable_ref_record_key,
-+ .type = &reftable_ref_record_type,
++ .type = BLOCK_TYPE_REF,
+ .copy_from = &reftable_ref_record_copy_from,
+ .val_type = &reftable_ref_record_val_type,
+ .encode = &reftable_ref_record_encode,
+ .decode = &reftable_ref_record_decode,
+ .clear = &reftable_ref_record_clear_void,
++ .is_deletion = &reftable_ref_record_is_deletion_void,
+};
+
-+static byte obj_record_type(void)
-+{
-+ return BLOCK_TYPE_OBJ;
-+}
-+
+static void obj_record_key(const void *r, struct slice *dest)
+{
+ const struct obj_record *rec = (const struct obj_record *)r;
@@ reftable/record.c (new)
+ return start.len - in.len;
+}
+
++static bool not_a_deletion(const void *p)
++{
++ return false;
++}
++
+struct record_vtable obj_record_vtable = {
+ .key = &obj_record_key,
-+ .type = &obj_record_type,
++ .type = BLOCK_TYPE_OBJ,
+ .copy_from = &obj_record_copy_from,
+ .val_type = &obj_record_val_type,
+ .encode = &obj_record_encode,
+ .decode = &obj_record_decode,
+ .clear = &obj_record_clear,
++ .is_deletion = not_a_deletion,
+};
+
-+void reftable_log_record_print(struct reftable_log_record *log, int hash_size)
++void reftable_log_record_print(struct reftable_log_record *log,
++ uint32_t hash_id)
+{
+ char hex[SHA256_SIZE + 1] = { 0 };
+
+ printf("log{%s(%" PRIu64 ") %s <%s> %" PRIu64 " %04d\n", log->ref_name,
+ log->update_index, log->name, log->email, log->time,
+ log->tz_offset);
-+ hex_format(hex, log->old_hash, hash_size);
++ hex_format(hex, log->old_hash, hash_size(hash_id));
+ printf("%s => ", hex);
-+ hex_format(hex, log->new_hash, hash_size);
++ hex_format(hex, log->new_hash, hash_size(hash_id));
+ printf("%s\n\n%s\n}\n", hex, log->message);
+}
+
-+static byte reftable_log_record_type(void)
-+{
-+ return BLOCK_TYPE_LOG;
-+}
-+
+static void reftable_log_record_key(const void *r, struct slice *dest)
+{
+ const struct reftable_log_record *rec =
@@ reftable/record.c (new)
+ int n;
+
+ if (key.len <= 9 || key.buf[key.len - 9] != 0) {
-+ return FORMAT_ERROR;
++ return REFTABLE_FORMAT_ERROR;
+ }
+
+ r->ref_name = reftable_realloc(r->ref_name, key.len - 8);
@@ reftable/record.c (new)
+ }
+
+ if (in.len < 2 * hash_size) {
-+ return FORMAT_ERROR;
++ return REFTABLE_FORMAT_ERROR;
+ }
+
+ r->old_hash = reftable_realloc(r->old_hash, hash_size);
@@ reftable/record.c (new)
+
+error:
+ slice_clear(&dest);
-+ return FORMAT_ERROR;
++ return REFTABLE_FORMAT_ERROR;
+}
+
+static bool null_streq(char *a, char *b)
@@ reftable/record.c (new)
+ a->update_index == b->update_index;
+}
+
++static bool reftable_log_record_is_deletion_void(const void *p)
++{
++ return reftable_log_record_is_deletion(
++ (const struct reftable_log_record *)p);
++}
++
+struct record_vtable reftable_log_record_vtable = {
+ .key = &reftable_log_record_key,
-+ .type = &reftable_log_record_type,
++ .type = BLOCK_TYPE_LOG,
+ .copy_from = &reftable_log_record_copy_from,
+ .val_type = &reftable_log_record_val_type,
+ .encode = &reftable_log_record_encode,
+ .decode = &reftable_log_record_decode,
+ .clear = &reftable_log_record_clear_void,
++ .is_deletion = &reftable_log_record_is_deletion_void,
+};
+
+struct record new_record(byte typ)
@@ reftable/record.c (new)
+ reftable_free(record_yield(rec));
+}
+
-+static byte index_record_type(void)
-+{
-+ return BLOCK_TYPE_INDEX;
-+}
-+
+static void index_record_key(const void *r, struct slice *dest)
+{
+ struct index_record *rec = (struct index_record *)r;
@@ reftable/record.c (new)
+
+struct record_vtable index_record_vtable = {
+ .key = &index_record_key,
-+ .type = &index_record_type,
++ .type = BLOCK_TYPE_INDEX,
+ .copy_from = &index_record_copy_from,
+ .val_type = &index_record_val_type,
+ .encode = &index_record_encode,
+ .decode = &index_record_decode,
+ .clear = &index_record_clear,
++ .is_deletion = ¬_a_deletion,
+};
+
+void record_key(struct record rec, struct slice *dest)
@@ reftable/record.c (new)
+
+byte record_type(struct record rec)
+{
-+ return rec.ops->type();
++ return rec.ops->type;
+}
+
+int record_encode(struct record rec, struct slice dest, int hash_size)
@@ reftable/record.c (new)
+
+void record_copy_from(struct record rec, struct record src, int hash_size)
+{
-+ assert(src.ops->type() == rec.ops->type());
++ assert(src.ops->type == rec.ops->type);
+
+ rec.ops->copy_from(rec.data, src.data, hash_size);
+}
@@ reftable/record.c (new)
+ return rec.ops->clear(rec.data);
+}
+
++bool record_is_deletion(struct record rec)
++{
++ return rec.ops->is_deletion(rec.data);
++}
++
+void record_from_ref(struct record *rec, struct reftable_ref_record *ref_rec)
+{
+ rec->data = ref_rec;
@@ reftable/record.h (new)
+ void (*key)(const void *rec, struct slice *dest);
+
+ /* The record type of ('r' for ref). */
-+ byte (*type)(void);
++ byte type;
+
+ void (*copy_from)(void *dest, const void *src, int hash_size);
+
@@ reftable/record.h (new)
+
+ /* deallocate and null the record. */
+ void (*clear)(void *rec);
++
++ /* is this a tombstone? */
++ bool (*is_deletion)(const void *rec);
+};
+
+/* record is a generic wrapper for different types of records. */
@@ reftable/record.h (new)
+int record_encode(struct record rec, struct slice dest, int hash_size);
+int record_decode(struct record rec, struct slice key, byte extra,
+ struct slice src, int hash_size);
++bool record_is_deletion(struct record rec);
+
+/* zeroes out the embedded record */
+void record_clear(struct record rec);
@@ reftable/reftable.h (new)
+int reftable_ref_record_is_deletion(const struct reftable_ref_record *ref);
+
+/* prints a reftable_ref_record onto stdout */
-+void reftable_ref_record_print(struct reftable_ref_record *ref, int hash_size);
++void reftable_ref_record_print(struct reftable_ref_record *ref,
++ uint32_t hash_id);
+
+/* frees and nulls all pointer values. */
+void reftable_ref_record_clear(struct reftable_ref_record *ref);
@@ reftable/reftable.h (new)
+ struct reftable_log_record *b, int hash_size);
+
+/* dumps a reftable_log_record on stdout, for debugging/testing. */
-+void reftable_log_record_print(struct reftable_log_record *log, int hash_size);
++void reftable_log_record_print(struct reftable_log_record *log,
++ uint32_t hash_id);
+
+/****************************************************************
+ Error handling
@@ reftable/reftable.h (new)
+/* different types of errors */
+enum reftable_error {
+ /* Unexpected file system behavior */
-+ IO_ERROR = -2,
++ REFTABLE_IO_ERROR = -2,
+
+ /* Format inconsistency on reading data
+ */
-+ FORMAT_ERROR = -3,
++ REFTABLE_FORMAT_ERROR = -3,
+
+ /* File does not exist. Returned from block_source_from_file(), because
+ it needs special handling in stack.
+ */
-+ NOT_EXIST_ERROR = -4,
++ REFTABLE_NOT_EXIST_ERROR = -4,
+
+ /* Trying to write out-of-date data. */
-+ LOCK_ERROR = -5,
++ REFTABLE_LOCK_ERROR = -5,
+
+ /* Misuse of the API:
+ - on writing a record with NULL ref_name.
@@ reftable/reftable.h (new)
+ - on writing a ref or log record before the stack's next_update_index
+ - on reading a reftable_ref_record from log iterator, or vice versa.
+ */
-+ API_ERROR = -6,
++ REFTABLE_API_ERROR = -6,
+
+ /* Decompression error */
-+ ZLIB_ERROR = -7,
++ REFTABLE_ZLIB_ERROR = -7,
+
+ /* Wrote a table without blocks. */
-+ EMPTY_TABLE_ERROR = -8,
++ REFTABLE_EMPTY_TABLE_ERROR = -8,
+};
+
+/* convert the numeric error code to a string. The string should not be
+ * deallocated. */
+const char *reftable_error_str(int err);
+
++/*
++ * Convert the numeric error code to an equivalent errno code.
++ */
++int reftable_error_to_errno(int err);
++
+/****************************************************************
+ Writing
+
@@ reftable/reftable.h (new)
+
+/* Set the range of update indices for the records we will add. When
+ writing a table into a stack, the min should be at least
-+ reftable_stack_next_update_index(), or API_ERROR is returned.
++ reftable_stack_next_update_index(), or REFTABLE_API_ERROR is returned.
+
+ For transactional updates, typically min==max. When converting an existing
+ ref database into a single reftable, this would be a range of update-index
@@ reftable/reftable.h (new)
+
+/* adds a reftable_ref_record. Must be called in ascending
+ order. The update_index must be within the limits set by
-+ reftable_writer_set_limits(), or API_ERROR is returned.
++ reftable_writer_set_limits(), or REFTABLE_API_ERROR is returned.
+
+ It is an error to write a ref record after a log record.
+ */
@@ reftable/reftable.h (new)
+ struct reftable_reader **stack, int n,
+ uint32_t hash_id);
+
-+/* returns the hash id used in this merged table. */
-+uint32_t reftable_merged_table_hash_id(struct reftable_merged_table *mt);
-+
+/* returns an iterator positioned just before 'name' */
+int reftable_merged_table_seek_ref(struct reftable_merged_table *mt,
+ struct reftable_iterator *it,
@@ reftable/stack.c (new)
+ reftable_calloc(sizeof(struct reftable_stack));
+ struct slice list_file_name = {};
+ int err = 0;
++
++ if (config.hash_id == 0) {
++ config.hash_id = SHA1_ID;
++ }
++
+ *dest = NULL;
+
+ slice_set_string(&list_file_name, dir);
-+ slice_append_string(&list_file_name, "/reftables.list");
++ slice_append_string(&list_file_name, "/tables.list");
+
+ p->list_file = slice_to_string(list_file_name);
+ slice_clear(&list_file_name);
@@ reftable/stack.c (new)
+ return err;
+}
+
-+static int fread_lines(FILE *f, char ***namesp)
++static int fd_read_lines(int fd, char ***namesp)
+{
-+ long size = 0;
-+ int err = fseek(f, 0, SEEK_END);
++ off_t size = lseek(fd, 0, SEEK_END);
+ char *buf = NULL;
-+ if (err < 0) {
-+ err = IO_ERROR;
-+ goto exit;
-+ }
-+ size = ftell(f);
++ int err = 0;
+ if (size < 0) {
-+ err = IO_ERROR;
++ err = REFTABLE_IO_ERROR;
+ goto exit;
+ }
-+ err = fseek(f, 0, SEEK_SET);
++ err = lseek(fd, 0, SEEK_SET);
+ if (err < 0) {
-+ err = IO_ERROR;
++ err = REFTABLE_IO_ERROR;
+ goto exit;
+ }
+
+ buf = reftable_malloc(size + 1);
-+ if (fread(buf, 1, size, f) != size) {
-+ err = IO_ERROR;
++ if (read(fd, buf, size) != size) {
++ err = REFTABLE_IO_ERROR;
+ goto exit;
+ }
+ buf[size] = 0;
+
+ parse_names(buf, size, namesp);
++
+exit:
+ reftable_free(buf);
+ return err;
@@ reftable/stack.c (new)
+
+int read_lines(const char *filename, char ***namesp)
+{
-+ FILE *f = fopen(filename, "r");
++ int fd = open(filename, O_RDONLY, 0644);
+ int err = 0;
-+ if (f == NULL) {
++ if (fd < 0) {
+ if (errno == ENOENT) {
+ *namesp = reftable_calloc(sizeof(char *));
+ return 0;
+ }
+
-+ return IO_ERROR;
++ return REFTABLE_IO_ERROR;
+ }
-+ err = fread_lines(f, namesp);
-+ fclose(f);
++ err = fd_read_lines(fd, namesp);
++ close(fd);
+ return err;
+}
+
@@ reftable/stack.c (new)
+ merged_table_clear(st->merged);
+ reftable_merged_table_free(st->merged);
+ }
++ new_merged->suppress_deletions = true;
+ st->merged = new_merged;
+
+ {
@@ reftable/stack.c (new)
+ char **names_after = NULL;
+ struct timeval now = { 0 };
+ int err = gettimeofday(&now, NULL);
++ int err2 = 0;
+ if (err < 0) {
+ return err;
+ }
@@ reftable/stack.c (new)
+ free_names(names);
+ break;
+ }
-+ if (err != NOT_EXIST_ERROR) {
++ if (err != REFTABLE_NOT_EXIST_ERROR) {
+ free_names(names);
+ return err;
+ }
+
-+ err = read_lines(st->list_file, &names_after);
-+ if (err < 0) {
++ /* err == REFTABLE_NOT_EXIST_ERROR can be caused by a concurrent
++ writer. Check if there was one by checking if the name list
++ changed.
++ */
++ err2 = read_lines(st->list_file, &names_after);
++ if (err2 < 0) {
+ free_names(names);
-+ return err;
++ return err2;
+ }
+
+ if (names_equal(names_after, names)) {
+ free_names(names);
+ free_names(names_after);
-+ return -1;
++ return err;
+ }
+ free_names(names);
+ free_names(names_after);
@@ reftable/stack.c (new)
+{
+ int err = stack_try_add(st, write, arg);
+ if (err < 0) {
-+ if (err == LOCK_ERROR) {
-+ err = reftable_stack_reload(st);
++ if (err == REFTABLE_LOCK_ERROR) {
++ // Ignore error return, we want to propagate
++ // REFTABLE_LOCK_ERROR.
++ reftable_stack_reload(st);
+ }
+ return err;
+ }
+
-+ return reftable_stack_auto_compact(st);
++ if (!st->disable_auto_compact) {
++ return reftable_stack_auto_compact(st);
++ }
++
++ return 0;
+}
+
+static void format_name(struct slice *dest, uint64_t min, uint64_t max)
+{
+ char buf[100];
-+ snprintf(buf, sizeof(buf), "%012" PRIx64 "-%012" PRIx64, min, max);
++ snprintf(buf, sizeof(buf), "0x%012" PRIx64 "-0x%012" PRIx64, min, max);
+ slice_set_string(dest, buf);
+}
+
@@ reftable/stack.c (new)
+ O_EXCL | O_CREAT | O_WRONLY, 0644);
+ if (add->lock_file_fd < 0) {
+ if (errno == EEXIST) {
-+ err = LOCK_ERROR;
++ err = REFTABLE_LOCK_ERROR;
+ } else {
-+ err = IO_ERROR;
++ err = REFTABLE_IO_ERROR;
+ }
+ goto exit;
+ }
@@ reftable/stack.c (new)
+ }
+
+ if (err > 1) {
-+ err = LOCK_ERROR;
++ err = REFTABLE_LOCK_ERROR;
+ goto exit;
+ }
+
@@ reftable/stack.c (new)
+ err = write(add->lock_file_fd, table_list.buf, table_list.len);
+ slice_clear(&table_list);
+ if (err < 0) {
-+ err = IO_ERROR;
++ err = REFTABLE_IO_ERROR;
+ goto exit;
+ }
+
+ err = close(add->lock_file_fd);
+ add->lock_file_fd = 0;
+ if (err < 0) {
-+ err = IO_ERROR;
++ err = REFTABLE_IO_ERROR;
+ goto exit;
+ }
+
+ err = rename(slice_as_string(&add->lock_file_name),
+ add->stack->list_file);
+ if (err < 0) {
-+ err = IO_ERROR;
++ err = REFTABLE_IO_ERROR;
+ goto exit;
+ }
+
@@ reftable/stack.c (new)
+
+ tab_fd = mkstemp((char *)slice_as_string(&temp_tab_file_name));
+ if (tab_fd < 0) {
-+ err = IO_ERROR;
++ err = REFTABLE_IO_ERROR;
+ goto exit;
+ }
+
@@ reftable/stack.c (new)
+ }
+
+ err = reftable_writer_close(wr);
-+ if (err == EMPTY_TABLE_ERROR) {
++ if (err == REFTABLE_EMPTY_TABLE_ERROR) {
+ err = 0;
+ goto exit;
+ }
@@ reftable/stack.c (new)
+ err = close(tab_fd);
+ tab_fd = 0;
+ if (err < 0) {
-+ err = IO_ERROR;
++ err = REFTABLE_IO_ERROR;
+ goto exit;
+ }
+
+ if (wr->min_update_index < next_update_index) {
-+ err = API_ERROR;
++ err = REFTABLE_API_ERROR;
+ goto exit;
+ }
+
@@ reftable/stack.c (new)
+ err = rename(slice_as_string(&temp_tab_file_name),
+ slice_as_string(&tab_file_name));
+ if (err < 0) {
-+ err = IO_ERROR;
++ err = REFTABLE_IO_ERROR;
+ goto exit;
+ }
+
@@ reftable/stack.c (new)
+ if (errno == EEXIST) {
+ err = 1;
+ } else {
-+ err = IO_ERROR;
++ err = REFTABLE_IO_ERROR;
+ }
+ goto exit;
+ }
@@ reftable/stack.c (new)
+ } else if (sublock_file_fd < 0) {
+ if (errno == EEXIST) {
+ err = 1;
++ } else {
++ err = REFTABLE_IO_ERROR;
+ }
-+ err = IO_ERROR;
+ }
+ }
+
@@ reftable/stack.c (new)
+ expiry);
+ /* Compaction + tombstones can create an empty table out of non-empty
+ * tables. */
-+ is_empty_table = (err == EMPTY_TABLE_ERROR);
++ is_empty_table = (err == REFTABLE_EMPTY_TABLE_ERROR);
+ if (is_empty_table) {
+ err = 0;
+ }
@@ reftable/stack.c (new)
+ if (errno == EEXIST) {
+ err = 1;
+ } else {
-+ err = IO_ERROR;
++ err = REFTABLE_IO_ERROR;
+ }
+ goto exit;
+ }
@@ reftable/stack.c (new)
+ err = rename(slice_as_string(&temp_tab_file_name),
+ slice_as_string(&new_table_path));
+ if (err < 0) {
++ err = REFTABLE_IO_ERROR;
+ goto exit;
+ }
+ }
@@ reftable/stack.c (new)
+
+ err = write(lock_file_fd, ref_list_contents.buf, ref_list_contents.len);
+ if (err < 0) {
++ err = REFTABLE_IO_ERROR;
+ unlink(slice_as_string(&new_table_path));
+ goto exit;
+ }
+ err = close(lock_file_fd);
+ lock_file_fd = 0;
+ if (err < 0) {
++ err = REFTABLE_IO_ERROR;
+ unlink(slice_as_string(&new_table_path));
+ goto exit;
+ }
+
+ err = rename(slice_as_string(&lock_file_name), st->list_file);
+ if (err < 0) {
++ err = REFTABLE_IO_ERROR;
+ unlink(slice_as_string(&new_table_path));
+ goto exit;
+ }
@@ reftable/stack.c (new)
+{
+ uint64_t *sizes =
+ reftable_calloc(sizeof(uint64_t) * st->merged->stack_len);
++ int version = (st->config.hash_id == SHA1_ID) ? 1 : 2;
++ int overhead = footer_size(version) + header_size(version) - 1;
+ int i = 0;
+ for (i = 0; i < st->merged->stack_len; i++) {
-+ /* overhead is 24 + 68 = 92. */
-+ sizes[i] = st->merged->stack[i]->size - 91;
++ sizes[i] = st->merged->stack[i]->size - overhead;
+ }
+ return sizes;
+}
@@ reftable/stack.h (new)
+#define STACK_H
+
+#include "reftable.h"
++#include "system.h"
+
+struct reftable_stack {
+ char *list_file;
+ char *reftable_dir;
++ bool disable_auto_compact;
+
+ struct reftable_write_options config;
+
@@ reftable/update.sh (new)
@@
+#!/bin/sh
+
-+set -eux
++set -eu
+
-+((cd reftable-repo && git fetch origin && git checkout origin/master ) ||
-+git clone https://github.com/google/reftable reftable-repo) && \
-+cp reftable-repo/c/*.[ch] reftable/ && \
-+cp reftable-repo/c/include/*.[ch] reftable/ && \
-+cp reftable-repo/LICENSE reftable/ &&
++# Override this to import from somewhere else, say "../reftable".
++SRC=${SRC:-origin} BRANCH=${BRANCH:-origin/master}
++
++((git --git-dir reftable-repo/.git fetch ${SRC} && cd reftable-repo && git checkout ${BRANCH} ) ||
++ git clone https://github.com/google/reftable reftable-repo)
++
++cp reftable-repo/c/*.[ch] reftable/
++cp reftable-repo/c/include/*.[ch] reftable/
++cp reftable-repo/LICENSE reftable/
+git --git-dir reftable-repo/.git show --no-patch origin/master \
-+> reftable/VERSION && \
-+sed -i~ 's|if REFTABLE_IN_GITCORE|if 1 /* REFTABLE_IN_GITCORE */|' reftable/system.h
++ > reftable/VERSION
++
++mv reftable/system.h reftable/system.h~
++sed 's|if REFTABLE_IN_GITCORE|if 1 /* REFTABLE_IN_GITCORE */|' < reftable/system.h~ > reftable/system.h
++
++# Remove unittests and compatibility hacks we don't need here.
+rm reftable/*_test.c reftable/test_framework.* reftable/compat.*
-+git add reftable/*.[ch]
++
++git add reftable/*.[ch] reftable/LICENSE reftable/VERSION
## reftable/writer.c (new) ##
@@
@@ reftable/writer.c (new)
+ int err = 0;
+
+ if (ref->ref_name == NULL) {
-+ return API_ERROR;
++ return REFTABLE_API_ERROR;
+ }
+ if (ref->update_index < w->min_update_index ||
+ ref->update_index > w->max_update_index) {
-+ return API_ERROR;
++ return REFTABLE_API_ERROR;
+ }
+
+ record_from_ref(&rec, ©);
@@ reftable/writer.c (new)
+ struct reftable_log_record *log)
+{
+ if (log->ref_name == NULL) {
-+ return API_ERROR;
++ return REFTABLE_API_ERROR;
+ }
+
+ if (w->block_writer != NULL &&
@@ reftable/writer.c (new)
+ }
+
+ if (empty_table) {
-+ err = EMPTY_TABLE_ERROR;
++ err = REFTABLE_EMPTY_TABLE_ERROR;
+ goto exit;
+ }
+
10: ad72edbcfd4 ! 10: be2371cd6e4 Reftable support for git-core
@@ Commit message
TODO:
* Resolve spots marked with XXX
- * Test strategy?
+
+ * Detect and prevent directory/file conflicts in naming.
+
+ * Support worktrees (t0002-gitfile "linked repo" testcase)
Example use: see t/t0031-reftable.sh
@@ builtin/clone.c: int cmd_clone(int argc, const char **argv, const char *prefix)
}
- init_db(git_dir, real_git_dir, option_template, GIT_HASH_UNKNOWN, INIT_DB_QUIET);
-+ init_db(git_dir, real_git_dir, option_template,
-+ GIT_HASH_UNKNOWN,
-+ DEFAULT_REF_STORAGE, /* XXX */
-+ INIT_DB_QUIET);
++ init_db(git_dir, real_git_dir, option_template, GIT_HASH_UNKNOWN,
++ DEFAULT_REF_STORAGE, INIT_DB_QUIET);
if (real_git_dir)
git_dir = real_git_dir;
@@ builtin/init-db.c: static int needs_work_tree_config(const char *git_dir, const
}
-void initialize_repository_version(int hash_algo)
-+void initialize_repository_version(int hash_algo, const char *ref_storage_format)
++void initialize_repository_version(int hash_algo,
++ const char *ref_storage_format)
{
char repo_version_string[10];
int repo_version = GIT_REPO_VERSION;
@@ builtin/init-db.c: void initialize_repository_version(int hash_algo)
#endif
- if (hash_algo != GIT_HASH_SHA1)
-+ if (hash_algo != GIT_HASH_SHA1 || !strcmp(ref_storage_format, "reftable"))
++ if (hash_algo != GIT_HASH_SHA1 ||
++ !strcmp(ref_storage_format, "reftable"))
repo_version = GIT_REPO_VERSION_READ;
/* This forces creation of new config file */
@@ builtin/init-db.c: static int create_default_files(const char *template_path,
+ path = git_path_buf(&buf, "HEAD");
+ reinit = (!access(path, R_OK) ||
+ readlink(path, junk, sizeof(junk) - 1) != -1);
++
++ /*
++ * refs/heads is a file when using reftable. We can't reinitialize with
++ * a reftable because it will overwrite HEAD
++ */
++ if (reinit && (!strcmp(fmt->ref_storage, "reftable")) ==
++ is_directory(git_path_buf(&buf, "refs/heads"))) {
++ die("cannot switch ref storage format.");
++ }
+
/*
* We need to create a "refs" dir in any case so that older
* versions of git can tell that this is a repository.
+ */
+-
+ if (refs_init_db(&err))
+ die("failed to set up refs db: %s", err.buf);
+
@@ builtin/init-db.c: static int create_default_files(const char *template_path,
* Create the default symlink from ".git/HEAD" to the "master"
* branch, if it does not exist yet.
@@ builtin/init-db.c: static int create_default_files(const char *template_path,
if (!reinit) {
if (create_symref("HEAD", "refs/heads/master", NULL) < 0)
exit(1);
-+ } else {
-+ /*
-+ * XXX should check whether our ref backend matches the
-+ * original one; if not, either die() or convert
-+ */
- }
-
+- }
+-
- initialize_repository_version(fmt->hash_algo);
-+ initialize_repository_version(fmt->hash_algo, fmt->ref_storage);
++ }
++ initialize_repository_version(fmt->hash_algo, fmt->ref_storage);
++
/* Check filemode trustability */
path = git_path_buf(&buf, "config");
+ filemode = TEST_FILEMODE;
@@ builtin/init-db.c: static void validate_hash_algorithm(struct repository_format *repo_fmt, int hash
}
int init_db(const char *git_dir, const char *real_git_dir,
- const char *template_dir, int hash, unsigned int flags)
+ const char *template_dir, int hash, const char *ref_storage_format,
-+ unsigned int flags)
++ unsigned int flags)
{
int reinit;
int exist_ok = flags & INIT_DB_EXIST_OK;
@@ builtin/init-db.c: int init_db(const char *git_dir, const char *real_git_dir,
* is an attempt to reinitialize new repository with an old tool.
*/
check_repository_format(&repo_fmt);
-+ repo_fmt.ref_storage = xstrdup(ref_storage_format);
++ repo_fmt.ref_storage = xstrdup(ref_storage_format);
validate_hash_algorithm(&repo_fmt, hash);
@@ builtin/init-db.c: int cmd_init_db(int argc, const char **argv, const char *pref
flags |= INIT_DB_EXIST_OK;
- return init_db(git_dir, real_git_dir, template_dir, hash_algo, flags);
-+ return init_db(git_dir, real_git_dir, template_dir, hash_algo, ref_storage_format, flags);
++ return init_db(git_dir, real_git_dir, template_dir, hash_algo,
++ ref_storage_format, flags);
}
## cache.h ##
@@ cache.h: int path_inside_repo(const char *prefix, const char *path);
int init_db(const char *git_dir, const char *real_git_dir,
const char *template_dir, int hash_algo,
-+ const char *ref_storage_format,
- unsigned int flags);
+- unsigned int flags);
-void initialize_repository_version(int hash_algo);
-+void initialize_repository_version(int hash_algo, const char *ref_storage_format);
++ const char *ref_storage_format, unsigned int flags);
++void initialize_repository_version(int hash_algo,
++ const char *ref_storage_format);
void sanitize_stdfds(void);
int daemonize(void);
@@ cache.h: struct repository_format {
## refs.c ##
@@
+ #include "argv-array.h"
+ #include "repository.h"
+
++const char *default_ref_storage(void)
++{
++ const char *test = getenv("GIT_TEST_REFTABLE");
++ return test ? "reftable" : "files";
++}
++
/*
* List of all available backends
*/
@@ refs.c: struct ref_store *get_main_ref_store(struct repository *r)
if (!r->gitdir)
BUG("attempting to get main_ref_store outside of repository");
-- r->refs = ref_store_init(r->gitdir, REF_STORE_ALL_CAPS);
-+ r->refs = ref_store_init(r->gitdir,
-+ r->ref_storage_format ? r->ref_storage_format :
-+ DEFAULT_REF_STORAGE,
-+ REF_STORE_ALL_CAPS);
- return r->refs;
+- r->refs_private = ref_store_init(r->gitdir, REF_STORE_ALL_CAPS);
++ r->refs_private = ref_store_init(
++ r->gitdir,
++ r->ref_storage_format ? r->ref_storage_format : DEFAULT_REF_STORAGE,
++ REF_STORE_ALL_CAPS);
+ return r->refs_private;
}
@@ refs.c: struct ref_store *get_submodule_ref_store(const char *submodule)
@@ refs.h: struct string_list;
struct string_list_item;
struct worktree;
-+#define DEFAULT_REF_STORAGE "files"
++/* Returns the ref storage backend to use by default. */
++const char *default_ref_storage(void);
+
/*
* Resolve a reference, recursively following symbolic refererences.
@@ refs/reftable-backend.c (new)
+ unsigned int store_flags;
+
+ int err;
++ char *repo_dir;
+ char *reftable_dir;
+ struct reftable_stack *stack;
+};
@@ refs/reftable-backend.c (new)
+
+ base_ref_store_init(ref_store, &refs_be_reftable);
+ refs->store_flags = store_flags;
-+
++ refs->repo_dir = xstrdup(path);
+ strbuf_addf(&sb, "%s/reftable", path);
+ refs->reftable_dir = xstrdup(sb.buf);
+ strbuf_reset(&sb);
+
-+ strbuf_addf(&sb, "%s/refs", path);
-+ safe_create_dir(sb.buf, 1);
-+ strbuf_reset(&sb);
-+
-+ strbuf_addf(&sb, "%s/HEAD", path);
-+ write_file(sb.buf, "ref: refs/.invalid");
-+ strbuf_reset(&sb);
-+
-+ strbuf_addf(&sb, "%s/refs/heads", path);
-+ write_file(sb.buf, "this repository uses the reftable format");
-+
+ refs->err = reftable_new_stack(&refs->stack, refs->reftable_dir, cfg);
+ strbuf_release(&sb);
+ return ref_store;
@@ refs/reftable-backend.c (new)
+{
+ struct git_reftable_ref_store *refs =
+ (struct git_reftable_ref_store *)ref_store;
++ struct strbuf sb = STRBUF_INIT;
++
+ safe_create_dir(refs->reftable_dir, 1);
-+ return 0;
++
++ strbuf_addf(&sb, "%s/HEAD", refs->repo_dir);
++ write_file(sb.buf, "ref: refs/.invalid");
++ strbuf_reset(&sb);
++
++ strbuf_addf(&sb, "%s/refs", refs->repo_dir);
++ safe_create_dir(sb.buf, 1);
++ strbuf_reset(&sb);
++
++ strbuf_addf(&sb, "%s/refs/heads", refs->repo_dir);
++ write_file(sb.buf, "this repository uses the reftable format");
++
++ return 0;
+}
+
+struct git_reftable_iterator {
@@ refs/reftable-backend.c (new)
+ }
+
+ base_ref_iterator_init(&ri->base, &reftable_ref_iterator_vtable, 1);
-+ ri->prefix = prefix;
++ ri->prefix = prefix;
+ ri->base.oid = &ri->oid;
+ ri->flags = flags;
+ ri->ref_store = ref_store;
@@ refs/reftable-backend.c (new)
+ const char *resolved = refs_resolve_ref_unsafe(
+ refs, refname, RESOLVE_REF_READING, &out_oid, &out_flags);
+ if (is_null_oid(want_oid) != (resolved == NULL)) {
-+ return LOCK_ERROR;
++ return REFTABLE_LOCK_ERROR;
+ }
+
+ if (resolved != NULL && !oideq(&out_oid, want_oid)) {
-+ return LOCK_ERROR;
++ return REFTABLE_LOCK_ERROR;
+ }
+
+ return 0;
@@ refs/reftable-backend.c (new)
+ (struct git_reftable_ref_store *)transaction->ref_store;
+ uint64_t ts = reftable_stack_next_update_index(refs->stack);
+ int err = 0;
++ int i = 0;
+ struct reftable_log_record *logs =
+ calloc(transaction->nr, sizeof(*logs));
+ struct ref_update **sorted =
@@ refs/reftable-backend.c (new)
+ QSORT(sorted, transaction->nr, ref_update_cmp);
+ reftable_writer_set_limits(writer, ts, ts);
+
-+ for (int i = 0; i < transaction->nr; i++) {
++ for (i = 0; i < transaction->nr; i++) {
+ struct ref_update *u = sorted[i];
+ if (u->flags & REF_HAVE_OLD) {
+ err = reftable_check_old_oid(transaction->ref_store,
@@ refs/reftable-backend.c (new)
+ }
+ }
+
-+ for (int i = 0; i < transaction->nr; i++) {
++ for (i = 0; i < transaction->nr; i++) {
+ struct ref_update *u = sorted[i];
+ struct reftable_log_record *log = &logs[i];
+ fill_reftable_log_record(log);
@@ refs/reftable-backend.c (new)
+ transaction->ref_store, u->refname, 0, &out_oid,
+ &out_flags);
+ struct reftable_ref_record ref = { NULL };
++ struct object_id peeled;
++ int peel_error = peel_object(&u->new_oid, &peeled);
++
+ ref.ref_name =
+ (char *)(resolved ? resolved : u->refname);
+ log->ref_name = ref.ref_name;
-+ ref.value = u->new_oid.hash;
++
++ if (!is_null_oid(&u->new_oid)) {
++ ref.value = u->new_oid.hash;
++ }
+ ref.update_index = ts;
++ if (!peel_error) {
++ ref.target_value = peeled.hash;
++ }
++
+ err = reftable_writer_add_ref(writer, &ref);
+ if (err < 0) {
+ goto exit;
@@ refs/reftable-backend.c (new)
+ }
+ }
+
-+ for (int i = 0; i < transaction->nr; i++) {
++ for (i = 0; i < transaction->nr; i++) {
+ err = reftable_writer_add_log(writer, &logs[i]);
+ clear_reftable_log_record(&logs[i]);
+ if (err < 0) {
@@ refs/reftable-backend.c (new)
+ err = reftable_stack_add(refs->stack, &write_transaction_table,
+ transaction);
+ if (err < 0) {
-+ strbuf_addf(errmsg, "reftable: transaction failure %s",
++ strbuf_addf(errmsg, "reftable: transaction failure: %s",
+ reftable_error_str(err));
+ return -1;
+ }
@@ refs/reftable-backend.c (new)
+ (struct write_delete_refs_arg *)argv;
+ uint64_t ts = reftable_stack_next_update_index(arg->stack);
+ int err = 0;
++ int i = 0;
+
+ reftable_writer_set_limits(writer, ts, ts);
-+ for (int i = 0; i < arg->refnames->nr; i++) {
++ for (i = 0; i < arg->refnames->nr; i++) {
+ struct reftable_ref_record ref = {
+ .ref_name = (char *)arg->refnames->items[i].string,
+ .update_index = ts,
@@ refs/reftable-backend.c (new)
+ }
+ }
+
-+ for (int i = 0; i < arg->refnames->nr; i++) {
++ for (i = 0; i < arg->refnames->nr; i++) {
+ struct reftable_log_record log = { NULL };
+ struct reftable_ref_record current = { NULL };
+ fill_reftable_log_record(&log);
@@ refs/reftable-backend.c (new)
+ return ITER_ERROR;
+ }
+
-+ if (reftable_log_record_is_deletion(&ri->log)) {
-+ /* XXX - Why does the reftable_stack filter these? */
-+ continue;
-+ }
+ ri->base.refname = ri->log.ref_name;
+ if (ri->last_name != NULL &&
+ !strcmp(ri->log.ref_name, ri->last_name)) {
-+ /* we want the refnames that we have reflogs for, so we
-+ * skip if we've already produced this name. This could
-+ * be faster by seeking directly to
-+ * reflog@update_index==0.
-+ */
++ /* we want the refnames that we have reflogs for, so we
++ * skip if we've already produced this name. This could
++ * be faster by seeking directly to
++ * reflog@update_index==0.
++ */
+ continue;
+ }
+
@@ refs/reftable-backend.c (new)
+ int cap = 0;
+ int len = 0;
+ int err = 0;
++ int i = 0;
+
+ if (refs->err < 0) {
+ return refs->err;
@@ refs/reftable-backend.c (new)
+ logs[len++] = log;
+ }
+
-+ for (int i = len; i--;) {
++ for (i = len; i--;) {
+ struct reftable_log_record *log = &logs[i];
+ struct object_id old_oid;
+ struct object_id new_oid;
@@ refs/reftable-backend.c (new)
+ }
+ }
+
-+ for (int i = 0; i < len; i++) {
++ for (i = 0; i < len; i++) {
+ reftable_log_record_clear(&logs[i]);
+ }
+ free(logs);
@@ refs/reftable-backend.c (new)
+ }
+
+ err = reftable_stack_read_ref(refs->stack, refname, &ref);
-+ if (err) {
++ if (err > 0) {
++ errno = ENOENT;
++ err = -1;
++ goto exit;
++ }
++ if (err < 0) {
++ errno = reftable_error_to_errno(err);
++ err = -1;
+ goto exit;
+ }
+ if (ref.target != NULL) {
@@ refs/reftable-backend.c (new)
+ strbuf_reset(referent);
+ strbuf_addstr(referent, ref.target);
+ *type |= REF_ISSYMREF;
-+ } else {
++ } else if (ref.value != NULL) {
+ hashcpy(oid->hash, ref.value);
-+ }
++ } else {
++ *type |= REF_ISBROKEN;
++ errno = EINVAL;
++ err = -1;
++ }
+exit:
+ reftable_ref_record_clear(&ref);
+ return err;
@@ refs/reftable-backend.c (new)
+ reftable_reflog_expire
+};
+ ## reftable/update.sh ##
+@@ reftable/update.sh: SRC=${SRC:-origin} BRANCH=${BRANCH:-origin/master}
+ cp reftable-repo/c/*.[ch] reftable/
+ cp reftable-repo/c/include/*.[ch] reftable/
+ cp reftable-repo/LICENSE reftable/
+-git --git-dir reftable-repo/.git show --no-patch origin/master \
++git --git-dir reftable-repo/.git show --no-patch ${BRANCH} \
+ > reftable/VERSION
+
+ mv reftable/system.h reftable/system.h~
+
## repository.c ##
@@ repository.c: int repo_init(struct repository *repo,
if (worktree)
@@ repository.c: int repo_init(struct repository *repo,
## repository.h ##
@@ repository.h: struct repository {
- /* The store in which the refs are held. */
- struct ref_store *refs;
+ */
+ struct ref_store *refs_private;
+ /* The format to use for the ref database. */
+ char *ref_storage_format;
@@ t/t0031-reftable.sh (new)
+
+. ./test-lib.sh
+
-+# XXX - fix GC
-+test_expect_success 'basic operation of reftable storage' '
++INVALID_SHA1=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
++
++initialize () {
+ rm -rf .git &&
+ git init --ref-storage=reftable &&
-+ mv .git/hooks .git/hooks-disabled &&
++ mv .git/hooks .git/hooks-disabled
++}
++
++test_expect_success 'delete ref' '
++ initialize &&
++ test_commit file &&
++ SHA=$(git show-ref -s --verify HEAD) &&
++ test_write_lines "$SHA refs/heads/master" "$SHA refs/tags/file" >expect &&
++ git show-ref > actual &&
++ ! git update-ref -d refs/tags/file $INVALID_SHA1 &&
++ test_cmp expect actual &&
++ git update-ref -d refs/tags/file $SHA &&
++ test_write_lines "$SHA refs/heads/master" >expect &&
++ git show-ref > actual &&
++ test_cmp expect actual
++'
++
++test_expect_success 'basic operation of reftable storage: commit, reflog, repack' '
++ initialize &&
+ test_commit file &&
+ test_write_lines refs/heads/master refs/tags/file >expect &&
+ git show-ref &&
@@ t/t0031-reftable.sh (new)
+ for count in $(test_seq 1 10)
+ do
+ test_commit "number $count" file.t $count number-$count ||
-+ return 1
++ return 1
+ done &&
-+(true || (test_pause &&
-+ git gc &&
++ git pack-refs &&
+ ls -1 .git/reftable >table-files &&
+ test_line_count = 2 table-files &&
-+ git reflog refs/heads/master >output &&
++ git reflog refs/heads/master >output &&
+ test_line_count = 11 output &&
-+ grep "commit (initial): first post" output &&
-+ grep "commit: number 10" output ))
++ grep "commit (initial): file" output &&
++ grep "commit: number 10" output &&
++ git gc &&
++ git reflog refs/heads/master >output &&
++ test_line_count = 0 output
++'
++
++# This matches show-ref's output
++print_ref() {
++ echo "$(git rev-parse "$1") $1"
++}
++
++test_expect_success 'peeled tags are stored' '
++ initialize &&
++ test_commit file &&
++ git tag -m "annotated tag" test_tag HEAD &&
++ {
++ print_ref "refs/heads/master" &&
++ print_ref "refs/tags/file" &&
++ print_ref "refs/tags/test_tag" &&
++ print_ref "refs/tags/test_tag^{}"
++ } >expect &&
++ git show-ref -d >actual &&
++ test_cmp expect actual
++'
++
++test_expect_success 'show-ref works on fresh repo' '
++ initialize &&
++ rm -rf .git &&
++ git init --ref-storage=reftable &&
++ >expect &&
++ ! git show-ref > actual &&
++ test_cmp expect actual
++'
++
++test_expect_success 'checkout unborn branch' '
++ initialize &&
++ git checkout -b master
++'
++
++test_expect_success 'do not clobber existing repo' '
++ rm -rf .git &&
++ git init --ref-storage=files &&
++ cat .git/HEAD > expect &&
++ test_commit file &&
++ (git init --ref-storage=reftable || true) &&
++ cat .git/HEAD > actual &&
++ test_cmp expect actual
+'
+
+test_done
-: ----------- > 11: 8af67b85e49 Add some reftable testing infrastructure
-: ----------- > 12: b02b83a92ad t: use update-ref and show-ref to reading/writing refsSignature · 1 line
gitgitgadget
PATCH v10 1/12 refs.h: clarify reflog iteration order
From: Han-Wen Nienhuys <[email protected]>
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- refs.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
refs.h +4 -1
@@ -444,18 +444,21 @@ int delete_refs(const char *msg, struct string_list *refnames,int refs_delete_reflog (struct ref_store *refs,const char *refname);int delete_reflog (const char *refname); -/* iterate over reflog entries */ +/* Iterate over reflog entries. */ typedef int each_reflog_ent_fn (struct object_id *old_oid,struct object_id *new_oid,const char *committer,timestamp_t timestamp,int tz,const char *msg,void *cb_data); +/* Iterate in over reflog entries, oldest entry first. */ int refs_for_each_reflog_ent (struct ref_store *refs,const char *refname,each_reflog_ent_fn fn,void *cb_data);int refs_for_each_reflog_ent_reverse (struct ref_store *refs,const char *refname,each_reflog_ent_fn fn,void *cb_data); + +/* Call a function for each reflog entry, oldest entry first. */ int for_each_reflog_ent (const char *refname,each_reflog_ent_fn fn,void *cb_data);int for_each_reflog_ent_reverse (const char *refname,each_reflog_ent_fn fn,void *cb_data);
Signature · 1 line
gitgitgadget
PATCH v10 2/12 Iterate over the "refs/" namespace in for_each_[raw]ref
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This happens implicitly in the files/packed ref backend; making it explicit simplifies adding alternate ref storage backends, such as reftable.
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- refs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
refs.c +3 -3
@@ -1569,7 +1569,7 @@ static int do_for_each_ref(struct ref_store *refs, const char *prefix,int refs_for_each_ref (struct ref_store *refs,each_ref_fn fn,void *cb_data) { -return do_for_each_ref (refs,"" , fn,0 ,0 , cb_data); +return do_for_each_ref (refs,"refs/" , fn,0 ,0 , cb_data); }int for_each_ref (each_ref_fn fn,void *cb_data)
@@ -1629,8 +1629,8 @@ int for_each_namespaced_ref(each_ref_fn fn, void *cb_data)int refs_for_each_rawref (struct ref_store *refs,each_ref_fn fn,void *cb_data) { -return do_for_each_ref (refs,"" , fn,0 , -DO_FOR_EACH_INCLUDE_BROKEN , cb_data); +return do_for_each_ref (refs,"refs/" , fn,0 ,DO_FOR_EACH_INCLUDE_BROKEN , + cb_data); }int for_each_rawref (each_ref_fn fn,void *cb_data)
Signature · 1 line
gitgitgadget
Re: [PATCH v10 02/12] Iterate over the "refs/" namespace in for_each_[raw]ref
Quoted 5 lines · Han-Wen Nienhuys via GitGitGadget
This happens implicitly in the files/packed ref backend; making it explicit simplifies adding alternate ref storage backends, such as reftable.
As an outsider to this part of the codebase, a little more explanation could be handy in the commit message. I found the backends you mentioned in refs, and it seems like they're the only two, but it's not obvious how this delta is related to those backends. Furthermore, grepping looks like this function whose behavior is changing is being called from somewhere else, with no change to that function (and it looks like the callsite's callback doesn't check whether a ref begins with refs/ or not).
All this to say - it's hard to convince myself this is a safe change, and I'd really like to read more to understand why you made it.
Quoted 33 lines
Signed-off-by Han-Wen Nienhuys <[email protected]>--- refs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)refs.c +3
-3@@ -1569,7 +1569,7 @@ static int do_for_each_ref(struct ref_store *refs, const char *prefix,int refs_for_each_ref (struct ref_store *refs,each_ref_fn fn,void *cb_data) { -return do_for_each_ref (refs,"" , fn,0 ,0 , cb_data); +return do_for_each_ref (refs,"refs/" , fn,0 ,0 , cb_data); }int for_each_ref (each_ref_fn fn,void *cb_data)@@ -1629,8 +1629,8 @@ int for_each_namespaced_ref(each_ref_fn fn, void *cb_data)int refs_for_each_rawref (struct ref_store *refs,each_ref_fn fn,void *cb_data) { -return do_for_each_ref (refs,"" , fn,0 , -DO_FOR_EACH_INCLUDE_BROKEN , cb_data); +return do_for_each_ref (refs,"refs/" , fn,0 ,DO_FOR_EACH_INCLUDE_BROKEN , + cb_data); }int for_each_rawref (each_ref_fn fn,void *cb_data)Signature · 1 line
gitgitgadget
Re: [PATCH v10 02/12] Iterate over the "refs/" namespace in for_each_[raw]ref
Quoted 17 lines · Emily Shaffer
Quoted 5 lines · Han-Wen Nienhuys via GitGitGadget
This happens implicitly in the files/packed ref backend; making it explicit simplifies adding alternate ref storage backends, such as reftable.
As an outsider to this part of the codebase, a little more explanation could be handy in the commit message. I found the backends you mentioned in refs, and it seems like they're the only two, but it's not obvious how this delta is related to those backends. Furthermore, grepping looks like this function whose behavior is changing is being called from somewhere else, with no change to that function (and it looks like the callsite's callback doesn't check whether a ref begins with refs/ or not).
Quoted 2 lines
All this to say - it's hard to convince myself this is a safe change, and I'd really like to read more to understand why you made it.
I'll be the first to admit that I'm on shaky ground here. However, given that the test suite passes, if this is breaking some behavior, it's probably not very well tested behavior.
Here is what I know:
Git stores refs in multiple places:
- normal refs, packed: .git/packed-refs
- normal refs, loose: under refs/*/
- special refs: HEAD, ORIG_HEAD, REBASE_HEAD.
Currently, the special refs can only be read with refs_read_raw_ref(). If you iterate over the refs in the files/packed backend, you can never find HEAD, ORIG_HEAD etc.
Reftable does not have different classes of ref storage, so this means that the whole space of refs (including HEAD) is managed by the reftable backend, and if you iterate over the refs, reftable will also produce HEAD. This is not spelled out in the spec, but JGit actually stores HEAD in reftable too, and we want to be interoperable.
Without this patch, commands like "git show-ref" will produce an entry "HEAD", which is a regression.
With this patch, the default iteration is limited to the "refs/" prefix, so we don't produce HEAD in the reftable backend by default.
Now, I have several questions:
* how does this interact with worktrees? It seems that there is a special worktree namespace?
* if you are doing a rebase, and have unreachable objects in ORIG_HEAD, REBASE_HEAD, how does git-gc ensure that these objects stay alive? I'd think that you need to iterate over all entries (including REBASE_HEAD and friends), but I haven't been able to understand how that works.
Signature · 11 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Pseudo ref handling (was Re: [PATCH v10 02/12] Iterate over the "refs/" namespace in for_each_[raw]ref)
Quoted 10 lines · Han-Wen Nienhuys
Quoted 3 lines
All this to say - it's hard to convince myself this is a safe change, and I'd really like to read more to understand why you made it.
I'll be the first to admit that I'm on shaky ground here. However, given that the test suite passes, if this is breaking some behavior, it's probably not very well tested behavior.
Here is what I know:
Quoted 8 lines
Git stores refs in multiple places:
- normal refs, packed: .git/packed-refs
- normal refs, loose: under refs/*/
- special refs: HEAD,ORIG_HEAD,REBASE_HEAD.Currently, the special refs can only be read with
refs_read_raw_ref(). If you iterate over the refs in the files/packed backend, you can never find HEAD,ORIG_HEADetc.
A large source of test failures of the reftable patch series is the handling of pseudo refs (HEAD, CHERRY_PICK_HEAD etc.)
These are written as
if (ref_type(refname) == REF_TYPE_PSEUDOREF) {
assert(refs == get_main_ref_store(the_repository));
ret = write_pseudoref(refname, new_oid, old_oid, &err);
} else {
t = ref_store_transaction_begin(refs, &err);
..write_pseudoref writes a file directly to the .git/ directory.
However, on read (for example, CHERRY_PICK_HEAD), the pseudo ref is given to lookup_commit_reference_by_name, which in the end calls refs_resolve_ref_unsafe(). If reftable is active, the pseudo ref isn't read from .git/CHERRY_PICK_HEAD.
If the pseudo refs are read through a ref backend, shouldn't they be written through a ref backend too?
This causes all rebase and all cherry-pick tests to fail with reftable.
Signature · 7 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
PATCH v10 3/12 create .git/refs in files-backend.c
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This prepares for supporting the reftable format, which will want create its own file system layout in .git
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- builtin/init-db.c | 2 -- refs/files-backend.c | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-)
builtin/init-db.c +0 -2
@@ -251,8 +251,6 @@ static int create_default_files(const char *template_path,* We need to create a "refs" dir in any case so that older * versions of git can tell that this is a repository. */ -safe_create_dir (git_path ("refs" ),1 ); -adjust_shared_perm (git_path ("refs" ));if (refs_init_db (&err))die ("failed to set up refs db: %s" , err.buf);
refs/files-backend.c +6 -0
@@ -3157,9 +3157,15 @@ static int files_init_db(struct ref_store *ref_store, struct strbuf *err)files_downcast (ref_store,REF_STORE_WRITE ,"init_db" );struct strbuf sb =STRBUF_INIT ; +files_ref_path (refs, &sb,"refs" ); +safe_create_dir (sb.buf,1 ); +/* adjust permissions even if directory already exists. */ +adjust_shared_perm (sb.buf); +/* * Create .git/refs/{heads,tags} */ +strbuf_reset (&sb);files_ref_path (refs, &sb,"refs/heads" );safe_create_dir (sb.buf,1 );
Signature · 1 line
gitgitgadget
Re: [PATCH v10 03/12] create .git/refs in files-backend.c
Quoted 16 lines · Han-Wen Nienhuys via GitGitGadget
This prepares for supporting the reftable format, which will want create its own file system layout in .git
Signed-off-by Han-Wen Nienhuys <[email protected]>--- builtin/init-db.c | 2 -- refs/files-backend.c | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-)builtin/init-db.c +0
-0
builtin/init-db.c static int create_default_files(const char *template_path, · line 255
Quoted 5 lines
builtin/init-db.c +0
-2@@ -251,8 +251,6 @@ static int create_default_files(const char *template_path,* We need to create a "refs" dir in any case so that older * versions of git can tell that this is a repository. */ -safe_create_dir (git_path ("refs" ),1 ); -adjust_shared_perm (git_path ("refs" ));
Is the reftable completely replacing the refs/ dir? Or is the idea that the refs/ dir is only used by the files backend? The commit message makes it sound like it's an additional format to support, so I'm a little confused. Why does the other currently-existing backend not need the refs/ dir at this stage?
Quoted 26 lines
builtin/init-db.c +0
-0if (refs_init_db (&err))die ("failed to set up refs db: %s" , err.buf);refs/files-backend.c +6
-0@@ -3157,9 +3157,15 @@ static int files_init_db(struct ref_store *ref_store, struct strbuf *err)files_downcast (ref_store,REF_STORE_WRITE ,"init_db" );struct strbuf sb =STRBUF_INIT ; +files_ref_path (refs, &sb,"refs" ); +safe_create_dir (sb.buf,1 ); +/* adjust permissions even if directory already exists. */ +adjust_shared_perm (sb.buf); +/* * Create .git/refs/{heads,tags} */ +strbuf_reset (&sb);files_ref_path (refs, &sb,"refs/heads" );safe_create_dir (sb.buf,1 );Signature · 1 line
gitgitgadget
Re: [PATCH v10 03/12] create .git/refs in files-backend.c
Quoted 8 lines · Emily Shaffer
builtin/init-db.c static int create_default_files(const char *template_path, · line 255
Quoted 2 lines
builtin/init-db.c +0
-2-safe_create_dir (git_path ("refs" ),1 ); -adjust_shared_perm (git_path ("refs" ));Is the reftable completely replacing the refs/ dir? Or is the idea that the refs/ dir is only used by the files backend? The commit message makes it sound like it's an additional format to support, so I'm a little confused. Why does the other currently-existing backend not need the refs/ dir at this stage?
Other current backend being the files-backend, which creates it below, isn't this a no-op at this stage?
What's more interesting is what would happen when a repo is initialized with the reftable backend as its sole backend.
Even then, the repository discovery code of existing versions of Git needs to see a directory with a ".git" subdirectory, in the latter of which there must be HEAD and objects and refs subdirectories, so we'd need to create refs/ directory even if reftable does not use it.
This step, from that point of view, may not be necessary. But as long as we make sure any repository creates refs/ and objects/ directories, regardless of the ref backend, we'd be OK.
Re: [PATCH v10 03/12] create .git/refs in files-backend.c
In reply to Emily ShafferQuoted 13 lines · Emily Shaffer
Quoted 11 lines · Han-Wen Nienhuys via GitGitGadget
This prepares for supporting the reftable format, which will want create its own file system layout in .git
Signed-off-by Han-Wen Nienhuys <[email protected]>--- builtin/init-db.c | 2 -- refs/files-backend.c | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-)
Quoted 16 lines
builtin/init-db.c static int create_default_files(const char *template_path, · line 255
Quoted 10 lines
builtin/init-db.c +0
-2@@ -251,8 +251,6 @@ static int create_default_files(const char *template_path,* We need to create a "refs" dir in any case so that older * versions of git can tell that this is a repository. */ -safe_create_dir (git_path ("refs" ),1 ); -adjust_shared_perm (git_path ("refs" ));Is the reftable completely replacing the refs/ dir? Or is the idea that the refs/ dir is only used by the files backend? The commit message makes it sound like it's an additional format to support, so I'm a little confused. Why does the other currently-existing backend not need the refs/ dir at this stage?
I've dropped this patch from the series; it probably was from before the compat hack that is described in
https://git.eclipse.org/r/#/c/157167/
for further clarification: reftable doesn't used the refs/ dir at all, but we still want to not confuse older git versions.
Signature · 11 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
PATCH v10 4/12 refs: document how ref_iterator_advance_fn should handle symrefs
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- refs/refs-internal.h | 5 +++++ 1 file changed, 5 insertions(+)
refs/refs-internal.h +5 -0
@@ -438,6 +438,11 @@ void base_ref_iterator_free(struct ref_iterator *iter);/* Virtual function declarations for ref_iterators: */ +/* +* backend-specific implementation of ref_iterator_advance. +* For symrefs, the function should set REF_ISSYMREF, and it should also +* dereference the symref to provide the OID referent. +*/ typedef int ref_iterator_advance_fn (struct ref_iterator *ref_iterator); typedef int ref_iterator_peel_fn(struct ref_iterator *ref_iterator,
Signature · 1 line
gitgitgadget
PATCH v10 5/12 Add .gitattributes for the reftable/ directory
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- reftable/.gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 reftable/.gitattributes
reftable/.gitattributes new file +1 -0
@@ -0,0 +1 @@ +/zlib-compat.c whitespace=-indent-with-non-tab,-trailing-space
Signature · 1 line
gitgitgadget
PATCH v10 6/12 reftable: file format documentation
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Jonathan Nieder <[email protected]>
Shawn Pearce explains:
Some repositories contain a lot of references (e.g. android at 866k, rails at 31k). The reftable format provides:
- Near constant time lookup for any single reference, even when the repository is cold and not in process or kernel cache.
- Near constant time verification a SHA-1 is referred to by at least one reference (for allow-tip-sha1-in-want).
- Efficient lookup of an entire namespace, such as refs/tags/.
- Support atomic push O(size_of_update) operations.
- Combine reflog storage with ref storage.
This file format spec was originally written in July, 2017 by Shawn Pearce. Some refinements since then were made by Shawn and by Han-Wen Nienhuys based on experiences implementing and experimenting with the format. (All of this was in the context of our work at Google and Google is happy to contribute the result to the Git project.)
Imported from JGit[1]'s current version (c217d33ff, "Documentation/technical/reftable: improve repo layout", 2020-02-04) of Documentation/technical/reftable.md and converted to asciidoc by running
pandoc -t asciidoc -f markdown reftable.md >reftable.txt
using pandoc 2.2.1. The result required the following additional minor changes:
- removed the [TOC] directive to add a table of contents, since asciidoc does not support it
- replaced git-scm.com/docs links with linkgit: directives that link to other pages within Git's documentation
[1] https://eclipse.googlesource.com/jgit/jgit
Signed-off-by Jonathan Nieder <[email protected]>
--- Documentation/Makefile | 1 + Documentation/technical/reftable.txt | 1067 ++++++++++++++++++++++++++ 2 files changed, 1068 insertions(+) create mode 100644 Documentation/technical/reftable.txt
Documentation/Makefile +1 -0
@@ -93,6 +93,7 @@ TECH_DOCS += technical/protocol-capabilities TECH_DOCS += technical/protocol-common TECH_DOCS += technical/protocol-v2 TECH_DOCS += technical/racy-git +TECH_DOCS += technical/reftable TECH_DOCS += technical/send-pack-pipeline TECH_DOCS += technical/shallow TECH_DOCS += technical/signature-format
Documentation/technical/reftable.txt new file +1067 -0
1,067 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v10 7/12 reftable: define version 2 of the spec to accomodate SHA256
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- Documentation/technical/reftable.txt | 50 ++++++++++++++++------------ 1 file changed, 28 insertions(+), 22 deletions(-)
Documentation/technical/reftable.txt +28 -22
@@ -193,8 +193,8 @@ and non-aligned files. Very small files (e.g. 1 only ref block) may omit `padding` and the ref index to reduce total file size. -Header -^^^^^^ +Header (version 1) +^^^^^^^^^^^^^^^^^^ A 24-byte header appears at the beginning of the file:
@@ -215,6 +215,24 @@ used in a stack for link:#Update-transactions[transactions], these fields can order the files such that the prior file’s `max_update_index + 1` is the next file’s `min_update_index`. +Header (version 2) +^^^^^^^^^^^^^^^^^^ + +A 28-byte header appears at the beginning of the file: + +.... +'REFT' +uint8( version_number = 1 ) +uint24( block_size ) +uint64( min_update_index ) +uint64( max_update_index ) +uint32( hash_id ) +.... + +The header is identical to `version_number=1`, with the 4-byte hash ID +("sha1" for SHA1 and "s256" for SHA-256) append to the header. + + First ref block ^^^^^^^^^^^^^^^
@@ -671,14 +689,8 @@ Footer After the last block of the file, a file footer is written. It begins like the file header, but is extended with additional data. -A 68-byte footer appears at the end: - .... - 'REFT' - uint8( version_number = 1 ) - uint24( block_size ) - uint64( min_update_index ) - uint64( max_update_index ) + HEADER uint64( ref_index_position ) uint64( (obj_position << 5) | obj_id_len )
@@ -701,12 +713,16 @@ obj blocks. * `obj_index_position`: byte position for the start of the obj index. * `log_index_position`: byte position for the start of the log index. +The size of the footer is 68 bytes for version 1, and 72 bytes for +version 2. + Reading the footer ++++++++++++++++++ -Readers must seek to `file_length - 68` to access the footer. A trusted -external source (such as `stat(2)`) is necessary to obtain -`file_length`. When reading the footer, readers must verify: +Readers must first read the file start to determine the version +number. Then they seek to `file_length - FOOTER_LENGTH` to access the +footer. A trusted external source (such as `stat(2)`) is necessary to +obtain `file_length`. When reading the footer, readers must verify: * 4-byte magic is correct * 1-byte version number is recognized
@@ -1055,13 +1071,3 @@ impossible. A common format that can be supported by all major Git implementations (git-core, JGit, libgit2) is strongly preferred. - -Future -~~~~~~ - -Longer hashes -^^^^^^^^^^^^^ - -Version will bump (e.g. 2) to indicate `value` uses a different object -id length other than 20. The length could be stored in an expanded file -header, or hardcoded as part of the version.
Signature · 1 line
gitgitgadget
PATCH v10 8/12 reftable: clarify how empty tables should be written
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The format allows for some ambiguity, as a lone footer also starts with a valid file header. However, the current JGit code will barf on this. This commit codifies this behavior into the standard.
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- Documentation/technical/reftable.txt | 7 +++++++ 1 file changed, 7 insertions(+)
Documentation/technical/reftable.txt +7 -0
@@ -731,6 +731,13 @@ version) Once verified, the other fields of the footer can be accessed. +Empty tables +++++++++++++ + +A reftable may be empty. In this case, the file starts with a header +and is immediately followed by a footer. + + Varint encoding ^^^^^^^^^^^^^^^
Signature · 1 line
gitgitgadget
PATCH v10 9/12 Add reftable library
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Reftable is a format for storing the ref database. Its rationale and specification is in the preceding commit.
This imports the upstream library as one big commit. For understanding the code, it is suggested to read the code in the following order:
* The specification under Documentation/technical/reftable.txt
* reftable.h - the public API
* record.{c,h} - reading and writing records
* block.{c,h} - reading and writing blocks.
* writer.{c,h} - writing a complete reftable file.
* merged.{c,h} and pq.{c,h} - reading a stack of reftables
* stack.{c,h} - writing and compacting stacks of reftable on the filesystem.
Signed-off-by Han-Wen Nienhuys <[email protected]>
--- reftable/LICENSE | 31 ++ reftable/README.md | 11 + reftable/VERSION | 5 + reftable/basics.c | 209 ++++++++ reftable/basics.h | 53 ++ reftable/block.c | 425 +++++++++++++++ reftable/block.h | 124 +++++ reftable/blocksource.h | 22 + reftable/bytes.c | 0 reftable/config.h | 1 + reftable/constants.h | 21 + reftable/dump.c | 97 ++++ reftable/file.c | 98 ++++ reftable/iter.c | 234 ++++++++ reftable/iter.h | 60 +++ reftable/merged.c | 327 ++++++++++++ reftable/merged.h | 36 ++ reftable/pq.c | 115 ++++ reftable/pq.h | 34 ++ reftable/reader.c | 754 ++++++++++++++++++++++++++ reftable/reader.h | 68 +++ reftable/record.c | 1126 +++++++++++++++++++++++++++++++++++++++ reftable/record.h | 121 +++++ reftable/reftable.h | 527 ++++++++++++++++++ reftable/slice.c | 224 ++++++++ reftable/slice.h | 76 +++ reftable/stack.c | 1151 ++++++++++++++++++++++++++++++++++++++++ reftable/stack.h | 44 ++ reftable/system.h | 53 ++ reftable/tree.c | 67 +++ reftable/tree.h | 34 ++ reftable/update.sh | 23 + reftable/writer.c | 661 +++++++++++++++++++++++ reftable/writer.h | 60 +++ reftable/zlib-compat.c | 92 ++++ 35 files changed, 6984 insertions(+) create mode 100644 reftable/LICENSE create mode 100644 reftable/README.md create mode 100644 reftable/VERSION create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/blocksource.h create mode 100644 reftable/bytes.c create mode 100644 reftable/config.h create mode 100644 reftable/constants.h create mode 100644 reftable/dump.c create mode 100644 reftable/file.c create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/reftable.h create mode 100644 reftable/slice.c create mode 100644 reftable/slice.h create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/system.h create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100755 reftable/update.sh create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h create mode 100644 reftable/zlib-compat.c
reftable/LICENSE new file +31 -0
@@ -0,0 +1,31 @@ +BSD License + +Copyright (c) 2020, Google LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of Google LLC nor the names of its contributors may +be used to endorse or promote products derived from this software +without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
reftable/README.md new file +11 -0
@@ -0,0 +1,11 @@ + +The source code in this directory comes from https://github.com/google/reftable. + +The VERSION file keeps track of the current version of the reftable library. + +To update the library, do: + + sh reftable/update.sh + +Bugfixes should be accompanied by a test and applied to upstream project at +https://github.com/google/reftable.
reftable/VERSION new file +5 -0
@@ -0,0 +1,5 @@ +commit a6d6b31bea256044621d789df64a8159647f21bc +Author: Han-Wen Nienhuys <[email protected]> +Date: Mon Apr 27 20:46:21 2020 +0200 + + C: prefix error codes with REFTABLE_
reftable/basics.c new file +209 -0
209 lines not on this page · raw message
reftable/basics.h new file +53 -0
53 lines not on this page · raw message
reftable/block.c new file +425 -0
425 lines not on this page · raw message
reftable/block.h new file +124 -0
124 lines not on this page · raw message
reftable/blocksource.h new file +22 -0
@@ -0,0 +1,22 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef BLOCKSOURCE_H +#define BLOCKSOURCE_H + +#include "reftable.h" + +uint64_t block_source_size (struct reftable_block_source source); +int block_source_read_block (struct reftable_block_source source, +struct reftable_block *dest,uint64_t off, +uint32_t size); +void block_source_return_block (struct reftable_block_source source, +struct reftable_block *ret); +void block_source_close (struct reftable_block_source source); + +#endif
reftable/bytes.c new file +0 -0
reftable/config.h new file +1 -0
@@ -0,0 +1 @@ +/* empty */
reftable/constants.h new file +21 -0
@@ -0,0 +1,21 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef CONSTANTS_H +#define CONSTANTS_H + +#define BLOCK_TYPE_LOG 'g' +#define BLOCK_TYPE_INDEX 'i' +#define BLOCK_TYPE_REF 'r' +#define BLOCK_TYPE_OBJ 'o' +#define BLOCK_TYPE_ANY 0 + +#define MAX_RESTARTS ((1 << 16) - 1) +#define DEFAULT_BLOCK_SIZE 4096 + +#endif
reftable/dump.c new file +97 -0
97 lines not on this page · raw message
reftable/file.c new file +98 -0
98 lines not on this page · raw message
reftable/iter.c new file +234 -0
234 lines not on this page · raw message
reftable/iter.h new file +60 -0
60 lines not on this page · raw message
reftable/merged.c new file +327 -0
327 lines not on this page · raw message
reftable/merged.h new file +36 -0
@@ -0,0 +1,36 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef MERGED_H +#define MERGED_H + +#include "pq.h" +#include "reftable.h" + +struct reftable_merged_table { +struct reftable_reader **stack; +int stack_len; +uint32_t hash_id; +bool suppress_deletions; + +uint64_t min; +uint64_t max; +}; + +struct merged_iter { +struct reftable_iterator *stack; +uint32_t hash_id; +int stack_len; +byte typ; +bool suppress_deletions; +struct merged_iter_pqueue pq; +}; + +void merged_table_clear (struct reftable_merged_table *mt); + +#endif
reftable/pq.c new file +115 -0
115 lines not on this page · raw message
reftable/pq.h new file +34 -0
@@ -0,0 +1,34 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef PQ_H +#define PQ_H + +#include "record.h" + +struct pq_entry { +int index; +struct record rec; +}; + +int pq_less (struct pq_entry a,struct pq_entry b); + +struct merged_iter_pqueue { +struct pq_entry *heap; +int len; +int cap; +}; + +struct pq_entry merged_iter_pqueue_top (struct merged_iter_pqueue pq); +bool merged_iter_pqueue_is_empty (struct merged_iter_pqueue pq); +void merged_iter_pqueue_check (struct merged_iter_pqueue pq); +struct pq_entry merged_iter_pqueue_remove (struct merged_iter_pqueue *pq); +void merged_iter_pqueue_add (struct merged_iter_pqueue *pq,struct pq_entry e); +void merged_iter_pqueue_clear (struct merged_iter_pqueue *pq); + +#endif
reftable/reader.c new file +754 -0
754 lines not on this page · raw message
reftable/reader.h new file +68 -0
68 lines not on this page · raw message
reftable/record.c new file +1126 -0
275 lines not on this page · raw message
reftable/record.h new file +121 -0
121 lines not on this page · raw message
reftable/reftable.h new file +527 -0
527 lines not on this page · raw message
reftable/slice.c new file +224 -0
224 lines not on this page · raw message
reftable/slice.h new file +76 -0
76 lines not on this page · raw message
reftable/stack.c new file +1151 -0
1,151 lines not on this page · raw message
reftable/stack.h new file +44 -0
44 lines not on this page · raw message
reftable/system.h new file +53 -0
53 lines not on this page · raw message
reftable/tree.c new file +67 -0
67 lines not on this page · raw message
reftable/writer.c new file +661 -0
661 lines not on this page · raw message
reftable/writer.h new file +60 -0
60 lines not on this page · raw message
reftable/zlib-compat.c new file +92 -0
92 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
gitgitgadget
Part of the diff is not shown here. View the raw message.
Re: [PATCH v10 09/12] Add reftable library
On 2020-04-27 20:13:35+0000, Han-Wen Nienhuys via GitGitGadget <[email protected]> wrote:
[..snip..]
reftable/LICENSE line 4
Quoted 10 lines
reftable/LICENSE new file +4
-0@@ -0,0 +1,31 @@ +BSD License + +Copyright (c) 2020, Google LLC +All rights reserved.
Linking against GPLv2 code will make this code GPLv2, right?
reftable/README.md line 8
Quoted 11 lines
reftable/README.md new file +8
-0@@ -0,0 +1,11 @@ + +The source code in this directory comes from https://github.com/google/reftable. + +The VERSION file keeps track of the current version of the reftable library. + +To update the library, do: + + sh reftable/update.sh
I agree with Dscho, we should do a git-subtree merge instead.
[..snip..]
reftable/README.md line 11
Quoted 3 lines
reftable/README.md new file +3
-0+#include "system.h" + +void put_be24(byte *out, uint32_t i)
So, we introduce a new type byte?
reftable/README.md
Quoted 4 lines
reftable/README.md new file +4
-0+{ + out[0] = (byte)((i >> 16) & 0xff); + out[1] = (byte)((i >> 8) & 0xff); + out[2] = (byte)((i)&0xff);
At my first glance, I thought that code is:
out[2] = (byte)( (i)(&0xff) )
It's totally un-parsable.
Anyway, we're gonna accept "expand tab to 4 spaces" in Git, now?
reftable/README.md
Quoted 6 lines
reftable/README.md new file +6
-0+} + +uint32_t get_be24(byte *in) +{ + return (uint32_t)(in[0]) << 16 | (uint32_t)(in[1]) << 8 | + (uint32_t)(in[2]);
I don't think we need this much cast. One cast can rule them all.
reftable/README.md
Quoted 6 lines
reftable/README.md new file +6
-0+} + +void put_be16(uint8_t *out, uint16_t i) +{ + out[0] = (uint8_t)((i >> 8) & 0xff); + out[1] = (uint8_t)((i)&0xff);
This is un-parsable, see above.
reftable/README.md
Quoted 3 lines
reftable/README.md new file +3
-0+} + +int binsearch(int sz, int (*f)(int k, void *args), void *args)
binary search?
I think we want all size to be size_t
reftable/README.md
Quoted 7 lines
reftable/README.md new file +7
-0+{ + int lo = 0; + int hi = sz; + + /* invariant: (hi == sz) || f(hi) == true + (lo == 0 && f(0) == true) || fi(lo) == false + */
Comment style
reftable/README.md
Quoted 6 lines
reftable/README.md new file +6
-0+ while (hi - lo > 1) { + int mid = lo + (hi - lo) / 2; + + int val = f(mid, args); + if (val) { + hi = mid;
But this line of code makes me think this function is binary partition instead.
reftable/README.md
Quoted 3 lines
reftable/README.md new file +3
-0+ } else { + lo = mid; + }
bracket style
Quoted 2 lines
reftable/README.md new file +2
-0+ } +
[..snip..]
Quoted 6 lines
reftable/basics.h new file +0
-0@@ -0,0 +1,53 @@
[..snip..]
reftable/basics.h line 28
Quoted 3 lines
reftable/basics.h new file +3
-0+find smallest index i in [0, sz) at which f(i) is true, assuming +that f is ascending. Return sz if f(i) is false for all indices. +*/
So, this is about partitioning, not searching
reftable/basics.h
Quoted 3 lines
reftable/basics.h new file +3
-0+int binsearch (int sz,int (*f)(int k,void *args),void *args); + +int block_writer_add (struct block_writer *w,struct record rec)
So, we're gonna pass a large structure into a function now?
reftable/basics.h
Quoted 12 lines
reftable/basics.h new file +12
-0+{ +struct slice empty = {0 }; +struct slice last = w->entries % w->restart_interval ==0 ? empty : + w->last_key; +struct slice out = { + .buf = w->buf + w->next, + .len = w->block_size - w->next, + }; + +struct slice start = out; + +bool restart = false;
Do we want new type: bool?
Quoted 8 lines
reftable/basics.h new file +8
-0+struct slice key = {0 }; +int n =0 ; + +record_key (rec, &key); + n =encode_key (&restart, out, last, key,record_val_type (rec)); +if (n <0 ) { + goto err; + }
reftable/basics.h
Quoted 10 lines
reftable/basics.h new file +10
-0+slice_consume (&out, n); + + n =record_encode (rec, out, w->hash_size); +if (n <0 ) { + goto err; + } +slice_consume (&out, n); + +if (block_writer_register_restart (w, start.len - out.len, restart, + key) <0 ) {
Eh, naming is hard, and so does this function. It's overly verbose, and long. I haven't go though all of this patch (because it's too long), but I guess some of them can be made static and their name can be shortened.
Quoted 10 lines
reftable/basics.h new file +10
-0+ goto err; + } + +slice_clear (&key); +return 0 ; + +err: +slice_clear (&key); +return -1 ; +}
[..snip..]
reftable/basics.h
Quoted 3 lines
reftable/basics.h new file +3
-0+block_source_return_block (block->source, block); + block->data = uncompressed.buf; + block->len = dst_len;/* XXX: 4 bytes missing? */
Even this is a very lengthy patch, some bytes is missing but we couldn't figured out.
reftable/basics.h
Quoted 13 lines
reftable/basics.h new file +13
-0+ block->source =malloc_block_source (); + full_block_size = src_len + block_header_skip; + }else if (full_block_size ==0 ) { + full_block_size = sz; + }else if (sz < full_block_size && sz < block->len && + block->data[sz] !=0 ) { +/* If the block is smaller than the full block size, it is +padded (data followed by '\0') or the next block is +unaligned. */ + full_block_size = sz; + } + + {
This cute curly braces is new to me
Quoted 28 lines
reftable/basics.h new file +28
-0+uint16_t restart_count =get_be16 (block->data + sz -2 ); +uint32_t restart_start = sz -2 -3 * restart_count; + +byte *restart_bytes = block->data + restart_start; + +/* transfer ownership. */ + br->block = *block; + block->data =NULL ; + block->len =0 ; + + br->hash_size = hash_size; + br->block_len = restart_start; + br->full_block_size = full_block_size; + br->header_off = header_off; + br->restart_count = restart_count; + br->restart_bytes = restart_bytes; + } + +return 0 ; +} + +static uint32_t block_reader_restart_offset (struct block_reader *br,int i) +{ +return get_be24 (br->restart_bytes +3 * i); +} + +void block_reader_start (struct block_reader *br,struct block_iter *it) +{
reftable/basics.h
Quoted 10 lines
reftable/basics.h new file +10
-0+ it->br = br; +slice_resize (&it->last_key,0 ); + it->next_off = br->header_off +4 ; +} + +struct restart_find_args { +int error; +struct slice key; +struct block_reader *r; +};
I would like to see all structure declaration at top of file.
reftable/basics.h
Quoted 2 lines
reftable/basics.h new file +2
-0+ +static int restart_key_less (int idx,void *args)
idx? index? Shouldn't it be size_t?
reftable/basics.h
Quoted 6 lines
reftable/basics.h new file +6
-0+{ +struct restart_find_args *a = (struct restart_find_args *)args; +uint32_t off =block_reader_restart_offset (a->r, idx); +struct slice in = { + .buf = a->r->block.data + off, + .len = a->r->block_len - off,
C99 designated initialisation.
We aren't ready, yet.
reftable/basics.h
Quoted 9 lines
reftable/basics.h new file +9
-0+void block_iter_copy_from (struct block_iter *dest,struct block_iter *src) +{ + dest->br = src->br; + dest->next_off = src->next_off; +slice_copy (&dest->last_key, src->last_key); +} + +/* return < 0 for error, 0 for OK, > 0 for EOF. */ +int block_iter_next (struct block_iter *it,struct record rec)
If this function is used only in this file, it should be static, otherwise, the comment should go to header file
reftable/block.h line 22
Quoted 10 lines
reftable/block.h new file +7
-0+/* +Writes reftable blocks. The block_writer is reused across blocks to minimize +allocation overhead. +*/ +struct block_writer { +byte *buf; +uint32_t block_size;
I suppose this means 2^32 block should be enough for everyone?
reftable/bytes.c
Quoted 3 lines
reftable/bytes.c new file +0
-0
Empty?
reftable/config.h line 1
Quoted 7 lines
reftable/config.h new file +1
-0@@ -0,0 +1 @@ +/* empty */
Empty?
Quoted 17 lines
reftable/dump.c new file +11
-0@@ -0,0 +1,97 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "system.h" + +#include "reftable.h"
reftable/dump.c line 21
Quoted 10 lines
reftable/dump.c new file +10
-0+ +static int dump_table (const char *tablename) +{ +struct block_source src = {0 }; +int err =block_source_from_file (&src, tablename); +if (err <0 ) { +return err; + } + +struct reader *r =NULL ;
-Wdeclaration-after-statement
Quoted 4 lines
reftable/dump.c new file +4
-0+ err =new_reader (&r, src, tablename); +if (err <0 ) { +return err; + }
reftable/dump.c line 35
Quoted 10 lines
reftable/dump.c new file +10
-0+ + { +struct iterator it = {0 }; + err =reader_seek_ref (r, &it,"" ); +if (err <0 ) { +return err; + } + +struct ref_record ref = {0 }; +while (1 ) {
In the same series, we see both while (true) and while (1)
reftable/dump.c line 72
Quoted 1 line
reftable/dump.c new file +1
-0+int main (int argc,char *argv[])
Do we ship this file?
(I haven't checked change to Makefile, yet)
reftable/dump.c line 81
Quoted 9 lines
reftable/dump.c new file +9
-0+{ +int opt; +const char *table =NULL ; +while ((opt =getopt (argc, argv,"t:" )) !=-1 ) { +switch (opt) { +case 't' : + table =xstrdup (optarg); +break ; +case '?' :
We uses "-h|--help" for usage.
reftable/dump.c line 84
Quoted 3 lines
reftable/dump.c new file +3
-0+printf ("usage: %s [-table tablefile]\n" , argv[0 ]); +return 2 ; +break ;
break after return
Quoted 17 lines
reftable/iter.c new file +11
-0@@ -0,0 +1,234 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "iter.h" + +#include "system.h"
reftable/iter.c line 21
Quoted 10 lines
reftable/iter.c new file +10
-0+ +#include "block.h" +#include "constants.h" +#include "reader.h" +#include "reftable.h" + +bool iterator_is_null (struct reftable_iterator it) +{ +return it.ops ==NULL ; +}
Do we want this verbose?
Or it'll be use in some kind of vtable?
reftable/iter.c line 87
Quoted 7 lines
reftable/iter.c new file +7
-0+ +static int filtering_ref_iterator_next (void *iter_arg,struct record rec) +{ +struct filtering_ref_iterator *fri = + (struct filtering_ref_iterator *)iter_arg; +struct reftable_ref_record *ref = + (struct reftable_ref_record *)rec.data;
We're passing a pointer, references a member of a variable in local scope around, this will ask for trouble if callee doesn't aware about it, and save that pointer somewhere for later access.
I really think we shouldn't merge a big patch like this into Git, it's too hard to follow and it's harder to reasoning.
reftable/iter.h
Quoted 41 lines
reftable/iter.c new file +38
-0+ +while (true) { +int err =reftable_iterator_next_ref (fri->it, ref); +if (err !=0 ) { +return err; + } + +if (fri->double_check) { +struct reftable_iterator it = {0 }; + +int err =reftable_reader_seek_ref (fri->r, &it, + ref->ref_name); +if (err ==0 ) { + err =reftable_iterator_next_ref (it, ref); + } + +reftable_iterator_destroy (&it); + +if (err <0 ) { +return err; + } + +if (err >0 ) { +continue ; + } + } + +if ((ref->target_value !=NULL && + !memcmp (fri->oid.buf, ref->target_value, fri->oid.len)) || + (ref->value !=NULL && + !memcmp (fri->oid.buf, ref->value, fri->oid.len))) { +return 0 ; + } + } +} + +} +reftable/iter.h new file +0
-0
I stopped at this. It's too tiresome to read this patch.
Overall, I have those general comments, when I have _NOT_ read the design:
* This patch introduce a total different coding style
* size should be in size_t, not int
* passing large structure around, and passing pointer to some member of that structure around, this will ask for trouble.
* Some code are un-parsable,
* I strongly agree with Dscho, we should merge this patch as subtree instead, or we should provide API so other party can provide plugin ref-backend.
Signature · 1 line
Danh
Re: [PATCH v10 09/12] Add reftable library
Quoted 6 lines · Danh Doan
reftable/LICENSE line 4
Quoted 4 lines
reftable/LICENSE new file +4
-0+BSD License + +Copyright (c) 2020, Google LLC +All rights reserved.Linking against GPLv2 code will make this code GPLv2, right?
Wrong. A program (e.g. "git") can have various parts, some of which is licensed under GPL, some other parts under BSD, yet some other parts under LGPL. Read the second paragraph of README we ship without source.
I think you are thinking about the obligation a distributor of the linked binary compiled from such source has, which is that they need to make available not just the GPL part but the rest of the program used to compile the whole, including parts licensed under BSD and other licenses.
Re: [PATCH v10 09/12] Add reftable library
Quoted 9 lines · Junio C Hamano
Quoted 6 lines · Danh Doan
reftable/LICENSE line 4
Quoted 4 lines
reftable/LICENSE new file +4
-0+BSD License + +Copyright (c) 2020, Google LLC +All rights reserved.Linking against GPLv2 code will make this code GPLv2, right?
Quoted 4 lines
Wrong. A program (e.g. "git") can have various parts, some of which is licensed under GPL, some other parts under BSD, yet some other parts under LGPL. Read the second paragraph of README we ship without source.
without --> with our
;-)
Quoted 6 lines
I think you are thinking about the obligation a distributor of the linked binary compiled from such source has, which is that they need to make available not just the GPL part but the rest of the program used to compile the whole, including parts licensed under BSD and other licenses.
Re: [PATCH v10 09/12] Add reftable library
In reply to Danh DoanQuoted 5 lines · Danh Doan
reftable/basics.c line 13
Quoted 3 lines
reftable/basics.c new file +3
-0+#include "system.h" + +void put_be24 (byte *out,uint32_t i)So, we introduce a new type
byte?
The reftable is structured as a library. The declaration for 'byte', as well as 'bool' is private to the library.
Quoted 10 lines
reftable/basics.c line 17
Quoted 4 lines
reftable/basics.c new file +4
-0+{ + out[0 ] = (byte)((i >>16 ) &0xff ); + out[1 ] = (byte)((i >>8 ) &0xff ); + out[2 ] = (byte)((i)&0xff );At my first glance, I thought that code is:
out[2] = (byte)( (i)(&0xff) )It's totally un-parsable.
I removed the parens here, in
https://github.com/google/reftable/commit/6f883714588d2658122fefd85a2e2e76401ae2be
that also changes a number of 'int's to 'size_t's.
Quoted 4 lines
reftable/basics.c line 23
Quoted 2 lines
reftable/basics.c new file +2
-0+return (uint32_t )(in[0 ]) <<16 | (uint32_t )(in[1 ]) <<8 | + (uint32_t )(in[2 ]);I don't think we need this much cast. One cast can rule them all.
would you have a concrete suggestion to rewrite this? The rules for undefined behavior and integer casts are subtle, and I know they are a cause of concern in crypto code, and I'd rather be correct than rely on deep knowledge of the C standard.
Quoted 5 lines
Quoted 1 line
+int
binsearch(int sz, int (*f)(int k, void *args), void *args)binary search?
I think we want all size to be
size_t
fixed this.
Quoted 10 lines
reftable/basics.c line 39
Quoted 8 lines
reftable/basics.c new file +7
-0+{ +int lo =0 ; +int hi = sz; + +/* invariant: (hi == sz) || f(hi) == true +(lo == 0 && f(0) == true) || fi(lo) == false +*/ Comment style
Review style
Quoted 7 lines
Quoted 6 lines
reftable/basics.h new file +0
-0@@ -0,0 +1,53 @@[..snip..]
Quoted 5 lines
reftable/basics.h line 28
Quoted 3 lines
reftable/basics.h new file +3
-0+find smallest index i in [0, sz) at which f(i) is true, assuming +that f is ascending. Return sz if f(i) is false for all indices. +*/ So, this is about partitioning, not searching
there is precedent for this to be called binary search. See for example, https://golang.org/pkg/sort/#Search
Quoted 3 lines
Quoted 1 line
+int
block_writer_add(struct block_writer *w, struct record rec)So, we're gonna pass a large structure into a function now?
The record structure contains two pointers.
Quoted 4 lines
reftable/basics.h
Quoted 2 lines
reftable/basics.h new file +2
-0+if (block_writer_register_restart (w, start.len - out.len, restart, + key) <0 ) {Eh, naming is hard, and so does this function.
I can't parse this sentence.
Quoted 4 lines
It's overly verbose,
and long. I haven't go though all of this patch (because it's too long), but I guess some of them can be made static and their name can be shortened.
For uniformity all functions taking a struct Xyz* as the first argument are called
xyz_do_something
since this takes a block_writer, and it registers a restart, it is called block_writer_register_restart. The naming of this is uniform throughout the library.
Quoted 6 lines
reftable/basics.h
Quoted 3 lines
reftable/basics.h new file +3
-0+block_source_return_block (block->source, block); + block->data = uncompressed.buf; + block->len = dst_len;/* XXX: 4 bytes missing? */ Even this is a very lengthy patch, some bytes is missing but we couldn't figured out.
good point, I'll look into this again.
Quoted 3 lines
Quoted 1 line
+ {
This cute curly braces is new to me
Pleased to make your acquaintance!
Quoted 3 lines
Quoted 1 line
+static int
restart_key_less(int idx, void *args)idx? index? Shouldn't it be
size_t?
fixed this.
Quoted 10 lines
reftable/basics.h
Quoted 7 lines
reftable/basics.h new file +6
-0+{ +struct restart_find_args *a = (struct restart_find_args *)args; +uint32_t off =block_reader_restart_offset (a->r, idx); +struct slice in = { + .buf = a->r->block.data + off, + .len = a->r->block_len - off,C99 designated initialisation.
We aren't ready, yet.
I'm sorry, gcc -std=c89 said this was OK.
Quoted 4 lines
Quoted 4 lines
reftable/basics.h new file +4
-0+void block_iter_copy_from (struct block_iter *dest,struct block_iter *src) +{ + dest->br = src->br; + dest->next_off = src->next_off;
Quoted 8 lines
reftable/basics.h
Quoted 5 lines
reftable/basics.h new file +5
-0+slice_copy (&dest->last_key, src->last_key); +} + +/* return < 0 for error, 0 for OK, > 0 for EOF. */ +int block_iter_next (struct block_iter *it,struct record rec)If this function is used only in this file, it should be static, otherwise, the comment should go to header file
moved comment to header file.
Quoted 4 lines
Quoted 4 lines
reftable/block.h new file +1
-0+/*
Quoted 8 lines
reftable/block.h line 22
Quoted 6 lines
reftable/block.h new file +6
-0+Writes reftable blocks. The block_writer is reused across blocks to minimize +allocation overhead. +*/ +struct block_writer { +byte *buf; +uint32_t block_size;I suppose this means 2^32 block should be enough for everyone?
No. 2^24-1 is the maximum size, as detailed in the specification.
Quoted 5 lines
reftable/bytes.c
Quoted 3 lines
reftable/bytes.c new file +0
-0Empty?
thanks, I've removed these.
Quoted 9 lines
reftable/config.h line 1
Quoted 7 lines
reftable/config.h new file +1
-0@@ -0,0 +1 @@ +/* empty */ Empty?
Removed
Quoted 1 line
Quoted 1 line
+++
b/reftable/dump.c
This was moved to a different directory in upstream; I've removed it now.
Quoted 7 lines
reftable/config.h
Quoted 4 lines
reftable/config.h new file +4
-0+bool iterator_is_null (struct reftable_iterator it) +{ +return it.ops ==NULL ; +}Do we want this verbose?
Or it'll be use in some kind of vtable?
I've added a doc comment.
Quoted 11 lines
reftable/config.h
Quoted 7 lines
reftable/config.h new file +7
-0+ +static int filtering_ref_iterator_next (void *iter_arg,struct record rec) +{ +struct filtering_ref_iterator *fri = + (struct filtering_ref_iterator *)iter_arg; +struct reftable_ref_record *ref = + (struct reftable_ref_record *)rec.data;We're passing a pointer, references a member of a variable in local scope around, this will ask for trouble if callee doesn't aware about it, and save that pointer somewhere for later access.
I don't understand your criticism; could you be more precise?
Signature · 11 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Re: [PATCH v10 09/12] Add reftable library
Quoted 1 line · Han-Wen Nienhuys
I've added a doc comment.
I've pushed these fixes to
https://github.com/gitgitgadget/git/pull/539. Should I be sending new patches over email for each review round?
Signature · 11 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado