Reftable support git-core
Subsystems (2)
PATCH v10 10/12 Reftable support for git-core
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
For background, see the previous commit introducing the library.
TODO:
* Resolve spots marked with XXX
* Detect and prevent directory/file conflicts in naming.
* Support worktrees (t0002-gitfile "linked repo" testcase)
Example use: see t/t0031-reftable.sh
--- .../technical/repository-version.txt | 7 + Makefile | 26 +- builtin/clone.c | 3 +- builtin/init-db.c | 61 +- cache.h | 6 +- refs.c | 26 +- refs.h | 3 + refs/refs-internal.h | 1 + refs/reftable-backend.c | 1045 +++++++++++++++++ reftable/update.sh | 2 +- repository.c | 2 + repository.h | 3 + setup.c | 12 +- t/t0031-reftable.sh | 99 ++ 14 files changed, 1262 insertions(+), 34 deletions(-) create mode 100644 refs/reftable-backend.c create mode 100755 t/t0031-reftable.sh
Documentation/technical/repository-version.txt +7 -0
@@ -100,3 +100,10 @@ If set, by default "git config" reads from both "config" and multiple working directory mode, "config" file is shared while "config.worktree" is per-working directory (i.e., it's in GIT_COMMON_DIR/worktrees/<id>/config.worktree) + +==== `refStorage` + +Specifies the file format for the ref database. Values are `files` +(for the traditional packed + loose ref format) and `reftable` for the +binary reftable format. See https://github.com/google/reftable for +more information.
Makefile +24 -2
@@ -814,6 +814,7 @@ TEST_SHELL_PATH = $(SHELL_PATH) LIB_FILE = libgit.a XDIFF_LIB = xdiff/lib.a VCSSVN_LIB = vcs-svn/lib.a +REFTABLE_LIB = reftable/libreftable.a GENERATED_H += command-list.h
@@ -961,6 +962,7 @@ LIB_OBJS += rebase-interactive.o LIB_OBJS += reflog-walk.o LIB_OBJS += refs.o LIB_OBJS += refs/files-backend.o +LIB_OBJS += refs/reftable-backend.o LIB_OBJS += refs/iterator.o LIB_OBJS += refs/packed-backend.o LIB_OBJS += refs/ref-cache.o
@@ -1164,7 +1166,7 @@ THIRD_PARTY_SOURCES += compat/regex/% THIRD_PARTY_SOURCES += sha1collisiondetection/% THIRD_PARTY_SOURCES += sha1dc/% -GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) +GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) EXTLIBS = GIT_USER_AGENT = git/$(GIT_VERSION)
@@ -2348,11 +2350,28 @@ VCSSVN_OBJS += vcs-svn/fast_export.o VCSSVN_OBJS += vcs-svn/svndiff.o VCSSVN_OBJS += vcs-svn/svndump.o +REFTABLE_OBJS += reftable/basics.o +REFTABLE_OBJS += reftable/block.o +REFTABLE_OBJS += reftable/bytes.o +REFTABLE_OBJS += reftable/file.o +REFTABLE_OBJS += reftable/iter.o +REFTABLE_OBJS += reftable/merged.o +REFTABLE_OBJS += reftable/pq.o +REFTABLE_OBJS += reftable/reader.o +REFTABLE_OBJS += reftable/record.o +REFTABLE_OBJS += reftable/slice.o +REFTABLE_OBJS += reftable/stack.o +REFTABLE_OBJS += reftable/tree.o +REFTABLE_OBJS += reftable/writer.o +REFTABLE_OBJS += reftable/zlib-compat.o + + TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) OBJECTS := $(LIB_OBJS) $(BUILTIN_OBJS) $(PROGRAM_OBJS) $(TEST_OBJS) \ $(XDIFF_OBJS) \ $(VCSSVN_OBJS) \ $(FUZZ_OBJS) \ + $(REFTABLE_OBJS) \ common-main.o \ git.o ifndef NO_CURL
@@ -2489,6 +2508,9 @@ $(XDIFF_LIB): $(XDIFF_OBJS) $(VCSSVN_LIB): $(VCSSVN_OBJS) $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ +$(REFTABLE_LIB): $(REFTABLE_OBJS) + $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ + export DEFAULT_EDITOR DEFAULT_PAGER Documentation/GIT-EXCLUDED-PROGRAMS: FORCE
@@ -3104,7 +3126,7 @@ cocciclean: clean: profile-clean coverage-clean cocciclean $(RM) *.res $(RM) $(OBJECTS) - $(RM) $(LIB_FILE) $(XDIFF_LIB) $(VCSSVN_LIB) + $(RM) $(LIB_FILE) $(XDIFF_LIB) $(VCSSVN_LIB) $(REFTABLE_LIB) $(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) git$X $(RM) $(TEST_PROGRAMS) $(RM) $(FUZZ_PROGRAMS)
builtin/clone.c +2 -1
@@ -1108,7 +1108,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix) } } -init_db (git_dir, real_git_dir, option_template,GIT_HASH_UNKNOWN ,INIT_DB_QUIET ); +init_db (git_dir, real_git_dir, option_template,GIT_HASH_UNKNOWN , +DEFAULT_REF_STORAGE ,INIT_DB_QUIET );if (real_git_dir) git_dir = real_git_dir;
builtin/init-db.c +44 -17
140 lines not on this page · raw message
cache.h +4 -2
@@ -628,8 +628,9 @@ int path_inside_repo(const char *prefix, const char *path);int init_db (const char *git_dir,const char *real_git_dir,const char *template_dir,int hash_algo, -unsigned int flags); -void initialize_repository_version (int hash_algo); +const char *ref_storage_format,unsigned int flags); +void initialize_repository_version (int hash_algo, +const char *ref_storage_format);void sanitize_stdfds (void );int daemonize (void );
@@ -1043,6 +1044,7 @@ struct repository_format {int is_bare;int hash_algo;char *work_tree; +char *ref_storage;struct string_list unknown_extensions; };
refs.c +18 -8
@@ -17,10 +17,16 @@#include "argv-array.h" #include "repository.h" +const char *default_ref_storage (void ) +{ +const char *test =getenv ("GIT_TEST_REFTABLE" ); +return test ?"reftable" :"files" ; +} +/* * List of all available backends */ -static struct ref_storage_be *refs_backends = &refs_be_files; +static struct ref_storage_be *refs_backends = &refs_be_reftable;static struct ref_storage_be *find_ref_storage_backend (const char *name) {
@@ -1836,13 +1842,13 @@ static struct ref_store *lookup_ref_store_map(struct hashmap *map,* Create, record, and return a ref_store instance for the specified * gitdir. */ -static struct ref_store *ref_store_init (const char *gitdir, +static struct ref_store *ref_store_init (const char *gitdir,const char *be_name,unsigned int flags) { -const char *be_name ="files" ; -struct ref_storage_be *be =find_ref_storage_backend (be_name); +struct ref_storage_be *be;struct ref_store *refs; + be =find_ref_storage_backend (be_name);if (!be)BUG ("reference backend %s is unknown" , be_name);
@@ -1858,7 +1864,10 @@ struct ref_store *get_main_ref_store(struct repository *r)if (!r->gitdir)BUG ("attempting to get main_ref_store outside of repository" ); - r->refs_private =ref_store_init (r->gitdir,REF_STORE_ALL_CAPS ); + r->refs_private =ref_store_init ( + r->gitdir, + r->ref_storage_format ? r->ref_storage_format :DEFAULT_REF_STORAGE , +REF_STORE_ALL_CAPS );return r->refs_private; }
@@ -1913,7 +1922,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule) goto done;/* assume that add_submodule_odb() has been called */ - refs =ref_store_init (submodule_sb.buf, + refs =ref_store_init (submodule_sb.buf,DEFAULT_REF_STORAGE ,/* XXX */ REF_STORE_READ |REF_STORE_ODB );register_ref_store_map (&submodule_ref_stores,"submodule" , refs, submodule);
@@ -1927,6 +1936,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule)struct ref_store *get_worktree_ref_store (const struct worktree *wt) { +const char *format =DEFAULT_REF_STORAGE ;/* XXX */ struct ref_store *refs;const char *id;
@@ -1940,9 +1950,9 @@ struct ref_store *get_worktree_ref_store(const struct worktree *wt)if (wt->id) refs =ref_store_init (git_common_path ("worktrees/%s" , wt->id), -REF_STORE_ALL_CAPS ); + format,REF_STORE_ALL_CAPS );else - refs =ref_store_init (get_git_common_dir (), + refs =ref_store_init (get_git_common_dir (), format,REF_STORE_ALL_CAPS );if (refs)
refs.h +3 -0
@@ -9,6 +9,9 @@ struct string_list;struct string_list_item ;struct worktree ; +/* Returns the ref storage backend to use by default. */ +const char *default_ref_storage (void ); +/* * Resolve a reference, recursively following symbolic refererences. *
refs/refs-internal.h +1 -0
@@ -661,6 +661,7 @@ struct ref_storage_be { };extern struct ref_storage_be refs_be_files; +extern struct ref_storage_be refs_be_reftable;extern struct ref_storage_be refs_be_packed;/*
refs/reftable-backend.c new file +1045 -0
1,045 lines not on this page · raw message
reftable/update.sh +1 -1
@@ -11,7 +11,7 @@ SRC=${SRC:-origin} BRANCH=${BRANCH:-origin/master} cp reftable-repo/c/*.[ch] reftable/ cp reftable-repo/c/include/*.[ch] reftable/ cp reftable-repo/LICENSE reftable/ -git --git-dir reftable-repo/.git show --no-patch origin/master \ +git --git-dir reftable-repo/.git show --no-patch ${BRANCH} \ > reftable/VERSION mv reftable/system.h reftable/system.h~
repository.c +2 -0
@@ -178,6 +178,8 @@ int repo_init(struct repository *repo,if (worktree)repo_set_worktree (repo, worktree); + repo->ref_storage_format =xstrdup_or_null (format.ref_storage); +clear_repository_format (&format);return 0 ;
repository.h +3 -0
@@ -74,6 +74,9 @@ struct repository {*/ struct ref_store *refs_private; +/* The format to use for the ref database. */ +char *ref_storage_format; +/* * Contains path to often used file names. */
setup.c +9 -3
@@ -468,9 +468,11 @@ static int check_repo_format(const char *var, const char *value, void *vdata)if (!value)return config_error_nonbool (var); data->partial_clone =xstrdup (value); - }else if (!strcmp (ext,"worktreeconfig" )) + }else if (!strcmp (ext,"worktreeconfig" )) { data->worktree_config =git_config_bool (var, value); -else + }else if (!strcmp (ext,"refstorage" )) { + data->ref_storage =xstrdup (value); + }else string_list_append (&data->unknown_extensions, ext); }
@@ -559,6 +561,7 @@ void clear_repository_format(struct repository_format *format)string_list_clear (&format->unknown_extensions,0 );free (format->work_tree);free (format->partial_clone); +free (format->ref_storage);init_repository_format (format); }
@@ -1204,8 +1207,11 @@ const char *setup_git_directory_gently(int *nongit_ok) gitdir =DEFAULT_GIT_DIR_ENVIRONMENT ;setup_git_env (gitdir); } -if (startup_info->have_repository) +if (startup_info->have_repository) {repo_set_hash_algo (the_repository, repo_fmt.hash_algo); + the_repository->ref_storage_format = +xstrdup_or_null (repo_fmt.ref_storage); + } }strbuf_release (&dir);
t/t0031-reftable.sh new file +99 -0
99 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v10 11/12 Add some reftable testing infrastructure
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
* Add GIT_TEST_REFTABLE environment var to control default ref storage
* Add test_prerequisite REFTABLE. Skip t/t3210-pack-refs.sh for REFTABLE.
--- builtin/clone.c | 2 +- builtin/init-db.c | 2 +- refs.c | 13 +++++++------ t/t3210-pack-refs.sh | 6 ++++++ t/test-lib.sh | 5 +++++ 5 files changed, 20 insertions(+), 8 deletions(-)
builtin/clone.c +1 -1
@@ -1109,7 +1109,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix) }init_db (git_dir, real_git_dir, option_template,GIT_HASH_UNKNOWN , -DEFAULT_REF_STORAGE ,INIT_DB_QUIET ); +default_ref_storage (),INIT_DB_QUIET );if (real_git_dir) git_dir = real_git_dir;
builtin/init-db.c +1 -1
@@ -542,7 +542,7 @@ static const char *const init_db_usage[] = {int cmd_init_db (int argc,const char **argv,const char *prefix) {const char *git_dir; -const char *ref_storage_format =DEFAULT_REF_STORAGE ; +const char *ref_storage_format =default_ref_storage ();const char *real_git_dir =NULL ;const char *work_tree;const char *template_dir =NULL ;
refs.c +7 -6
@@ -1864,10 +1864,11 @@ struct ref_store *get_main_ref_store(struct repository *r)if (!r->gitdir)BUG ("attempting to get main_ref_store outside of repository" ); - r->refs_private =ref_store_init ( - r->gitdir, - r->ref_storage_format ? r->ref_storage_format :DEFAULT_REF_STORAGE , -REF_STORE_ALL_CAPS ); + r->refs_private =ref_store_init (r->gitdir, + r->ref_storage_format ? + r->ref_storage_format : +default_ref_storage (), +REF_STORE_ALL_CAPS );return r->refs_private; }
@@ -1922,7 +1923,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule) goto done;/* assume that add_submodule_odb() has been called */ - refs =ref_store_init (submodule_sb.buf,DEFAULT_REF_STORAGE ,/* XXX */ + refs =ref_store_init (submodule_sb.buf,default_ref_storage (),REF_STORE_READ |REF_STORE_ODB );register_ref_store_map (&submodule_ref_stores,"submodule" , refs, submodule);
@@ -1936,7 +1937,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule)struct ref_store *get_worktree_ref_store (const struct worktree *wt) { -const char *format =DEFAULT_REF_STORAGE ;/* XXX */ +const char *format =default_ref_storage ();struct ref_store *refs;const char *id;
t/t3210-pack-refs.sh +6 -0
@@ -11,6 +11,12 @@ semantic is still the same. ' . ./test-lib.sh +if test_have_prereq REFTABLE +then + skip_all='skipping pack-refs tests; incompatible with reftable' + test_done +fi + test_expect_success 'enable reflogs' ' git config core.logallrefupdates true '
t/test-lib.sh +5 -0
@@ -1502,6 +1502,11 @@ FreeBSD) ;; esac +if test -n "$GIT_TEST_REFTABLE" +then + test_set_prereq REFTABLE +fi + ( COLUMNS=1 && test $COLUMNS = 1 ) && test_set_prereq COLUMNS_CAN_BE_1 test -z "$NO_PERL" && test_set_prereq PERL test -z "$NO_PTHREADS" && test_set_prereq PTHREADS
Signature · 1 line
gitgitgadget
PATCH v10 12/12 t: use update-ref and show-ref to reading/writing refs
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Reading and writing .git/refs/* assumes that refs are stored in the 'files' ref backend.
--- t/t0002-gitfile.sh | 2 +- t/t1400-update-ref.sh | 32 ++++++++++++++++---------------- t/t1506-rev-parse-diagnosis.sh | 2 +- t/t6050-replace.sh | 2 +- t/t9020-remote-svn.sh | 4 ++-- 5 files changed, 21 insertions(+), 21 deletions(-)
t/t0002-gitfile.sh +1 -1
@@ -62,7 +62,7 @@ test_expect_success 'check commit-tree' ' ' test_expect_success 'check rev-list' ' - echo $SHA >"$REAL/HEAD" && + git update-ref "HEAD" "$SHA" && test "$SHA" = "$(git rev-list HEAD)" '
t/t1400-update-ref.sh +16 -16
@@ -37,15 +37,15 @@ test_expect_success setup ' test_expect_success "create $m" ' git update-ref $m $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success "create $m with oldvalue verification" ' git update-ref $m $B $A && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "fail to delete $m with stale ref" ' test_must_fail git update-ref -d $m $A && - test $B = "$(cat .git/$m)" + test $B = "$(git show-ref -s --verify $m)" ' test_expect_success "delete $m" ' test_when_finished "rm -f .git/$m" &&
@@ -56,7 +56,7 @@ test_expect_success "delete $m" ' test_expect_success "delete $m without oldvalue verification" ' test_when_finished "rm -f .git/$m" && git update-ref $m $A && - test $A = $(cat .git/$m) && + test $A = $(git show-ref -s --verify $m) && git update-ref -d $m && test_path_is_missing .git/$m '
@@ -69,15 +69,15 @@ test_expect_success "fail to create $n" ' test_expect_success "create $m (by HEAD)" ' git update-ref HEAD $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success "create $m (by HEAD) with oldvalue verification" ' git update-ref HEAD $B $A && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "fail to delete $m (by HEAD) with stale ref" ' test_must_fail git update-ref -d HEAD $A && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "delete $m (by HEAD)" ' test_when_finished "rm -f .git/$m" &&
@@ -178,14 +178,14 @@ test_expect_success '--no-create-reflog overrides core.logAllRefUpdates=always' test_expect_success "create $m (by HEAD)" ' git update-ref HEAD $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success 'pack refs' ' git pack-refs --all ' test_expect_success "move $m (by HEAD)" ' git update-ref HEAD $B $A && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "delete $m (by HEAD) should remove both packed and loose $m" ' test_when_finished "rm -f .git/$m" &&
@@ -255,7 +255,7 @@ test_expect_success '(not) change HEAD with wrong SHA1' ' ' test_expect_success "(not) changed .git/$m" ' test_when_finished "rm -f .git/$m" && - ! test $B = $(cat .git/$m) + ! test $B = $(git show-ref -s --verify $m) ' rm -f .git/logs/refs/heads/master
@@ -263,19 +263,19 @@ test_expect_success "create $m (logged by touch)" ' test_config core.logAllRefUpdates false && GIT_COMMITTER_DATE="2005-05-26 23:30" \ git update-ref --create-reflog HEAD $A -m "Initial Creation" && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success "update $m (logged by touch)" ' test_config core.logAllRefUpdates false && GIT_COMMITTER_DATE="2005-05-26 23:31" \ git update-ref HEAD $B $A -m "Switch" && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "set $m (logged by touch)" ' test_config core.logAllRefUpdates false && GIT_COMMITTER_DATE="2005-05-26 23:41" \ git update-ref HEAD $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success 'empty directory removal' '
@@ -319,19 +319,19 @@ test_expect_success "create $m (logged by config)" ' test_config core.logAllRefUpdates true && GIT_COMMITTER_DATE="2005-05-26 23:32" \ git update-ref HEAD $A -m "Initial Creation" && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success "update $m (logged by config)" ' test_config core.logAllRefUpdates true && GIT_COMMITTER_DATE="2005-05-26 23:33" \ git update-ref HEAD'" $B $A "'-m "Switch" && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "set $m (logged by config)" ' test_config core.logAllRefUpdates true && GIT_COMMITTER_DATE="2005-05-26 23:43" \ git update-ref HEAD $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' cat >expect <<EOF
t/t1506-rev-parse-diagnosis.sh +1 -1
@@ -207,7 +207,7 @@ test_expect_success 'arg before dashdash must be a revision (ambiguous)' ' { # we do not want to use rev-parse here, because # we are testing it - cat .git/refs/heads/foobar && + git show-ref -s refs/heads/foobar && printf "%s\n" -- } >expect && git rev-parse foobar -- >actual &&
t/t6050-replace.sh +1 -1
@@ -135,7 +135,7 @@ test_expect_success 'tag replaced commit' ' test_expect_success '"git fsck" works' ' git fsck master >fsck_master.out && test_i18ngrep "dangling commit $R" fsck_master.out && - test_i18ngrep "dangling tag $(cat .git/refs/tags/mytag)" fsck_master.out && + test_i18ngrep "dangling tag $(git show-ref -s refs/tags/mytag)" fsck_master.out && test -z "$(git fsck)" '
t/t9020-remote-svn.sh +2 -2
@@ -48,8 +48,8 @@ test_expect_success REMOTE_SVN 'simple fetch' ' ' test_debug ' - cat .git/refs/svn/svnsim/master - cat .git/refs/remotes/svnsim/master + git show-ref -s refs/svn/svnsim/master + git show-ref -s refs/remotes/svnsim/master ' test_expect_success REMOTE_SVN 'repeated fetch, nothing shall change' '
Signature · 1 line
gitgitgadget
v11 cover Reftable support git-core
In reply to Han-Wen Nienhuys via GitGitGadgetThis adds the reftable library, and hooks it up as a ref backend.
Includes testing support, to test: make -C t/ GIT_TEST_REFTABLE=1
Summary 18964 tests pass 2871 tests fail
Some issues:
* worktree support looks broken
* submodules (eg. t1013)
* many tests inspect .git/logs/ directly.
v14
* Fixes for Windows (thanks, Dscho!)
* Detect file/directory conflicts.
* Plug some memory leaks.
* Fixes for autocompaction
Han-Wen Nienhuys (10):
refs.h: clarify reflog iteration order
Iterate over the "refs/" namespace in for_each_[raw]ref
refs: document how ref_iterator_advance_fn should handle symrefs Add .gitattributes for the reftable/ directory
reftable: define version 2 of the spec to accomodate SHA256
reftable: clarify how empty tables should be written Add reftable library
Reftable support for git-core
Add some reftable testing infrastructure
t: use update-ref and show-ref to reading/writing refs
Johannes Schindelin (1):
vcxproj: adjust for the reftable changes
Jonathan Nieder (1):
reftable: file format documentation
Documentation/Makefile | 1 + Documentation/technical/reftable.txt | 1080 +++++++++++++++ .../technical/repository-version.txt | 7 + Makefile | 27 +- builtin/clone.c | 3 +- builtin/init-db.c | 56 +- cache.h | 6 +- config.mak.uname | 2 +- contrib/buildsystems/Generators/Vcxproj.pm | 11 +- refs.c | 33 +- refs.h | 8 +- refs/refs-internal.h | 6 + refs/reftable-backend.c | 1045 ++++++++++++++ reftable/.gitattributes | 1 + reftable/LICENSE | 31 + reftable/README.md | 11 + reftable/VERSION | 5 + reftable/basics.c | 215 +++ reftable/basics.h | 53 + reftable/block.c | 436 ++++++ reftable/block.h | 126 ++ reftable/blocksource.h | 22 + reftable/constants.h | 21 + reftable/file.c | 99 ++ reftable/iter.c | 240 ++++ reftable/iter.h | 63 + reftable/merged.c | 327 +++++ reftable/merged.h | 38 + reftable/pq.c | 115 ++ reftable/pq.h | 34 + reftable/reader.c | 758 ++++++++++ reftable/reader.h | 68 + reftable/record.c | 1131 +++++++++++++++ reftable/record.h | 121 ++ reftable/refname.c | 215 +++ reftable/refname.h | 39 + reftable/reftable.c | 91 ++ reftable/reftable.h | 564 ++++++++ reftable/slice.c | 225 +++ reftable/slice.h | 76 + reftable/stack.c | 1229 +++++++++++++++++ reftable/stack.h | 45 + reftable/system.h | 54 + reftable/tree.c | 67 + reftable/tree.h | 34 + reftable/update.sh | 24 + reftable/writer.c | 661 +++++++++ reftable/writer.h | 60 + reftable/zlib-compat.c | 92 ++ repository.c | 2 + repository.h | 3 + setup.c | 12 +- t/t0002-gitfile.sh | 2 +- t/t0031-reftable.sh | 109 ++ t/t1400-update-ref.sh | 32 +- t/t1506-rev-parse-diagnosis.sh | 2 +- t/t3210-pack-refs.sh | 6 + t/t6050-replace.sh | 2 +- t/t9020-remote-svn.sh | 4 +- t/test-lib.sh | 5 + 60 files changed, 9798 insertions(+), 57 deletions(-) create mode 100644 Documentation/technical/reftable.txt create mode 100644 refs/reftable-backend.c create mode 100644 reftable/.gitattributes create mode 100644 reftable/LICENSE create mode 100644 reftable/README.md create mode 100644 reftable/VERSION create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/blocksource.h create mode 100644 reftable/constants.h create mode 100644 reftable/file.c create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/refname.c create mode 100644 reftable/refname.h create mode 100644 reftable/reftable.c create mode 100644 reftable/reftable.h create mode 100644 reftable/slice.c create mode 100644 reftable/slice.h create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/system.h create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100755 reftable/update.sh create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h create mode 100644 reftable/zlib-compat.c create mode 100755 t/t0031-reftable.sh
base-commit: b34789c0b0d3b137f0bb516b417bd8d75e0cb306
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-539%2Fhanwen%2Freftable-v11
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-539/hanwen/reftable-v11
Pull-Request: https://github.com/gitgitgadget/git/pull/539
Range-diff vs v10:
1: d3d8ed2032c = 1: dfa5fd74f85 refs.h: clarify reflog iteration order
2: 45fd65f72e0 = 2: 340c5c415e1 Iterate over the "refs/" namespace in for_each_[raw]ref
3: bc89bcd9c8c < -: ----------- create .git/refs in files-backend.c
4: fd67cdff0cd = 3: 6553285043b refs: document how ref_iterator_advance_fn should handle symrefs
5: 5373828f854 = 4: 7dc47c7756f Add .gitattributes for the reftable/ directory
6: 8eeed29ebbf = 5: 06fcb49e903 reftable: file format documentation
7: 5ebc272e23d = 6: 093fa74a3d0 reftable: define version 2 of the spec to accomodate SHA256
8: 95aae041613 = 7: 6d9031372ce reftable: clarify how empty tables should be written
9: 59209a5ad39 ! 8: 6ee6c44752c Add reftable library
@@ reftable/README.md (new)
## reftable/VERSION (new) ##
@@
-+commit a6d6b31bea256044621d789df64a8159647f21bc
++commit 3a486f79abcd17e88e3bca62f43188a7c8b80ff3
+Author: Han-Wen Nienhuys <[email protected]>
-+Date: Mon Apr 27 20:46:21 2020 +0200
++Date: Mon May 4 19:20:21 2020 +0200
+
-+ C: prefix error codes with REFTABLE_
++ C: include "cache.h" in git-core sleep_millisec()
## reftable/basics.c (new) ##
@@
@@ reftable/basics.c (new)
+{
+ out[0] = (byte)((i >> 16) & 0xff);
+ out[1] = (byte)((i >> 8) & 0xff);
-+ out[2] = (byte)((i)&0xff);
++ out[2] = (byte)(i & 0xff);
+}
+
+uint32_t get_be24(byte *in)
@@ reftable/basics.c (new)
+void put_be16(uint8_t *out, uint16_t i)
+{
+ out[0] = (uint8_t)((i >> 8) & 0xff);
-+ out[1] = (uint8_t)((i)&0xff);
++ out[1] = (uint8_t)(i & 0xff);
+}
+
-+int binsearch(int sz, int (*f)(int k, void *args), void *args)
++int binsearch(size_t sz, int (*f)(size_t k, void *args), void *args)
+{
-+ int lo = 0;
-+ int hi = sz;
++ size_t lo = 0;
++ size_t hi = sz;
+
+ /* invariant: (hi == sz) || f(hi) == true
+ (lo == 0 && f(0) == true) || fi(lo) == false
+ */
+ while (hi - lo > 1) {
-+ int mid = lo + (hi - lo) / 2;
++ size_t mid = lo + (hi - lo) / 2;
+
+ int val = f(mid, args);
+ if (val) {
@@ reftable/basics.c (new)
+
+const char *reftable_error_str(int err)
+{
++ static char buf[250];
+ switch (err) {
+ case REFTABLE_IO_ERROR:
+ return "I/O error";
@@ reftable/basics.c (new)
+ return "misuse of the reftable API";
+ case REFTABLE_ZLIB_ERROR:
+ return "zlib failure";
++ case REFTABLE_NAME_CONFLICT:
++ return "file/directory conflict";
++ case REFTABLE_REFNAME_ERROR:
++ return "invalid refname";
+ case -1:
+ return "general error";
+ default:
-+ return "unknown error code";
++ snprintf(buf, sizeof(buf), "unknown error code %d", err);
++ return buf;
+ }
+}
+
-+int reftable_error_to_errno(int err) {
++int reftable_error_to_errno(int err)
++{
+ switch (err) {
+ case REFTABLE_IO_ERROR:
+ return EIO;
@@ reftable/basics.c (new)
+ }
+}
+
-+
+void *(*reftable_malloc_ptr)(size_t sz) = &malloc;
+void *(*reftable_realloc_ptr)(void *, size_t) = &realloc;
+void (*reftable_free_ptr)(void *) = &free;
@@ reftable/basics.h (new)
+ find smallest index i in [0, sz) at which f(i) is true, assuming
+ that f is ascending. Return sz if f(i) is false for all indices.
+*/
-+int binsearch(int sz, int (*f)(int k, void *args), void *args);
++int binsearch(size_t sz, int (*f)(size_t k, void *args), void *args);
+
+/*
+ Frees a NULL terminated array of malloced strings. The array itself is also
@@ reftable/block.c (new)
+ memcpy(w->buf + block_header_skip, compressed.buf,
+ dest_len);
+ w->next = dest_len + block_header_skip;
++ slice_clear(&compressed);
+ break;
+ }
+ }
@@ reftable/block.c (new)
+ if (typ == BLOCK_TYPE_LOG) {
+ struct slice uncompressed = { 0 };
+ int block_header_skip = 4 + header_off;
-+ uLongf dst_len = sz - block_header_skip;
++ uLongf dst_len = sz - block_header_skip; /* total size of dest
++ buffer. */
+ uLongf src_len = block->len - block_header_skip;
+
++ /* Log blocks specify the *uncompressed* size in their header.
++ */
+ slice_resize(&uncompressed, sz);
++
++ /* Copy over the block header verbatim. It's not compressed. */
+ memcpy(uncompressed.buf, block->data, block_header_skip);
+
++ /* Uncompress */
+ if (Z_OK != uncompress_return_consumed(
+ uncompressed.buf + block_header_skip,
+ &dst_len, block->data + block_header_skip,
@@ reftable/block.c (new)
+ return REFTABLE_ZLIB_ERROR;
+ }
+
++ if (dst_len + block_header_skip != sz) {
++ return REFTABLE_FORMAT_ERROR;
++ }
++
++ /* We're done with the input data. */
+ block_source_return_block(block->source, block);
+ block->data = uncompressed.buf;
-+ block->len = dst_len; /* XXX: 4 bytes missing? */
++ block->len = sz;
+ block->source = malloc_block_source();
+ full_block_size = src_len + block_header_skip;
+ } else if (full_block_size == 0) {
@@ reftable/block.c (new)
+ struct block_reader *r;
+};
+
-+static int restart_key_less(int idx, void *args)
++static int restart_key_less(size_t idx, void *args)
+{
+ struct restart_find_args *a = (struct restart_find_args *)args;
+ uint32_t off = block_reader_restart_offset(a->r, idx);
@@ reftable/block.c (new)
+ slice_copy(&dest->last_key, src->last_key);
+}
+
-+/* return < 0 for error, 0 for OK, > 0 for EOF. */
+int block_iter_next(struct block_iter *it, struct record rec)
+{
+ if (it->next_off >= it->br->block_len) {
@@ reftable/block.h (new)
+int block_reader_first_key(struct block_reader *br, struct slice *key);
+
+void block_iter_copy_from(struct block_iter *dest, struct block_iter *src);
++
++/* return < 0 for error, 0 for OK, > 0 for EOF. */
+int block_iter_next(struct block_iter *it, struct record rec);
+
+/* Seek to `want` with in the block pointed to by `it` */
@@ reftable/blocksource.h (new)
+
+#endif
- ## reftable/bytes.c (new) ##
-
- ## reftable/config.h (new) ##
-@@
-+/* empty */
-
## reftable/constants.h (new) ##
@@
+/*
@@ reftable/constants.h (new)
+
+#endif
- ## reftable/dump.c (new) ##
-@@
-+/*
-+Copyright 2020 Google LLC
-+
-+Use of this source code is governed by a BSD-style
-+license that can be found in the LICENSE file or at
-+https://developers.google.com/open-source/licenses/bsd
-+*/
-+
-+#include "system.h"
-+
-+#include "reftable.h"
-+
-+static int dump_table(const char *tablename)
-+{
-+ struct block_source src = { 0 };
-+ int err = block_source_from_file(&src, tablename);
-+ if (err < 0) {
-+ return err;
-+ }
-+
-+ struct reader *r = NULL;
-+ err = new_reader(&r, src, tablename);
-+ if (err < 0) {
-+ return err;
-+ }
-+
-+ {
-+ struct iterator it = { 0 };
-+ err = reader_seek_ref(r, &it, "");
-+ if (err < 0) {
-+ return err;
-+ }
-+
-+ struct ref_record ref = { 0 };
-+ while (1) {
-+ err = iterator_next_ref(it, &ref);
-+ if (err > 0) {
-+ break;
-+ }
-+ if (err < 0) {
-+ return err;
-+ }
-+ ref_record_print(&ref, 20);
-+ }
-+ iterator_destroy(&it);
-+ ref_record_clear(&ref);
-+ }
-+
-+ {
-+ struct iterator it = { 0 };
-+ err = reader_seek_log(r, &it, "");
-+ if (err < 0) {
-+ return err;
-+ }
-+ struct log_record log = { 0 };
-+ while (1) {
-+ err = iterator_next_log(it, &log);
-+ if (err > 0) {
-+ break;
-+ }
-+ if (err < 0) {
-+ return err;
-+ }
-+ log_record_print(&log, 20);
-+ }
-+ iterator_destroy(&it);
-+ log_record_clear(&log);
-+ }
-+ return 0;
-+}
-+
-+int main(int argc, char *argv[])
-+{
-+ int opt;
-+ const char *table = NULL;
-+ while ((opt = getopt(argc, argv, "t:")) != -1) {
-+ switch (opt) {
-+ case 't':
-+ table = xstrdup(optarg);
-+ break;
-+ case '?':
-+ printf("usage: %s [-table tablefile]\n", argv[0]);
-+ return 2;
-+ break;
-+ }
-+ }
-+
-+ if (table != NULL) {
-+ int err = dump_table(table);
-+ if (err < 0) {
-+ fprintf(stderr, "%s: %s: %s\n", argv[0], table,
-+ error_str(err));
-+ return 1;
-+ }
-+ }
-+ return 0;
-+}
-
## reftable/file.c (new) ##
@@
+/*
@@ reftable/file.c (new)
+ p->size = st.st_size;
+ p->fd = fd;
+
++ assert(bs->ops == NULL);
+ bs->ops = &file_vtable;
+ bs->arg = p;
+ }
+ return 0;
+}
+
-+int reftable_fd_write(void *arg, byte *data, int sz)
++int reftable_fd_write(void *arg, byte *data, size_t sz)
+{
+ int *fdp = (int *)arg;
+ return write(*fdp, data, sz);
@@ reftable/iter.c (new)
+
+void iterator_set_empty(struct reftable_iterator *it)
+{
++ assert(it->ops == NULL);
+ it->iter_arg = NULL;
+ it->ops = &empty_vtable;
+}
@@ reftable/iter.c (new)
+ (struct filtering_ref_iterator *)iter_arg;
+ struct reftable_ref_record *ref =
+ (struct reftable_ref_record *)rec.data;
-+
++ int err = 0;
+ while (true) {
-+ int err = reftable_iterator_next_ref(fri->it, ref);
++ err = reftable_iterator_next_ref(fri->it, ref);
+ if (err != 0) {
-+ return err;
++ break;
+ }
+
+ if (fri->double_check) {
+ struct reftable_iterator it = { 0 };
+
-+ int err = reftable_reader_seek_ref(fri->r, &it,
-+ ref->ref_name);
++ err = reftable_reader_seek_ref(fri->r, &it,
++ ref->ref_name);
+ if (err == 0) {
+ err = reftable_iterator_next_ref(it, ref);
+ }
@@ reftable/iter.c (new)
+ reftable_iterator_destroy(&it);
+
+ if (err < 0) {
-+ return err;
++ break;
+ }
+
+ if (err > 0) {
@@ reftable/iter.c (new)
+ return 0;
+ }
+ }
++
++ reftable_ref_record_clear(ref);
++ return err;
+}
+
+struct reftable_iterator_vtable filtering_ref_iterator_vtable = {
@@ reftable/iter.c (new)
+void iterator_from_filtering_ref_iterator(struct reftable_iterator *it,
+ struct filtering_ref_iterator *fri)
+{
++ assert(it->ops == NULL);
+ it->iter_arg = fri;
+ it->ops = &filtering_ref_iterator_vtable;
+}
@@ reftable/iter.c (new)
+void iterator_from_indexed_table_ref_iter(struct reftable_iterator *it,
+ struct indexed_table_ref_iter *itr)
+{
++ assert(it->ops == NULL);
+ it->iter_arg = itr;
+ it->ops = &indexed_table_ref_iter_vtable;
+}
@@ reftable/iter.h (new)
+
+void iterator_set_empty(struct reftable_iterator *it);
+int iterator_next(struct reftable_iterator it, struct record rec);
++
++/* Returns true for a zeroed out iterator, such as the one returned from
++ iterator_destroy. */
+bool iterator_is_null(struct reftable_iterator it);
+
+/* iterator that produces only ref records that point to `oid` */
@@ reftable/merged.c (new)
+ reftable_free(mi->stack);
+}
+
-+static int merged_iter_advance_subiter(struct merged_iter *mi, int idx)
++static int merged_iter_advance_subiter(struct merged_iter *mi, size_t idx)
+{
+ if (iterator_is_null(mi->stack[idx])) {
+ return 0;
@@ reftable/merged.c (new)
+static void iterator_from_merged_iter(struct reftable_iterator *it,
+ struct merged_iter *mi)
+{
++ assert(it->ops == NULL);
+ it->iter_arg = mi;
+ it->ops = &merged_iter_vtable;
+}
@@ reftable/merged.c (new)
+ return mt->min;
+}
+
-+static int merged_table_seek_record(struct reftable_merged_table *mt,
-+ struct reftable_iterator *it,
-+ struct record rec)
++int merged_table_seek_record(struct reftable_merged_table *mt,
++ struct reftable_iterator *it, struct record rec)
+{
+ struct reftable_iterator *iters = reftable_calloc(
+ sizeof(struct reftable_iterator) * mt->stack_len);
@@ reftable/merged.h (new)
+};
+
+void merged_table_clear(struct reftable_merged_table *mt);
++int merged_table_seek_record(struct reftable_merged_table *mt,
++ struct reftable_iterator *it, struct record rec);
+
+#endif
@@ reftable/reader.c (new)
+static void iterator_from_table_iter(struct reftable_iterator *it,
+ struct table_iter *ti)
+{
++ assert(it->ops == NULL);
+ it->iter_arg = ti;
+ it->ops = &table_iter_vtable;
+}
@@ reftable/reader.c (new)
+ if (err == 0) {
+ *p = rd;
+ } else {
++ block_source_close(&src);
+ reftable_free(rd);
+ }
+ return err;
@@ reftable/reader.c (new)
+ struct record got_rec = { 0 };
+ int err = 0;
+
++ /* Look through the reverse index. */
+ record_from_obj(&want_rec, &want);
-+
+ err = reader_seek(r, &oit, want_rec);
+ if (err != 0) {
-+ return err;
++ goto exit;
+ }
+
++ /* read out the obj_record */
+ record_from_obj(&got_rec, &got);
+ err = iterator_next(oit, got_rec);
-+ reftable_iterator_destroy(&oit);
+ if (err < 0) {
-+ return err;
++ goto exit;
+ }
+
+ if (err > 0 ||
+ memcmp(want.hash_prefix, got.hash_prefix, r->object_id_len)) {
++ /* didn't find it; return empty iterator */
+ iterator_set_empty(it);
-+ return 0;
++ err = 0;
++ goto exit;
+ }
+
+ {
@@ reftable/reader.c (new)
+ hash_size(r->hash_id),
+ got.offsets, got.offset_len);
+ if (err < 0) {
-+ record_clear(got_rec);
-+ return err;
++ goto exit;
+ }
+ got.offsets = NULL;
-+ record_clear(got_rec);
-+
+ iterator_from_indexed_table_ref_iter(it, itr);
+ }
+
-+ return 0;
++exit:
++ reftable_iterator_destroy(&oit);
++ record_clear(got_rec);
++ return err;
+}
+
+static int reftable_reader_refs_for_unindexed(struct reftable_reader *r,
@@ reftable/record.c (new)
+ memcpy(r->message, dest.buf, dest.len);
+ r->message[dest.len] = 0;
+
++ slice_clear(&dest);
+ return start.len - in.len;
+
+error:
@@ reftable/record.c (new)
+
+struct record new_record(byte typ)
+{
-+ struct record rec;
++ struct record rec = { NULL };
+ switch (typ) {
+ case BLOCK_TYPE_REF: {
+ struct reftable_ref_record *r =
@@ reftable/record.c (new)
+
+void record_clear(struct record rec)
+{
-+ return rec.ops->clear(rec.data);
++ rec.ops->clear(rec.data);
+}
+
+bool record_is_deletion(struct record rec)
@@ reftable/record.c (new)
+
+void record_from_ref(struct record *rec, struct reftable_ref_record *ref_rec)
+{
++ assert(rec->ops == NULL);
+ rec->data = ref_rec;
+ rec->ops = &reftable_ref_record_vtable;
+}
+
+void record_from_obj(struct record *rec, struct obj_record *obj_rec)
+{
++ assert(rec->ops == NULL);
+ rec->data = obj_rec;
+ rec->ops = &obj_record_vtable;
+}
+
+void record_from_index(struct record *rec, struct index_record *index_rec)
+{
++ assert(rec->ops == NULL);
+ rec->data = index_rec;
+ rec->ops = &index_record_vtable;
+}
+
+void record_from_log(struct record *rec, struct reftable_log_record *log_rec)
+{
++ assert(rec->ops == NULL);
+ rec->data = log_rec;
+ rec->ops = &reftable_log_record_vtable;
+}
@@ reftable/record.h (new)
+
+#endif
+ ## reftable/refname.c (new) ##
+@@
++/*
++ Copyright 2020 Google LLC
++
++ Use of this source code is governed by a BSD-style
++ license that can be found in the LICENSE file or at
++ https://developers.google.com/open-source/licenses/bsd
++*/
++
++#include "system.h"
++#include "reftable.h"
++#include "basics.h"
++#include "refname.h"
++#include "slice.h"
++
++struct find_arg {
++ char **names;
++ const char *want;
++};
++
++static int find_name(size_t k, void *arg)
++{
++ struct find_arg *f_arg = (struct find_arg *)arg;
++
++ return strcmp(f_arg->names[k], f_arg->want) >= 0;
++}
++
++int modification_has_ref(struct modification *mod, const char *name)
++{
++ struct reftable_ref_record ref = { 0 };
++ int err = 0;
++
++ if (mod->add_len > 0) {
++ struct find_arg arg = {
++ .names = mod->add,
++ .want = name,
++ };
++ int idx = binsearch(mod->add_len, find_name, &arg);
++ if (idx < mod->add_len && !strcmp(mod->add[idx], name)) {
++ return 0;
++ }
++ }
++
++ if (mod->del_len > 0) {
++ struct find_arg arg = {
++ .names = mod->del,
++ .want = name,
++ };
++ int idx = binsearch(mod->del_len, find_name, &arg);
++ if (idx < mod->del_len && !strcmp(mod->del[idx], name)) {
++ return 1;
++ }
++ }
++
++ err = reftable_table_read_ref(mod->tab, name, &ref);
++ reftable_ref_record_clear(&ref);
++ return err;
++}
++
++static void modification_clear(struct modification *mod)
++{
++ FREE_AND_NULL(mod->add);
++ FREE_AND_NULL(mod->del);
++ mod->add_len = 0;
++ mod->del_len = 0;
++}
++
++int modification_has_ref_with_prefix(struct modification *mod,
++ const char *prefix)
++{
++ struct reftable_iterator it = { NULL };
++ struct reftable_ref_record ref = { NULL };
++ int err = 0;
++
++ if (mod->add_len > 0) {
++ struct find_arg arg = {
++ .names = mod->add,
++ .want = prefix,
++ };
++ int idx = binsearch(mod->add_len, find_name, &arg);
++ if (idx < mod->add_len &&
++ !strncmp(prefix, mod->add[idx], strlen(prefix))) {
++ goto exit;
++ }
++ }
++
++ err = reftable_table_seek_ref(mod->tab, &it, prefix);
++ if (err) {
++ goto exit;
++ }
++
++ while (true) {
++ err = reftable_iterator_next_ref(it, &ref);
++ if (err) {
++ goto exit;
++ }
++
++ if (mod->del_len > 0) {
++ struct find_arg arg = {
++ .names = mod->del,
++ .want = ref.ref_name,
++ };
++ int idx = binsearch(mod->del_len, find_name, &arg);
++ if (idx < mod->del_len &&
++ !strcmp(ref.ref_name, mod->del[idx])) {
++ continue;
++ }
++ }
++
++ if (strncmp(ref.ref_name, prefix, strlen(prefix))) {
++ err = 1;
++ goto exit;
++ }
++ err = 0;
++ goto exit;
++ }
++
++exit:
++ reftable_ref_record_clear(&ref);
++ reftable_iterator_destroy(&it);
++ return err;
++}
++
++int validate_ref_name(const char *name)
++{
++ while (true) {
++ char *next = strchr(name, '/');
++ if (!*name) {
++ return REFTABLE_REFNAME_ERROR;
++ }
++ if (!next) {
++ return 0;
++ }
++ if (next - name == 0 || (next - name == 1 && *name == '.') ||
++ (next - name == 2 && name[0] == '.' && name[1] == '.'))
++ return REFTABLE_REFNAME_ERROR;
++ name = next + 1;
++ }
++ return 0;
++}
++
++int validate_ref_record_addition(struct reftable_table tab,
++ struct reftable_ref_record *recs, size_t sz)
++{
++ struct modification mod = {
++ .tab = tab,
++ .add = reftable_calloc(sizeof(char *) * sz),
++ .del = reftable_calloc(sizeof(char *) * sz),
++ };
++ int i = 0;
++ int err = 0;
++ for (; i < sz; i++) {
++ if (reftable_ref_record_is_deletion(&recs[i])) {
++ mod.del[mod.del_len++] = recs[i].ref_name;
++ } else {
++ mod.add[mod.add_len++] = recs[i].ref_name;
++ }
++ }
++
++ err = modification_validate(&mod);
++ modification_clear(&mod);
++ return err;
++}
++
++static void slice_trim_component(struct slice *sl)
++{
++ while (sl->len > 0) {
++ bool is_slash = (sl->buf[sl->len - 1] == '/');
++ sl->len--;
++ if (is_slash)
++ break;
++ }
++}
++
++int modification_validate(struct modification *mod)
++{
++ struct slice slashed = { 0 };
++ int err = 0;
++ int i = 0;
++ for (; i < mod->add_len; i++) {
++ err = validate_ref_name(mod->add[i]);
++ if (err) {
++ goto exit;
++ }
++ slice_set_string(&slashed, mod->add[i]);
++ slice_append_string(&slashed, "/");
++
++ err = modification_has_ref_with_prefix(
++ mod, slice_as_string(&slashed));
++ if (err == 0) {
++ err = REFTABLE_NAME_CONFLICT;
++ goto exit;
++ }
++ if (err < 0) {
++ goto exit;
++ }
++
++ slice_set_string(&slashed, mod->add[i]);
++ while (slashed.len) {
++ slice_trim_component(&slashed);
++ err = modification_has_ref(mod,
++ slice_as_string(&slashed));
++ if (err == 0) {
++ err = REFTABLE_NAME_CONFLICT;
++ goto exit;
++ }
++ if (err < 0) {
++ goto exit;
++ }
++ }
++ }
++ err = 0;
++exit:
++ slice_clear(&slashed);
++ return err;
++}
+
+ ## reftable/refname.h (new) ##
+@@
++/*
++ Copyright 2020 Google LLC
++
++ Use of this source code is governed by a BSD-style
++ license that can be found in the LICENSE file or at
++ https://developers.google.com/open-source/licenses/bsd
++*/
++#ifndef REFNAME_H
++#define REFNAME_H
++
++struct modification {
++ struct reftable_table tab;
++
++ char **add;
++ size_t add_len;
++
++ char **del;
++ size_t del_len;
++};
++
++// -1 = error, 0 = found, 1 = not found
++int modification_has_ref(struct modification *mod, const char *name);
++
++// -1 = error, 0 = found, 1 = not found.
++int modification_has_ref_with_prefix(struct modification *mod,
++ const char *prefix);
++
++// 0 = OK.
++int validate_ref_name(const char *name);
++
++int validate_ref_record_addition(struct reftable_table tab,
++ struct reftable_ref_record *recs, size_t sz);
++
++int modification_validate(struct modification *mod);
++
++/* illegal name, or dir/file conflict */
++#define REFTABLE_REFNAME_ERROR -9
++
++#endif
+
+ ## reftable/reftable.c (new) ##
+@@
++/*
++Copyright 2020 Google LLC
++
++Use of this source code is governed by a BSD-style
++license that can be found in the LICENSE file or at
++https://developers.google.com/open-source/licenses/bsd
++*/
++
++#include "reftable.h"
++#include "record.h"
++#include "reader.h"
++#include "merged.h"
++
++struct reftable_table_vtable {
++ int (*seek)(void *tab, struct reftable_iterator *it, struct record);
++};
++
++static int reftable_reader_seek_void(void *tab, struct reftable_iterator *it,
++ struct record rec)
++{
++ return reader_seek((struct reftable_reader *)tab, it, rec);
++}
++
++static struct reftable_table_vtable reader_vtable = {
++ .seek = reftable_reader_seek_void,
++};
++
++static int reftable_merged_table_seek_void(void *tab,
++ struct reftable_iterator *it,
++ struct record rec)
++{
++ return merged_table_seek_record((struct reftable_merged_table *)tab, it,
++ rec);
++}
++
++static struct reftable_table_vtable merged_table_vtable = {
++ .seek = reftable_merged_table_seek_void,
++};
++
++int reftable_table_seek_ref(struct reftable_table tab,
++ struct reftable_iterator *it, const char *name)
++{
++ struct reftable_ref_record ref = {
++ .ref_name = (char *)name,
++ };
++ struct record rec = { 0 };
++ record_from_ref(&rec, &ref);
++ return tab.ops->seek(tab.table_arg, it, rec);
++}
++
++void reftable_table_from_reader(struct reftable_table *tab,
++ struct reftable_reader *reader)
++{
++ assert(tab->ops == NULL);
++ tab->ops = &reader_vtable;
++ tab->table_arg = reader;
++}
++
++void reftable_table_from_merged_table(struct reftable_table *tab,
++ struct reftable_merged_table *merged)
++{
++ assert(tab->ops == NULL);
++ tab->ops = &merged_table_vtable;
++ tab->table_arg = merged;
++}
++
++int reftable_table_read_ref(struct reftable_table tab, const char *name,
++ struct reftable_ref_record *ref)
++{
++ struct reftable_iterator it = { 0 };
++ int err = reftable_table_seek_ref(tab, &it, name);
++ if (err) {
++ goto exit;
++ }
++
++ err = reftable_iterator_next_ref(it, ref);
++ if (err) {
++ goto exit;
++ }
++
++ if (strcmp(ref->ref_name, name) ||
++ reftable_ref_record_is_deletion(ref)) {
++ reftable_ref_record_clear(ref);
++ err = 1;
++ goto exit;
++ }
++
++exit:
++ reftable_iterator_destroy(&it);
++ return err;
++}
+
## reftable/reftable.h (new) ##
@@
+/*
@@ reftable/reftable.h (new)
+
+ /* Wrote a table without blocks. */
+ REFTABLE_EMPTY_TABLE_ERROR = -8,
++
++ /* Dir/file conflict. */
++ REFTABLE_NAME_CONFLICT = -9,
++
++ /* Illegal ref name. */
++ REFTABLE_REFNAME_ERROR = -10,
+};
+
+/* convert the numeric error code to a string. The string should not be
@@ reftable/reftable.h (new)
+ * Defaults to SHA1 if unset
+ */
+ uint32_t hash_id;
++
++ /* boolean: do not check ref names for validity or dir/file conflicts.
++ */
++ int skip_name_check;
+};
+
+/* reftable_block_stats holds statistics for a single block type */
@@ reftable/reftable.h (new)
+
+/* reftable_new_writer creates a new writer */
+struct reftable_writer *
-+reftable_new_writer(int (*writer_func)(void *, uint8_t *, int),
++reftable_new_writer(int (*writer_func)(void *, uint8_t *, size_t),
+ void *writer_arg, struct reftable_write_options *opts);
+
+/* write to a file descriptor. fdp should be an int* pointing to the fd. */
-+int reftable_fd_write(void *fdp, uint8_t *data, int size);
++int reftable_fd_write(void *fdp, uint8_t *data, size_t size);
+
+/* Set the range of update indices for the records we will add. When
+ writing a table into a stack, the min should be at least
@@ reftable/reftable.h (new)
+struct reftable_reader;
+
+/* reftable_new_reader opens a reftable for reading. If successful, returns 0
-+ * code and sets pp. The name is used for creating a
-+ * stack. Typically, it is the basename of the file.
++ * code and sets pp. The name is used for creating a stack. Typically, it is the
++ * basename of the file. The block source `src` is owned by the reader, and is
++ * closed on calling reftable_reader_destroy().
+ */
+int reftable_new_reader(struct reftable_reader **pp,
-+ struct reftable_block_source, const char *name);
++ struct reftable_block_source src, const char *name);
+
+/* reftable_reader_seek_ref returns an iterator where 'name' would be inserted
+ in the table. To seek to the start of the table, use name = "".
@@ reftable/reftable.h (new)
+void reftable_merged_table_free(struct reftable_merged_table *m);
+
+/****************************************************************
++ Generic tables
++
++ A unified API for reading tables, either merged tables, or single readers.
++ ****************************************************************/
++
++struct reftable_table {
++ struct reftable_table_vtable *ops;
++ void *table_arg;
++};
++
++int reftable_table_seek_ref(struct reftable_table tab,
++ struct reftable_iterator *it, const char *name);
++
++void reftable_table_from_reader(struct reftable_table *tab,
++ struct reftable_reader *reader);
++void reftable_table_from_merged_table(struct reftable_table *tab,
++ struct reftable_merged_table *table);
++
++/* convenience function to read a single ref. Returns < 0 for error, 0
++ for success, and 1 if ref not found. */
++int reftable_table_read_ref(struct reftable_table tab, const char *name,
++ struct reftable_ref_record *ref);
++
++/****************************************************************
+ Mutable ref database
+
+ The stack presents an interface to a mutable sequence of reftables.
@@ reftable/reftable.h (new)
+/* statistics on past compactions. */
+struct reftable_compaction_stats {
+ uint64_t bytes; /* total number of bytes written */
++ uint64_t entries_written; /* total number of entries written, including
++ failures. */
+ int attempts; /* how often we tried to compact */
+ int failures; /* failures happen on concurrent updates */
+};
@@ reftable/slice.c (new)
+ }
+}
+
-+int slice_write(struct slice *b, byte *data, int sz)
++int slice_write(struct slice *b, byte *data, size_t sz)
+{
+ if (b->len + sz > b->cap) {
+ int newcap = 2 * b->cap + 1;
@@ reftable/slice.c (new)
+ return sz;
+}
+
-+int slice_write_void(void *b, byte *data, int sz)
++int slice_write_void(void *b, byte *data, size_t sz)
+{
+ return slice_write((struct slice *)b, data, sz);
+}
@@ reftable/slice.c (new)
+void block_source_from_slice(struct reftable_block_source *bs,
+ struct slice *buf)
+{
++ assert(bs->ops == NULL);
+ bs->ops = &slice_vtable;
+ bs->arg = buf;
+}
@@ reftable/slice.h (new)
+int slice_compare(struct slice a, struct slice b);
+
+/* Append `data` to the `dest` slice. */
-+int slice_write(struct slice *dest, byte *data, int sz);
++int slice_write(struct slice *dest, byte *data, size_t sz);
+
+/* Append `add` to `dest. */
+void slice_append(struct slice *dest, struct slice add);
+
+/* Like slice_write, but suitable for passing to reftable_new_writer
+ */
-+int slice_write_void(void *b, byte *data, int sz);
++int slice_write_void(void *b, byte *data, size_t sz);
+
+/* Find the longest shared prefix size of `a` and `b` */
+int common_prefix_size(struct slice a, struct slice b);
@@ reftable/stack.c (new)
+#include "system.h"
+#include "merged.h"
+#include "reader.h"
++#include "refname.h"
+#include "reftable.h"
+#include "writer.h"
+
@@ reftable/stack.c (new)
+{
+ struct reftable_stack *p =
+ reftable_calloc(sizeof(struct reftable_stack));
-+ struct slice list_file_name = {};
++ struct slice list_file_name = { 0 };
+ int err = 0;
+
+ if (config.hash_id == 0) {
@@ reftable/stack.c (new)
+ }
+ }
+ }
++
+exit:
+ slice_clear(&table_path);
+ {
@@ reftable/stack.c (new)
+ free_names(names);
+ free_names(names_after);
+
-+ delay = delay + (delay * rand()) / RAND_MAX + 100;
-+ usleep(delay);
++ delay = delay + (delay * rand()) / RAND_MAX + 1;
++ sleep_millisec(delay);
+ }
+
+ return 0;
@@ reftable/stack.c (new)
+ int err = stack_try_add(st, write, arg);
+ if (err < 0) {
+ if (err == REFTABLE_LOCK_ERROR) {
-+ // Ignore error return, we want to propagate
-+ // REFTABLE_LOCK_ERROR.
++ /* Ignore error return, we want to propagate
++ REFTABLE_LOCK_ERROR.
++ */
+ reftable_stack_reload(st);
+ }
+ return err;
@@ reftable/stack.c (new)
+void reftable_addition_close(struct reftable_addition *add)
+{
+ int i = 0;
-+ struct slice nm = {};
++ struct slice nm = { 0 };
+ for (i = 0; i < add->new_tables_len; i++) {
+ slice_set_string(&nm, add->stack->list_file);
+ slice_append_string(&nm, "/");
+ slice_append_string(&nm, add->new_tables[i]);
+ unlink(slice_as_string(&nm));
-+
+ reftable_free(add->new_tables[i]);
+ add->new_tables[i] = NULL;
+ }
@@ reftable/stack.c (new)
+
+ free_names(add->names);
+ add->names = NULL;
++ slice_clear(&nm);
+}
+
+int reftable_addition_commit(struct reftable_addition *add)
@@ reftable/stack.c (new)
+ struct reftable_writer *wr = NULL;
+ int err = 0;
+ int tab_fd = 0;
-+ uint64_t next_update_index = 0;
+
+ slice_resize(&next_name, 0);
-+ format_name(&next_name, next_update_index, next_update_index);
++ format_name(&next_name, add->next_update_index, add->next_update_index);
+
+ slice_set_string(&temp_tab_file_name, add->stack->reftable_dir);
+ slice_append_string(&temp_tab_file_name, "/");
@@ reftable/stack.c (new)
+ goto exit;
+ }
+
-+ if (wr->min_update_index < next_update_index) {
++ err = stack_check_addition(add->stack,
++ slice_as_string(&temp_tab_file_name));
++ if (err < 0) {
++ goto exit;
++ }
++
++ if (wr->min_update_index < add->next_update_index) {
+ err = REFTABLE_API_ERROR;
+ goto exit;
+ }
@@ reftable/stack.c (new)
+ struct reftable_ref_record ref = { 0 };
+ struct reftable_log_record log = { 0 };
+
++ uint64_t entries = 0;
++
+ int i = 0, j = 0;
+ for (i = first, j = 0; i <= last; i++) {
+ struct reftable_reader *t = st->merged->stack[i];
@@ reftable/stack.c (new)
+ if (err < 0) {
+ break;
+ }
++ entries++;
+ }
++ reftable_iterator_destroy(&it);
+
+ err = reftable_merged_table_seek_log(mt, &it, "");
+ if (err < 0) {
@@ reftable/stack.c (new)
+ continue;
+ }
+
-+ /* XXX collect stats? */
-+
+ if (config != NULL && config->time > 0 &&
+ log.time < config->time) {
+ continue;
@@ reftable/stack.c (new)
+ if (err < 0) {
+ break;
+ }
++ entries++;
+ }
+
+exit:
@@ reftable/stack.c (new)
+ reftable_merged_table_free(mt);
+ }
+ reftable_ref_record_clear(&ref);
-+
++ reftable_log_record_clear(&log);
++ st->stats.entries_written += entries;
+ return err;
+}
+
@@ reftable/stack.c (new)
+ }
+ goto exit;
+ }
++ /* Don't want to write to the lock for now. */
++ close(lock_file_fd);
++ lock_file_fd = 0;
++
+ have_lock = true;
+ err = stack_uptodate(st);
+ if (err != 0) {
@@ reftable/stack.c (new)
+ }
+ have_lock = false;
+
++ /* Reload the stack before deleting. On windows, we can only delete the
++ files after we closed them.
++ */
++ err = reftable_stack_reload_maybe_reuse(st, first < last);
++
+ {
+ char **p = delete_on_success;
+ while (*p) {
@@ reftable/stack.c (new)
+ }
+ }
+
-+ err = reftable_stack_reload_maybe_reuse(st, first < last);
+exit:
+ free_names(delete_on_success);
+ {
@@ reftable/stack.c (new)
+int fastlog2(uint64_t sz)
+{
+ int l = 0;
-+ assert(sz > 0);
++ if (sz == 0) {
++ return 0;
++ }
+ for (; sz; sz /= 2) {
+ l++;
+ }
@@ reftable/stack.c (new)
+ int next = 0;
+ struct segment cur = { 0 };
+ int i = 0;
++
++ if (n == 0) {
++ *seglen = 0;
++ return segs;
++ }
+ for (i = 0; i < n; i++) {
+ int log = fastlog2(sizes[i]);
+ if (cur.log != log && cur.bytes > 0) {
@@ reftable/stack.c (new)
+ cur.end = i + 1;
+ cur.bytes += sizes[i];
+ }
-+ if (next > 0) {
-+ segs[next++] = cur;
-+ }
++ segs[next++] = cur;
+ *seglen = next;
+ return segs;
+}
@@ reftable/stack.c (new)
+ uint64_t *sizes =
+ reftable_calloc(sizeof(uint64_t) * st->merged->stack_len);
+ int version = (st->config.hash_id == SHA1_ID) ? 1 : 2;
-+ int overhead = footer_size(version) + header_size(version) - 1;
++ int overhead = header_size(version) - 1;
+ int i = 0;
+ for (i = 0; i < st->merged->stack_len; i++) {
+ sizes[i] = st->merged->stack[i]->size - overhead;
@@ reftable/stack.c (new)
+int reftable_stack_read_ref(struct reftable_stack *st, const char *refname,
+ struct reftable_ref_record *ref)
+{
++ struct reftable_table tab = { NULL };
++ reftable_table_from_merged_table(&tab, reftable_stack_merged_table(st));
++ return reftable_table_read_ref(tab, refname, ref);
++}
++
++int reftable_stack_read_log(struct reftable_stack *st, const char *refname,
++ struct reftable_log_record *log)
++{
+ struct reftable_iterator it = { 0 };
+ struct reftable_merged_table *mt = reftable_stack_merged_table(st);
-+ int err = reftable_merged_table_seek_ref(mt, &it, refname);
++ int err = reftable_merged_table_seek_log(mt, &it, refname);
+ if (err) {
+ goto exit;
+ }
+
-+ err = reftable_iterator_next_ref(it, ref);
++ err = reftable_iterator_next_log(it, log);
+ if (err) {
+ goto exit;
+ }
+
-+ if (strcmp(ref->ref_name, refname) ||
-+ reftable_ref_record_is_deletion(ref)) {
++ if (strcmp(log->ref_name, refname) ||
++ reftable_log_record_is_deletion(log)) {
+ err = 1;
+ goto exit;
+ }
+
+exit:
++ if (err) {
++ reftable_log_record_clear(log);
++ }
+ reftable_iterator_destroy(&it);
+ return err;
+}
+
-+int reftable_stack_read_log(struct reftable_stack *st, const char *refname,
-+ struct reftable_log_record *log)
++int stack_check_addition(struct reftable_stack *st, const char *new_tab_name)
+{
-+ struct reftable_iterator it = { 0 };
-+ struct reftable_merged_table *mt = reftable_stack_merged_table(st);
-+ int err = reftable_merged_table_seek_log(mt, &it, refname);
-+ if (err) {
++ int err = 0;
++ struct reftable_block_source src = { 0 };
++ struct reftable_reader *rd = NULL;
++ struct reftable_table tab = { NULL };
++ struct reftable_ref_record *refs = NULL;
++ struct reftable_iterator it = { NULL };
++ int cap = 0;
++ int len = 0;
++ int i = 0;
++
++ if (st->config.skip_name_check) {
++ return 0;
++ }
++
++ err = reftable_block_source_from_file(&src, new_tab_name);
++ if (err < 0) {
+ goto exit;
+ }
+
-+ err = reftable_iterator_next_log(it, log);
-+ if (err) {
++ err = reftable_new_reader(&rd, src, new_tab_name);
++ if (err < 0) {
+ goto exit;
+ }
+
-+ if (strcmp(log->ref_name, refname) ||
-+ reftable_log_record_is_deletion(log)) {
-+ err = 1;
++ err = reftable_reader_seek_ref(rd, &it, "");
++ if (err > 0) {
++ err = 0;
+ goto exit;
+ }
++ if (err < 0) {
++ goto exit;
++ }
++
++ while (true) {
++ struct reftable_ref_record ref = { 0 };
++ err = reftable_iterator_next_ref(it, &ref);
++ if (err > 0) {
++ break;
++ }
++ if (err < 0) {
++ goto exit;
++ }
++
++ if (len >= cap) {
++ cap = 2 * cap + 1;
++ refs = reftable_realloc(refs, cap * sizeof(refs[0]));
++ }
++
++ refs[len++] = ref;
++ }
++
++ reftable_table_from_merged_table(&tab, reftable_stack_merged_table(st));
++
++ err = validate_ref_record_addition(tab, refs, len);
++
++ for (i = 0; i < len; i++) {
++ reftable_ref_record_clear(&refs[i]);
++ }
+
+exit:
++ free(refs);
+ reftable_iterator_destroy(&it);
++ reftable_reader_free(rd);
+ return err;
+}
@@ reftable/stack.h (new)
+ int first, int last,
+ struct reftable_log_expiry_config *config);
+int fastlog2(uint64_t sz);
++int stack_check_addition(struct reftable_stack *st, const char *new_tab_name);
+
+struct segment {
+ int start, end;
@@ reftable/system.h (new)
+#if 1 /* REFTABLE_IN_GITCORE */
+
+#include "git-compat-util.h"
++#include "cache.h"
+#include <zlib.h>
+
+#else
@@ reftable/update.sh (new)
+set -eu
+
+# Override this to import from somewhere else, say "../reftable".
-+SRC=${SRC:-origin} BRANCH=${BRANCH:-origin/master}
++SRC=${SRC:-origin}
++BRANCH=${BRANCH:-master}
+
-+((git --git-dir reftable-repo/.git fetch ${SRC} && cd reftable-repo && git checkout ${BRANCH} ) ||
++((git --git-dir reftable-repo/.git fetch ${SRC} ${BRANCH}:import && cd reftable-repo && git checkout -f $(git rev-parse import) ) ||
+ git clone https://github.com/google/reftable reftable-repo)
+
+cp reftable-repo/c/*.[ch] reftable/
+cp reftable-repo/c/include/*.[ch] reftable/
+cp reftable-repo/LICENSE reftable/
-+git --git-dir reftable-repo/.git show --no-patch origin/master \
++git --git-dir reftable-repo/.git show --no-patch HEAD \
+ > reftable/VERSION
+
+mv reftable/system.h reftable/system.h~
@@ reftable/writer.c (new)
+}
+
+struct reftable_writer *
-+reftable_new_writer(int (*writer_func)(void *, byte *, int), void *writer_arg,
-+ struct reftable_write_options *opts)
++reftable_new_writer(int (*writer_func)(void *, byte *, size_t),
++ void *writer_arg, struct reftable_write_options *opts)
+{
+ struct reftable_writer *wp =
+ reftable_calloc(sizeof(struct reftable_writer));
@@ reftable/writer.c (new)
+ }
+ w->pending_padding = 0;
+ if (empty_table) {
-+ // Empty tables need a header anyway.
++ /* Empty tables need a header anyway. */
+ byte header[28];
+ int n = writer_write_header(w, header);
+ err = padded_write(w, header, n, 0);
@@ reftable/writer.h (new)
+#include "tree.h"
+
+struct reftable_writer {
-+ int (*write)(void *, byte *, int);
++ int (*write)(void *, byte *, size_t);
+ void *write_arg;
+ int pending_padding;
+ struct slice last_key;
10: be2371cd6e4 ! 9: d731e1669b7 Reftable support for git-core
@@ Commit message
* Resolve spots marked with XXX
- * Detect and prevent directory/file conflicts in naming.
-
* Support worktrees (t0002-gitfile "linked repo" testcase)
Example use: see t/t0031-reftable.sh
@@ Makefile: TEST_SHELL_PATH = $(SHELL_PATH)
VCSSVN_LIB = vcs-svn/lib.a
+REFTABLE_LIB = reftable/libreftable.a
+ GENERATED_H += config-list.h
GENERATED_H += command-list.h
-
-@@ Makefile: LIB_OBJS += rebase-interactive.o
+@@ Makefile: LIB_OBJS += ref-filter.o
LIB_OBJS += reflog-walk.o
LIB_OBJS += refs.o
LIB_OBJS += refs/files-backend.o
@@ Makefile: THIRD_PARTY_SOURCES += compat/regex/%
EXTLIBS =
GIT_USER_AGENT = git/$(GIT_VERSION)
-@@ Makefile: VCSSVN_OBJS += vcs-svn/fast_export.o
+@@ Makefile: VCSSVN_OBJS += vcs-svn/sliding_window.o
VCSSVN_OBJS += vcs-svn/svndiff.o
VCSSVN_OBJS += vcs-svn/svndump.o
+REFTABLE_OBJS += reftable/basics.o
+REFTABLE_OBJS += reftable/block.o
-+REFTABLE_OBJS += reftable/bytes.o
+REFTABLE_OBJS += reftable/file.o
+REFTABLE_OBJS += reftable/iter.o
+REFTABLE_OBJS += reftable/merged.o
+REFTABLE_OBJS += reftable/pq.o
+REFTABLE_OBJS += reftable/reader.o
+REFTABLE_OBJS += reftable/record.o
++REFTABLE_OBJS += reftable/refname.o
++REFTABLE_OBJS += reftable/reftable.o
+REFTABLE_OBJS += reftable/slice.o
+REFTABLE_OBJS += reftable/stack.o
+REFTABLE_OBJS += reftable/tree.o
@@ builtin/init-db.c: static int create_default_files(const char *template_path,
+ reinit = (!access(path, R_OK) ||
+ readlink(path, junk, sizeof(junk) - 1) != -1);
+
-+ /*
-+ * refs/heads is a file when using reftable. We can't reinitialize with
-+ * a reftable because it will overwrite HEAD
-+ */
++ /*
++ * refs/heads is a file when using reftable. We can't reinitialize with
++ * a reftable because it will overwrite HEAD
++ */
+ if (reinit && (!strcmp(fmt->ref_storage, "reftable")) ==
+ is_directory(git_path_buf(&buf, "refs/heads"))) {
+ die("cannot switch ref storage format.");
@@ builtin/init-db.c: static int create_default_files(const char *template_path,
/*
* We need to create a "refs" dir in any case so that older
* versions of git can tell that this is a repository.
- */
--
- if (refs_init_db(&err))
- die("failed to set up refs db: %s", err.buf);
-
@@ builtin/init-db.c: static int create_default_files(const char *template_path,
* Create the default symlink from ".git/HEAD" to the "master"
* branch, if it does not exist yet.
@@ builtin/init-db.c: static int create_default_files(const char *template_path,
if (!reinit) {
if (create_symref("HEAD", "refs/heads/master", NULL) < 0)
exit(1);
-- }
--
-- initialize_repository_version(fmt->hash_algo);
-+ }
+ }
+- initialize_repository_version(fmt->hash_algo);
+ initialize_repository_version(fmt->hash_algo, fmt->ref_storage);
-+
+
/* Check filemode trustability */
path = git_path_buf(&buf, "config");
- filemode = TEST_FILEMODE;
@@ builtin/init-db.c: static void validate_hash_algorithm(struct repository_format *repo_fmt, int hash
}
@@ refs.c: struct ref_store *get_main_ref_store(struct repository *r)
BUG("attempting to get main_ref_store outside of repository");
- r->refs_private = ref_store_init(r->gitdir, REF_STORE_ALL_CAPS);
-+ r->refs_private = ref_store_init(
-+ r->gitdir,
-+ r->ref_storage_format ? r->ref_storage_format : DEFAULT_REF_STORAGE,
-+ REF_STORE_ALL_CAPS);
++ r->refs_private = ref_store_init(r->gitdir,
++ r->ref_storage_format ?
++ r->ref_storage_format :
++ DEFAULT_REF_STORAGE,
++ REF_STORE_ALL_CAPS);
return r->refs_private;
}
@@ refs/reftable-backend.c (new)
+ unsigned int store_flags;
+
+ int err;
-+ char *repo_dir;
++ char *repo_dir;
+ char *reftable_dir;
+ struct reftable_stack *stack;
+};
@@ refs/reftable-backend.c (new)
+
+ base_ref_store_init(ref_store, &refs_be_reftable);
+ refs->store_flags = store_flags;
-+ refs->repo_dir = xstrdup(path);
++ refs->repo_dir = xstrdup(path);
+ strbuf_addf(&sb, "%s/reftable", path);
+ refs->reftable_dir = xstrdup(sb.buf);
+ strbuf_reset(&sb);
@@ refs/reftable-backend.c (new)
+ strbuf_addf(&sb, "%s/refs/heads", refs->repo_dir);
+ write_file(sb.buf, "this repository uses the reftable format");
+
-+ return 0;
++ return 0;
+}
+
+struct git_reftable_iterator {
@@ refs/reftable-backend.c (new)
+ }
+
+ err = reftable_stack_read_ref(refs->stack, refname, &ref);
-+ if (err > 0) {
-+ errno = ENOENT;
-+ err = -1;
-+ goto exit;
-+ }
++ if (err > 0) {
++ errno = ENOENT;
++ err = -1;
++ goto exit;
++ }
+ if (err < 0) {
-+ errno = reftable_error_to_errno(err);
-+ err = -1;
++ errno = reftable_error_to_errno(err);
++ err = -1;
+ goto exit;
+ }
+ if (ref.target != NULL) {
@@ refs/reftable-backend.c (new)
+ } else if (ref.value != NULL) {
+ hashcpy(oid->hash, ref.value);
+ } else {
-+ *type |= REF_ISBROKEN;
-+ errno = EINVAL;
-+ err = -1;
-+ }
++ *type |= REF_ISBROKEN;
++ errno = EINVAL;
++ err = -1;
++ }
+exit:
+ reftable_ref_record_clear(&ref);
+ return err;
@@ refs/reftable-backend.c (new)
+ reftable_reflog_expire
+};
- ## reftable/update.sh ##
-@@ reftable/update.sh: SRC=${SRC:-origin} BRANCH=${BRANCH:-origin/master}
- cp reftable-repo/c/*.[ch] reftable/
- cp reftable-repo/c/include/*.[ch] reftable/
- cp reftable-repo/LICENSE reftable/
--git --git-dir reftable-repo/.git show --no-patch origin/master \
-+git --git-dir reftable-repo/.git show --no-patch ${BRANCH} \
- > reftable/VERSION
-
- mv reftable/system.h reftable/system.h~
-
## repository.c ##
@@ repository.c: int repo_init(struct repository *repo,
if (worktree)
@@ t/t0031-reftable.sh (new)
+ git checkout -b master
+'
+
++
++test_expect_success 'dir/file conflict' '
++ initialize &&
++ test_commit file &&
++ ! git branch master/forbidden
++'
++
++
+test_expect_success 'do not clobber existing repo' '
+ rm -rf .git &&
+ git init --ref-storage=files &&
@@ t/t0031-reftable.sh (new)
+ test_cmp expect actual
+'
+
++
+test_done
++
-: ----------- > 10: 513d585f0f8 vcxproj: adjust for the reftable changes
11: 8af67b85e49 ! 11: 846fe29fa4b Add some reftable testing infrastructure
@@ builtin/init-db.c: static const char *const init_db_usage[] = {
## refs.c ##
@@ refs.c: struct ref_store *get_main_ref_store(struct repository *r)
- if (!r->gitdir)
- BUG("attempting to get main_ref_store outside of repository");
-
-- r->refs_private = ref_store_init(
-- r->gitdir,
-- r->ref_storage_format ? r->ref_storage_format : DEFAULT_REF_STORAGE,
-- REF_STORE_ALL_CAPS);
-+ r->refs_private = ref_store_init(r->gitdir,
-+ r->ref_storage_format ?
-+ r->ref_storage_format :
+ r->refs_private = ref_store_init(r->gitdir,
+ r->ref_storage_format ?
+ r->ref_storage_format :
+- DEFAULT_REF_STORAGE,
+ default_ref_storage(),
-+ REF_STORE_ALL_CAPS);
+ REF_STORE_ALL_CAPS);
return r->refs_private;
}
-
@@ refs.c: struct ref_store *get_submodule_ref_store(const char *submodule)
goto done; 12: b02b83a92ad = 12: a7d1d2e721c t: use update-ref and show-ref to reading/writing refs
Signature · 1 line
gitgitgadget
PATCH v11 1/12 refs.h: clarify reflog iteration order
From: Han-Wen Nienhuys <[email protected]>
--- refs.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
refs.h +4 -1
@@ -432,18 +432,21 @@ int delete_refs(const char *msg, struct string_list *refnames,int refs_delete_reflog (struct ref_store *refs,const char *refname);int delete_reflog (const char *refname); -/* iterate over reflog entries */ +/* Iterate over reflog entries. */ typedef int each_reflog_ent_fn (struct object_id *old_oid,struct object_id *new_oid,const char *committer,timestamp_t timestamp,int tz,const char *msg,void *cb_data); +/* Iterate in over reflog entries, oldest entry first. */ int refs_for_each_reflog_ent (struct ref_store *refs,const char *refname,each_reflog_ent_fn fn,void *cb_data);int refs_for_each_reflog_ent_reverse (struct ref_store *refs,const char *refname,each_reflog_ent_fn fn,void *cb_data); + +/* Call a function for each reflog entry, oldest entry first. */ int for_each_reflog_ent (const char *refname,each_reflog_ent_fn fn,void *cb_data);int for_each_reflog_ent_reverse (const char *refname,each_reflog_ent_fn fn,void *cb_data);
Signature · 1 line
gitgitgadget
PATCH v11 2/12 Iterate over the "refs/" namespace in for_each_[raw]ref
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This happens implicitly in the files/packed ref backend; making it explicit simplifies adding alternate ref storage backends, such as reftable.
--- refs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
refs.c +3 -3
@@ -1525,7 +1525,7 @@ static int do_for_each_ref(struct ref_store *refs, const char *prefix,int refs_for_each_ref (struct ref_store *refs,each_ref_fn fn,void *cb_data) { -return do_for_each_ref (refs,"" , fn,0 ,0 , cb_data); +return do_for_each_ref (refs,"refs/" , fn,0 ,0 , cb_data); }int for_each_ref (each_ref_fn fn,void *cb_data)
@@ -1585,8 +1585,8 @@ int for_each_namespaced_ref(each_ref_fn fn, void *cb_data)int refs_for_each_rawref (struct ref_store *refs,each_ref_fn fn,void *cb_data) { -return do_for_each_ref (refs,"" , fn,0 , -DO_FOR_EACH_INCLUDE_BROKEN , cb_data); +return do_for_each_ref (refs,"refs/" , fn,0 ,DO_FOR_EACH_INCLUDE_BROKEN , + cb_data); }int for_each_rawref (each_ref_fn fn,void *cb_data)
Signature · 1 line
gitgitgadget
PATCH v11 3/12 refs: document how ref_iterator_advance_fn should handle symrefs
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
--- refs/refs-internal.h | 5 +++++ 1 file changed, 5 insertions(+)
refs/refs-internal.h +5 -0
@@ -438,6 +438,11 @@ void base_ref_iterator_free(struct ref_iterator *iter);/* Virtual function declarations for ref_iterators: */ +/* +* backend-specific implementation of ref_iterator_advance. +* For symrefs, the function should set REF_ISSYMREF, and it should also +* dereference the symref to provide the OID referent. +*/ typedef int ref_iterator_advance_fn (struct ref_iterator *ref_iterator); typedef int ref_iterator_peel_fn(struct ref_iterator *ref_iterator,
Signature · 1 line
gitgitgadget
PATCH v11 4/12 Add .gitattributes for the reftable/ directory
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
--- reftable/.gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 reftable/.gitattributes
reftable/.gitattributes new file +1 -0
@@ -0,0 +1 @@ +/zlib-compat.c whitespace=-indent-with-non-tab,-trailing-space
Signature · 1 line
gitgitgadget
PATCH v11 5/12 reftable: file format documentation
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Jonathan Nieder <[email protected]>
Shawn Pearce explains:
Some repositories contain a lot of references (e.g. android at 866k, rails at 31k). The reftable format provides:
- Near constant time lookup for any single reference, even when the repository is cold and not in process or kernel cache.
- Near constant time verification a SHA-1 is referred to by at least one reference (for allow-tip-sha1-in-want).
- Efficient lookup of an entire namespace, such as refs/tags/.
- Support atomic push O(size_of_update) operations.
- Combine reflog storage with ref storage.
This file format spec was originally written in July, 2017 by Shawn Pearce. Some refinements since then were made by Shawn and by Han-Wen Nienhuys based on experiences implementing and experimenting with the format. (All of this was in the context of our work at Google and Google is happy to contribute the result to the Git project.)
Imported from JGit[1]'s current version (c217d33ff, "Documentation/technical/reftable: improve repo layout", 2020-02-04) of Documentation/technical/reftable.md and converted to asciidoc by running
pandoc -t asciidoc -f markdown reftable.md >reftable.txt
using pandoc 2.2.1. The result required the following additional minor changes:
- removed the [TOC] directive to add a table of contents, since asciidoc does not support it
- replaced git-scm.com/docs links with linkgit: directives that link to other pages within Git's documentation
[1] https://eclipse.googlesource.com/jgit/jgit
--- Documentation/Makefile | 1 + Documentation/technical/reftable.txt | 1067 ++++++++++++++++++++++++++ 2 files changed, 1068 insertions(+) create mode 100644 Documentation/technical/reftable.txt
Documentation/Makefile +1 -0
@@ -93,6 +93,7 @@ TECH_DOCS += technical/protocol-capabilities TECH_DOCS += technical/protocol-common TECH_DOCS += technical/protocol-v2 TECH_DOCS += technical/racy-git +TECH_DOCS += technical/reftable TECH_DOCS += technical/send-pack-pipeline TECH_DOCS += technical/shallow TECH_DOCS += technical/signature-format
Documentation/technical/reftable.txt new file +1067 -0
1,067 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v11 6/12 reftable: define version 2 of the spec to accomodate SHA256
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
--- Documentation/technical/reftable.txt | 50 ++++++++++++++++------------ 1 file changed, 28 insertions(+), 22 deletions(-)
Documentation/technical/reftable.txt +28 -22
@@ -193,8 +193,8 @@ and non-aligned files. Very small files (e.g. 1 only ref block) may omit `padding` and the ref index to reduce total file size. -Header -^^^^^^ +Header (version 1) +^^^^^^^^^^^^^^^^^^ A 24-byte header appears at the beginning of the file:
@@ -215,6 +215,24 @@ used in a stack for link:#Update-transactions[transactions], these fields can order the files such that the prior file’s `max_update_index + 1` is the next file’s `min_update_index`. +Header (version 2) +^^^^^^^^^^^^^^^^^^ + +A 28-byte header appears at the beginning of the file: + +.... +'REFT' +uint8( version_number = 1 ) +uint24( block_size ) +uint64( min_update_index ) +uint64( max_update_index ) +uint32( hash_id ) +.... + +The header is identical to `version_number=1`, with the 4-byte hash ID +("sha1" for SHA1 and "s256" for SHA-256) append to the header. + + First ref block ^^^^^^^^^^^^^^^
@@ -671,14 +689,8 @@ Footer After the last block of the file, a file footer is written. It begins like the file header, but is extended with additional data. -A 68-byte footer appears at the end: - .... - 'REFT' - uint8( version_number = 1 ) - uint24( block_size ) - uint64( min_update_index ) - uint64( max_update_index ) + HEADER uint64( ref_index_position ) uint64( (obj_position << 5) | obj_id_len )
@@ -701,12 +713,16 @@ obj blocks. * `obj_index_position`: byte position for the start of the obj index. * `log_index_position`: byte position for the start of the log index. +The size of the footer is 68 bytes for version 1, and 72 bytes for +version 2. + Reading the footer ++++++++++++++++++ -Readers must seek to `file_length - 68` to access the footer. A trusted -external source (such as `stat(2)`) is necessary to obtain -`file_length`. When reading the footer, readers must verify: +Readers must first read the file start to determine the version +number. Then they seek to `file_length - FOOTER_LENGTH` to access the +footer. A trusted external source (such as `stat(2)`) is necessary to +obtain `file_length`. When reading the footer, readers must verify: * 4-byte magic is correct * 1-byte version number is recognized
@@ -1055,13 +1071,3 @@ impossible. A common format that can be supported by all major Git implementations (git-core, JGit, libgit2) is strongly preferred. - -Future -~~~~~~ - -Longer hashes -^^^^^^^^^^^^^ - -Version will bump (e.g. 2) to indicate `value` uses a different object -id length other than 20. The length could be stored in an expanded file -header, or hardcoded as part of the version.
Signature · 1 line
gitgitgadget
PATCH v11 7/12 reftable: clarify how empty tables should be written
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The format allows for some ambiguity, as a lone footer also starts with a valid file header. However, the current JGit code will barf on this. This commit codifies this behavior into the standard.
--- Documentation/technical/reftable.txt | 7 +++++++ 1 file changed, 7 insertions(+)
Documentation/technical/reftable.txt +7 -0
@@ -731,6 +731,13 @@ version) Once verified, the other fields of the footer can be accessed. +Empty tables +++++++++++++ + +A reftable may be empty. In this case, the file starts with a header +and is immediately followed by a footer. + + Varint encoding ^^^^^^^^^^^^^^^
Signature · 1 line
gitgitgadget
PATCH v11 8/12 Add reftable library
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Reftable is a format for storing the ref database. Its rationale and specification is in the preceding commit.
This imports the upstream library as one big commit. For understanding the code, it is suggested to read the code in the following order:
* The specification under Documentation/technical/reftable.txt
* reftable.h - the public API
* record.{c,h} - reading and writing records
* block.{c,h} - reading and writing blocks.
* writer.{c,h} - writing a complete reftable file.
* merged.{c,h} and pq.{c,h} - reading a stack of reftables
* stack.{c,h} - writing and compacting stacks of reftable on the filesystem.
--- reftable/LICENSE | 31 + reftable/README.md | 11 + reftable/VERSION | 5 + reftable/basics.c | 215 +++++++ reftable/basics.h | 53 ++ reftable/block.c | 436 ++++++++++++++ reftable/block.h | 126 ++++ reftable/blocksource.h | 22 + reftable/constants.h | 21 + reftable/file.c | 99 ++++ reftable/iter.c | 240 ++++++++ reftable/iter.h | 63 ++ reftable/merged.c | 327 +++++++++++ reftable/merged.h | 38 ++ reftable/pq.c | 115 ++++ reftable/pq.h | 34 ++ reftable/reader.c | 758 +++++++++++++++++++++++++ reftable/reader.h | 68 +++ reftable/record.c | 1131 ++++++++++++++++++++++++++++++++++++ reftable/record.h | 121 ++++ reftable/refname.c | 215 +++++++ reftable/refname.h | 39 ++ reftable/reftable.c | 91 +++ reftable/reftable.h | 564 ++++++++++++++++++ reftable/slice.c | 225 ++++++++ reftable/slice.h | 76 +++ reftable/stack.c | 1229 ++++++++++++++++++++++++++++++++++++++++ reftable/stack.h | 45 ++ reftable/system.h | 54 ++ reftable/tree.c | 67 +++ reftable/tree.h | 34 ++ reftable/update.sh | 24 + reftable/writer.c | 661 +++++++++++++++++++++ reftable/writer.h | 60 ++ reftable/zlib-compat.c | 92 +++ 35 files changed, 7390 insertions(+) create mode 100644 reftable/LICENSE create mode 100644 reftable/README.md create mode 100644 reftable/VERSION create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/blocksource.h create mode 100644 reftable/constants.h create mode 100644 reftable/file.c create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/refname.c create mode 100644 reftable/refname.h create mode 100644 reftable/reftable.c create mode 100644 reftable/reftable.h create mode 100644 reftable/slice.c create mode 100644 reftable/slice.h create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/system.h create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100755 reftable/update.sh create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h create mode 100644 reftable/zlib-compat.c
reftable/LICENSE new file +31 -0
@@ -0,0 +1,31 @@ +BSD License + +Copyright (c) 2020, Google LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of Google LLC nor the names of its contributors may +be used to endorse or promote products derived from this software +without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
reftable/README.md new file +11 -0
@@ -0,0 +1,11 @@ + +The source code in this directory comes from https://github.com/google/reftable. + +The VERSION file keeps track of the current version of the reftable library. + +To update the library, do: + + sh reftable/update.sh + +Bugfixes should be accompanied by a test and applied to upstream project at +https://github.com/google/reftable.
reftable/VERSION new file +5 -0
@@ -0,0 +1,5 @@ +commit 3a486f79abcd17e88e3bca62f43188a7c8b80ff3 +Author: Han-Wen Nienhuys <[email protected]> +Date: Mon May 4 19:20:21 2020 +0200 + + C: include "cache.h" in git-core sleep_millisec()
reftable/basics.c new file +215 -0
215 lines not on this page · raw message
reftable/basics.h new file +53 -0
53 lines not on this page · raw message
reftable/block.c new file +436 -0
436 lines not on this page · raw message
reftable/block.h new file +126 -0
126 lines not on this page · raw message
reftable/blocksource.h new file +22 -0
@@ -0,0 +1,22 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef BLOCKSOURCE_H +#define BLOCKSOURCE_H + +#include "reftable.h" + +uint64_t block_source_size (struct reftable_block_source source); +int block_source_read_block (struct reftable_block_source source, +struct reftable_block *dest,uint64_t off, +uint32_t size); +void block_source_return_block (struct reftable_block_source source, +struct reftable_block *ret); +void block_source_close (struct reftable_block_source source); + +#endif
reftable/constants.h new file +21 -0
@@ -0,0 +1,21 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef CONSTANTS_H +#define CONSTANTS_H + +#define BLOCK_TYPE_LOG 'g' +#define BLOCK_TYPE_INDEX 'i' +#define BLOCK_TYPE_REF 'r' +#define BLOCK_TYPE_OBJ 'o' +#define BLOCK_TYPE_ANY 0 + +#define MAX_RESTARTS ((1 << 16) - 1) +#define DEFAULT_BLOCK_SIZE 4096 + +#endif
reftable/file.c new file +99 -0
99 lines not on this page · raw message
reftable/iter.c new file +240 -0
240 lines not on this page · raw message
reftable/iter.h new file +63 -0
63 lines not on this page · raw message
reftable/merged.c new file +327 -0
327 lines not on this page · raw message
reftable/merged.h new file +38 -0
@@ -0,0 +1,38 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef MERGED_H +#define MERGED_H + +#include "pq.h" +#include "reftable.h" + +struct reftable_merged_table { +struct reftable_reader **stack; +int stack_len; +uint32_t hash_id; +bool suppress_deletions; + +uint64_t min; +uint64_t max; +}; + +struct merged_iter { +struct reftable_iterator *stack; +uint32_t hash_id; +int stack_len; +byte typ; +bool suppress_deletions; +struct merged_iter_pqueue pq; +}; + +void merged_table_clear (struct reftable_merged_table *mt); +int merged_table_seek_record (struct reftable_merged_table *mt, +struct reftable_iterator *it,struct record rec); + +#endif
reftable/pq.c new file +115 -0
115 lines not on this page · raw message
reftable/pq.h new file +34 -0
@@ -0,0 +1,34 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef PQ_H +#define PQ_H + +#include "record.h" + +struct pq_entry { +int index; +struct record rec; +}; + +int pq_less (struct pq_entry a,struct pq_entry b); + +struct merged_iter_pqueue { +struct pq_entry *heap; +int len; +int cap; +}; + +struct pq_entry merged_iter_pqueue_top (struct merged_iter_pqueue pq); +bool merged_iter_pqueue_is_empty (struct merged_iter_pqueue pq); +void merged_iter_pqueue_check (struct merged_iter_pqueue pq); +struct pq_entry merged_iter_pqueue_remove (struct merged_iter_pqueue *pq); +void merged_iter_pqueue_add (struct merged_iter_pqueue *pq,struct pq_entry e); +void merged_iter_pqueue_clear (struct merged_iter_pqueue *pq); + +#endif
reftable/reader.c new file +758 -0
758 lines not on this page · raw message
reftable/reader.h new file +68 -0
68 lines not on this page · raw message
reftable/record.c new file +1131 -0
338 lines not on this page · raw message
reftable/record.h new file +121 -0
121 lines not on this page · raw message
reftable/refname.c new file +215 -0
215 lines not on this page · raw message
reftable/reftable.c new file +91 -0
91 lines not on this page · raw message
reftable/reftable.h new file +564 -0
564 lines not on this page · raw message
reftable/slice.c new file +225 -0
225 lines not on this page · raw message
reftable/slice.h new file +76 -0
76 lines not on this page · raw message
reftable/stack.c new file +1229 -0
1,229 lines not on this page · raw message
reftable/stack.h new file +45 -0
45 lines not on this page · raw message
reftable/system.h new file +54 -0
54 lines not on this page · raw message
reftable/tree.c new file +67 -0
67 lines not on this page · raw message
reftable/writer.c new file +661 -0
661 lines not on this page · raw message
reftable/writer.h new file +60 -0
60 lines not on this page · raw message
reftable/zlib-compat.c new file +92 -0
92 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
gitgitgadget
Part of the diff is not shown here. View the raw message.
PATCH v11 9/12 Reftable support for git-core
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
For background, see the previous commit introducing the library.
TODO:
* Resolve spots marked with XXX
* Support worktrees (t0002-gitfile "linked repo" testcase)
Example use: see t/t0031-reftable.sh
--- .../technical/repository-version.txt | 7 + Makefile | 27 +- builtin/clone.c | 3 +- builtin/init-db.c | 56 +- cache.h | 6 +- refs.c | 27 +- refs.h | 3 + refs/refs-internal.h | 1 + refs/reftable-backend.c | 1045 +++++++++++++++++ repository.c | 2 + repository.h | 3 + setup.c | 12 +- t/t0031-reftable.sh | 109 ++ 13 files changed, 1271 insertions(+), 30 deletions(-) create mode 100644 refs/reftable-backend.c create mode 100755 t/t0031-reftable.sh
Documentation/technical/repository-version.txt +7 -0
@@ -100,3 +100,10 @@ If set, by default "git config" reads from both "config" and multiple working directory mode, "config" file is shared while "config.worktree" is per-working directory (i.e., it's in GIT_COMMON_DIR/worktrees/<id>/config.worktree) + +==== `refStorage` + +Specifies the file format for the ref database. Values are `files` +(for the traditional packed + loose ref format) and `reftable` for the +binary reftable format. See https://github.com/google/reftable for +more information.
Makefile +25 -2
@@ -810,6 +810,7 @@ TEST_SHELL_PATH = $(SHELL_PATH) LIB_FILE = libgit.a XDIFF_LIB = xdiff/lib.a VCSSVN_LIB = vcs-svn/lib.a +REFTABLE_LIB = reftable/libreftable.a GENERATED_H += config-list.h GENERATED_H += command-list.h
@@ -962,6 +963,7 @@ LIB_OBJS += ref-filter.o LIB_OBJS += reflog-walk.o LIB_OBJS += refs.o LIB_OBJS += refs/files-backend.o +LIB_OBJS += refs/reftable-backend.o LIB_OBJS += refs/iterator.o LIB_OBJS += refs/packed-backend.o LIB_OBJS += refs/ref-cache.o
@@ -1165,7 +1167,7 @@ THIRD_PARTY_SOURCES += compat/regex/% THIRD_PARTY_SOURCES += sha1collisiondetection/% THIRD_PARTY_SOURCES += sha1dc/% -GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) +GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) EXTLIBS = GIT_USER_AGENT = git/$(GIT_VERSION)
@@ -2360,11 +2362,29 @@ VCSSVN_OBJS += vcs-svn/sliding_window.o VCSSVN_OBJS += vcs-svn/svndiff.o VCSSVN_OBJS += vcs-svn/svndump.o +REFTABLE_OBJS += reftable/basics.o +REFTABLE_OBJS += reftable/block.o +REFTABLE_OBJS += reftable/file.o +REFTABLE_OBJS += reftable/iter.o +REFTABLE_OBJS += reftable/merged.o +REFTABLE_OBJS += reftable/pq.o +REFTABLE_OBJS += reftable/reader.o +REFTABLE_OBJS += reftable/record.o +REFTABLE_OBJS += reftable/refname.o +REFTABLE_OBJS += reftable/reftable.o +REFTABLE_OBJS += reftable/slice.o +REFTABLE_OBJS += reftable/stack.o +REFTABLE_OBJS += reftable/tree.o +REFTABLE_OBJS += reftable/writer.o +REFTABLE_OBJS += reftable/zlib-compat.o + + TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) OBJECTS := $(LIB_OBJS) $(BUILTIN_OBJS) $(PROGRAM_OBJS) $(TEST_OBJS) \ $(XDIFF_OBJS) \ $(VCSSVN_OBJS) \ $(FUZZ_OBJS) \ + $(REFTABLE_OBJS) \ common-main.o \ git.o ifndef NO_CURL
@@ -2505,6 +2525,9 @@ $(XDIFF_LIB): $(XDIFF_OBJS) $(VCSSVN_LIB): $(VCSSVN_OBJS) $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ +$(REFTABLE_LIB): $(REFTABLE_OBJS) + $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ + export DEFAULT_EDITOR DEFAULT_PAGER Documentation/GIT-EXCLUDED-PROGRAMS: FORCE
@@ -3120,7 +3143,7 @@ cocciclean: clean: profile-clean coverage-clean cocciclean $(RM) *.res $(RM) $(OBJECTS) - $(RM) $(LIB_FILE) $(XDIFF_LIB) $(VCSSVN_LIB) + $(RM) $(LIB_FILE) $(XDIFF_LIB) $(VCSSVN_LIB) $(REFTABLE_LIB) $(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) git$X $(RM) $(TEST_PROGRAMS) $(RM) $(FUZZ_PROGRAMS)
builtin/clone.c +2 -1
@@ -1108,7 +1108,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix) } } -init_db (git_dir, real_git_dir, option_template,GIT_HASH_UNKNOWN ,INIT_DB_QUIET ); +init_db (git_dir, real_git_dir, option_template,GIT_HASH_UNKNOWN , +DEFAULT_REF_STORAGE ,INIT_DB_QUIET );if (real_git_dir) git_dir = real_git_dir;
builtin/init-db.c +42 -14
132 lines not on this page · raw message
cache.h +4 -2
@@ -628,8 +628,9 @@ int path_inside_repo(const char *prefix, const char *path);int init_db (const char *git_dir,const char *real_git_dir,const char *template_dir,int hash_algo, -unsigned int flags); -void initialize_repository_version (int hash_algo); +const char *ref_storage_format,unsigned int flags); +void initialize_repository_version (int hash_algo, +const char *ref_storage_format);void sanitize_stdfds (void );int daemonize (void );
@@ -1043,6 +1044,7 @@ struct repository_format {int is_bare;int hash_algo;char *work_tree; +char *ref_storage;struct string_list unknown_extensions; };
refs.c +19 -8
@@ -17,10 +17,16 @@#include "argv-array.h" #include "repository.h" +const char *default_ref_storage (void ) +{ +const char *test =getenv ("GIT_TEST_REFTABLE" ); +return test ?"reftable" :"files" ; +} +/* * List of all available backends */ -static struct ref_storage_be *refs_backends = &refs_be_files; +static struct ref_storage_be *refs_backends = &refs_be_reftable;static struct ref_storage_be *find_ref_storage_backend (const char *name) {
@@ -1792,13 +1798,13 @@ static struct ref_store *lookup_ref_store_map(struct hashmap *map,* Create, record, and return a ref_store instance for the specified * gitdir. */ -static struct ref_store *ref_store_init (const char *gitdir, +static struct ref_store *ref_store_init (const char *gitdir,const char *be_name,unsigned int flags) { -const char *be_name ="files" ; -struct ref_storage_be *be =find_ref_storage_backend (be_name); +struct ref_storage_be *be;struct ref_store *refs; + be =find_ref_storage_backend (be_name);if (!be)BUG ("reference backend %s is unknown" , be_name);
@@ -1814,7 +1820,11 @@ struct ref_store *get_main_ref_store(struct repository *r)if (!r->gitdir)BUG ("attempting to get main_ref_store outside of repository" ); - r->refs_private =ref_store_init (r->gitdir,REF_STORE_ALL_CAPS ); + r->refs_private =ref_store_init (r->gitdir, + r->ref_storage_format ? + r->ref_storage_format : +DEFAULT_REF_STORAGE , +REF_STORE_ALL_CAPS );return r->refs_private; }
@@ -1869,7 +1879,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule) goto done;/* assume that add_submodule_odb() has been called */ - refs =ref_store_init (submodule_sb.buf, + refs =ref_store_init (submodule_sb.buf,DEFAULT_REF_STORAGE ,/* XXX */ REF_STORE_READ |REF_STORE_ODB );register_ref_store_map (&submodule_ref_stores,"submodule" , refs, submodule);
@@ -1883,6 +1893,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule)struct ref_store *get_worktree_ref_store (const struct worktree *wt) { +const char *format =DEFAULT_REF_STORAGE ;/* XXX */ struct ref_store *refs;const char *id;
@@ -1896,9 +1907,9 @@ struct ref_store *get_worktree_ref_store(const struct worktree *wt)if (wt->id) refs =ref_store_init (git_common_path ("worktrees/%s" , wt->id), -REF_STORE_ALL_CAPS ); + format,REF_STORE_ALL_CAPS );else - refs =ref_store_init (get_git_common_dir (), + refs =ref_store_init (get_git_common_dir (), format,REF_STORE_ALL_CAPS );if (refs)
refs.h +3 -0
@@ -9,6 +9,9 @@ struct string_list;struct string_list_item ;struct worktree ; +/* Returns the ref storage backend to use by default. */ +const char *default_ref_storage (void ); +/* * Resolve a reference, recursively following symbolic refererences. *
refs/refs-internal.h +1 -0
@@ -661,6 +661,7 @@ struct ref_storage_be { };extern struct ref_storage_be refs_be_files; +extern struct ref_storage_be refs_be_reftable;extern struct ref_storage_be refs_be_packed;/*
refs/reftable-backend.c new file +1045 -0
1,045 lines not on this page · raw message
repository.c +2 -0
@@ -178,6 +178,8 @@ int repo_init(struct repository *repo,if (worktree)repo_set_worktree (repo, worktree); + repo->ref_storage_format =xstrdup_or_null (format.ref_storage); +clear_repository_format (&format);return 0 ;
repository.h +3 -0
@@ -74,6 +74,9 @@ struct repository {*/ struct ref_store *refs_private; +/* The format to use for the ref database. */ +char *ref_storage_format; +/* * Contains path to often used file names. */
setup.c +9 -3
@@ -468,9 +468,11 @@ static int check_repo_format(const char *var, const char *value, void *vdata)if (!value)return config_error_nonbool (var); data->partial_clone =xstrdup (value); - }else if (!strcmp (ext,"worktreeconfig" )) + }else if (!strcmp (ext,"worktreeconfig" )) { data->worktree_config =git_config_bool (var, value); -else + }else if (!strcmp (ext,"refstorage" )) { + data->ref_storage =xstrdup (value); + }else string_list_append (&data->unknown_extensions, ext); }
@@ -559,6 +561,7 @@ void clear_repository_format(struct repository_format *format)string_list_clear (&format->unknown_extensions,0 );free (format->work_tree);free (format->partial_clone); +free (format->ref_storage);init_repository_format (format); }
@@ -1204,8 +1207,11 @@ const char *setup_git_directory_gently(int *nongit_ok) gitdir =DEFAULT_GIT_DIR_ENVIRONMENT ;setup_git_env (gitdir); } -if (startup_info->have_repository) +if (startup_info->have_repository) {repo_set_hash_algo (the_repository, repo_fmt.hash_algo); + the_repository->ref_storage_format = +xstrdup_or_null (repo_fmt.ref_storage); + } }strbuf_release (&dir);
t/t0031-reftable.sh new file +109 -0
109 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v11 10/12 vcxproj: adjust for the reftable changes
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Johannes Schindelin <[email protected]>
This allows Git to be compiled via Visual Studio again after integrating the hn/reftable branch.
--- config.mak.uname | 2 +- contrib/buildsystems/Generators/Vcxproj.pm | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-)
config.mak.uname +1 -1
@@ -710,7 +710,7 @@ vcxproj: # Make .vcxproj files and add them unset QUIET_GEN QUIET_BUILT_IN; \ perl contrib/buildsystems/generate -g Vcxproj - git add -f git.sln {*,*/lib,t/helper/*}/*.vcxproj + git add -f git.sln {*,*/lib,*/libreftable,t/helper/*}/*.vcxproj # Generate the LinkOrCopyBuiltins.targets and LinkOrCopyRemoteHttp.targets file (echo '<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">' && \
contrib/buildsystems/Generators/Vcxproj.pm +10 -1
@@ -77,7 +77,7 @@ sub createProject { my $libs_release = "\n "; my $libs_debug = "\n "; if (!$static_library) { - $libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); + $libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib|reftable\/libreftable\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); $libs_debug = $libs_release; $libs_debug =~ s/zlib\.lib/zlibd\.lib/g; $libs_debug =~ s/libcurl\.lib/libcurl-d\.lib/g;
@@ -231,6 +231,7 @@ sub createProject { EOM if (!$static_library || $target =~ 'vcs-svn' || $target =~ 'xdiff') { my $uuid_libgit = $$build_structure{"LIBS_libgit_GUID"}; + my $uuid_libreftable = $$build_structure{"LIBS_reftable/libreftable_GUID"}; my $uuid_xdiff_lib = $$build_structure{"LIBS_xdiff/lib_GUID"}; print F << "EOM";
@@ -240,6 +241,14 @@ sub createProject { <ReferenceOutputAssembly>false</ReferenceOutputAssembly> </ProjectReference> EOM + if (!($name =~ /xdiff|libreftable/)) { + print F << "EOM"; + <ProjectReference Include="$cdup\\reftable\\libreftable\\libreftable.vcxproj"> + <Project>$uuid_libreftable</Project> + <ReferenceOutputAssembly>false</ReferenceOutputAssembly> + </ProjectReference> +EOM + } if (!($name =~ 'xdiff')) { print F << "EOM"; <ProjectReference Include="$cdup\\xdiff\\lib\\xdiff_lib.vcxproj">
Signature · 1 line
gitgitgadget
PATCH v11 11/12 Add some reftable testing infrastructure
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
* Add GIT_TEST_REFTABLE environment var to control default ref storage
* Add test_prerequisite REFTABLE. Skip t/t3210-pack-refs.sh for REFTABLE.
--- builtin/clone.c | 2 +- builtin/init-db.c | 2 +- refs.c | 6 +++--- t/t3210-pack-refs.sh | 6 ++++++ t/test-lib.sh | 5 +++++ 5 files changed, 16 insertions(+), 5 deletions(-)
builtin/clone.c +1 -1
@@ -1109,7 +1109,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix) }init_db (git_dir, real_git_dir, option_template,GIT_HASH_UNKNOWN , -DEFAULT_REF_STORAGE ,INIT_DB_QUIET ); +default_ref_storage (),INIT_DB_QUIET );if (real_git_dir) git_dir = real_git_dir;
builtin/init-db.c +1 -1
@@ -545,7 +545,7 @@ static const char *const init_db_usage[] = {int cmd_init_db (int argc,const char **argv,const char *prefix) {const char *git_dir; -const char *ref_storage_format =DEFAULT_REF_STORAGE ; +const char *ref_storage_format =default_ref_storage ();const char *real_git_dir =NULL ;const char *work_tree;const char *template_dir =NULL ;
refs.c +3 -3
@@ -1823,7 +1823,7 @@ struct ref_store *get_main_ref_store(struct repository *r) r->refs_private =ref_store_init (r->gitdir, r->ref_storage_format ? r->ref_storage_format : -DEFAULT_REF_STORAGE , +default_ref_storage (),REF_STORE_ALL_CAPS );return r->refs_private; }
@@ -1879,7 +1879,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule) goto done;/* assume that add_submodule_odb() has been called */ - refs =ref_store_init (submodule_sb.buf,DEFAULT_REF_STORAGE ,/* XXX */ + refs =ref_store_init (submodule_sb.buf,default_ref_storage (),REF_STORE_READ |REF_STORE_ODB );register_ref_store_map (&submodule_ref_stores,"submodule" , refs, submodule);
@@ -1893,7 +1893,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule)struct ref_store *get_worktree_ref_store (const struct worktree *wt) { -const char *format =DEFAULT_REF_STORAGE ;/* XXX */ +const char *format =default_ref_storage ();struct ref_store *refs;const char *id;
t/t3210-pack-refs.sh +6 -0
@@ -11,6 +11,12 @@ semantic is still the same. ' . ./test-lib.sh +if test_have_prereq REFTABLE +then + skip_all='skipping pack-refs tests; incompatible with reftable' + test_done +fi + test_expect_success 'enable reflogs' ' git config core.logallrefupdates true '
t/test-lib.sh +5 -0
@@ -1514,6 +1514,11 @@ FreeBSD) ;; esac +if test -n "$GIT_TEST_REFTABLE" +then + test_set_prereq REFTABLE +fi + ( COLUMNS=1 && test $COLUMNS = 1 ) && test_set_prereq COLUMNS_CAN_BE_1 test -z "$NO_PERL" && test_set_prereq PERL test -z "$NO_PTHREADS" && test_set_prereq PTHREADS
Signature · 1 line
gitgitgadget
PATCH v11 12/12 t: use update-ref and show-ref to reading/writing refs
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Reading and writing .git/refs/* assumes that refs are stored in the 'files' ref backend.
--- t/t0002-gitfile.sh | 2 +- t/t1400-update-ref.sh | 32 ++++++++++++++++---------------- t/t1506-rev-parse-diagnosis.sh | 2 +- t/t6050-replace.sh | 2 +- t/t9020-remote-svn.sh | 4 ++-- 5 files changed, 21 insertions(+), 21 deletions(-)
t/t0002-gitfile.sh +1 -1
@@ -62,7 +62,7 @@ test_expect_success 'check commit-tree' ' ' test_expect_success 'check rev-list' ' - echo $SHA >"$REAL/HEAD" && + git update-ref "HEAD" "$SHA" && test "$SHA" = "$(git rev-list HEAD)" '
t/t1400-update-ref.sh +16 -16
@@ -37,15 +37,15 @@ test_expect_success setup ' test_expect_success "create $m" ' git update-ref $m $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success "create $m with oldvalue verification" ' git update-ref $m $B $A && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "fail to delete $m with stale ref" ' test_must_fail git update-ref -d $m $A && - test $B = "$(cat .git/$m)" + test $B = "$(git show-ref -s --verify $m)" ' test_expect_success "delete $m" ' test_when_finished "rm -f .git/$m" &&
@@ -56,7 +56,7 @@ test_expect_success "delete $m" ' test_expect_success "delete $m without oldvalue verification" ' test_when_finished "rm -f .git/$m" && git update-ref $m $A && - test $A = $(cat .git/$m) && + test $A = $(git show-ref -s --verify $m) && git update-ref -d $m && test_path_is_missing .git/$m '
@@ -69,15 +69,15 @@ test_expect_success "fail to create $n" ' test_expect_success "create $m (by HEAD)" ' git update-ref HEAD $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success "create $m (by HEAD) with oldvalue verification" ' git update-ref HEAD $B $A && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "fail to delete $m (by HEAD) with stale ref" ' test_must_fail git update-ref -d HEAD $A && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "delete $m (by HEAD)" ' test_when_finished "rm -f .git/$m" &&
@@ -178,14 +178,14 @@ test_expect_success '--no-create-reflog overrides core.logAllRefUpdates=always' test_expect_success "create $m (by HEAD)" ' git update-ref HEAD $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success 'pack refs' ' git pack-refs --all ' test_expect_success "move $m (by HEAD)" ' git update-ref HEAD $B $A && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "delete $m (by HEAD) should remove both packed and loose $m" ' test_when_finished "rm -f .git/$m" &&
@@ -255,7 +255,7 @@ test_expect_success '(not) change HEAD with wrong SHA1' ' ' test_expect_success "(not) changed .git/$m" ' test_when_finished "rm -f .git/$m" && - ! test $B = $(cat .git/$m) + ! test $B = $(git show-ref -s --verify $m) ' rm -f .git/logs/refs/heads/master
@@ -263,19 +263,19 @@ test_expect_success "create $m (logged by touch)" ' test_config core.logAllRefUpdates false && GIT_COMMITTER_DATE="2005-05-26 23:30" \ git update-ref --create-reflog HEAD $A -m "Initial Creation" && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success "update $m (logged by touch)" ' test_config core.logAllRefUpdates false && GIT_COMMITTER_DATE="2005-05-26 23:31" \ git update-ref HEAD $B $A -m "Switch" && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "set $m (logged by touch)" ' test_config core.logAllRefUpdates false && GIT_COMMITTER_DATE="2005-05-26 23:41" \ git update-ref HEAD $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success 'empty directory removal' '
@@ -319,19 +319,19 @@ test_expect_success "create $m (logged by config)" ' test_config core.logAllRefUpdates true && GIT_COMMITTER_DATE="2005-05-26 23:32" \ git update-ref HEAD $A -m "Initial Creation" && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success "update $m (logged by config)" ' test_config core.logAllRefUpdates true && GIT_COMMITTER_DATE="2005-05-26 23:33" \ git update-ref HEAD'" $B $A "'-m "Switch" && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "set $m (logged by config)" ' test_config core.logAllRefUpdates true && GIT_COMMITTER_DATE="2005-05-26 23:43" \ git update-ref HEAD $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' cat >expect <<EOF
t/t1506-rev-parse-diagnosis.sh +1 -1
@@ -207,7 +207,7 @@ test_expect_success 'arg before dashdash must be a revision (ambiguous)' ' { # we do not want to use rev-parse here, because # we are testing it - cat .git/refs/heads/foobar && + git show-ref -s refs/heads/foobar && printf "%s\n" -- } >expect && git rev-parse foobar -- >actual &&
t/t6050-replace.sh +1 -1
@@ -135,7 +135,7 @@ test_expect_success 'tag replaced commit' ' test_expect_success '"git fsck" works' ' git fsck master >fsck_master.out && test_i18ngrep "dangling commit $R" fsck_master.out && - test_i18ngrep "dangling tag $(cat .git/refs/tags/mytag)" fsck_master.out && + test_i18ngrep "dangling tag $(git show-ref -s refs/tags/mytag)" fsck_master.out && test -z "$(git fsck)" '
t/t9020-remote-svn.sh +2 -2
@@ -48,8 +48,8 @@ test_expect_success REMOTE_SVN 'simple fetch' ' ' test_debug ' - cat .git/refs/svn/svnsim/master - cat .git/refs/remotes/svnsim/master + git show-ref -s refs/svn/svnsim/master + git show-ref -s refs/remotes/svnsim/master ' test_expect_success REMOTE_SVN 'repeated fetch, nothing shall change' '
Signature · 1 line
gitgitgadget
Quoted 1 line · Han-Wen Nienhuys via GitGitGadget
This adds the reftable library, and hooks it up as a ref backend.
I queued this to 'pu', and we are seeing
https://github.com/git/git/runs/648285159
The ci/run-static-analysis.sh job's failure report ends like so:
HDR reftable/record.h
HDR reftable/refname.h
HDR reftable/reftable.h
HDR reftable/slice.h
HDR reftable/stack.h
HDR reftable/system.h
HDR reftable/tree.h
HDR reftable/writer.h
HDR remote.h
In file included from reftable/refname.hcc:2:0:./reftable/refname.h:12:24: error: field ‘tab’ has incomplete type
struct reftable_table tab;
^~~
./reftable/refname.h:32:13: error: ‘struct reftable_ref_record’ declared inside parameter list will not be visible outside of this definition or declaration [-Werror]
struct reftable_ref_record *recs, size_t sz);
^~~~~~~~~~~~~~~~~~~cc1: all warnings being treated as errors
make: *** [reftable/refname.hco] Error 1 Makefile:2842: recipe for target 'reftable/refname.hco' failed
make: *** Waiting for unfinished jobs....
Thanks.
v12 cover Reftable support git-core
In reply to Han-Wen Nienhuys via GitGitGadgetThis adds the reftable library, and hooks it up as a ref backend.
Includes testing support, to test: make -C t/ GIT_TEST_REFTABLE=1
Summary 18964 tests pass 2871 tests fail
Some issues:
* pseudorefs are broken
* many tests inspect .git/{logs,heads}/ directly.
v15
* fix static analysis error
* plug some memory leaks
Han-Wen Nienhuys (10):
refs.h: clarify reflog iteration order
Iterate over the "refs/" namespace in for_each_[raw]ref
refs: document how ref_iterator_advance_fn should handle symrefs Add .gitattributes for the reftable/ directory
reftable: define version 2 of the spec to accomodate SHA256
reftable: clarify how empty tables should be written Add reftable library
Reftable support for git-core
t: use update-ref and show-ref to reading/writing refs Add some reftable testing infrastructure
Johannes Schindelin (1):
vcxproj: adjust for the reftable changes
Jonathan Nieder (1):
reftable: file format documentation
Documentation/Makefile | 1 + Documentation/technical/reftable.txt | 1080 +++++++++++++++ .../technical/repository-version.txt | 7 + Makefile | 27 +- builtin/clone.c | 3 +- builtin/init-db.c | 56 +- cache.h | 6 +- config.mak.uname | 2 +- contrib/buildsystems/Generators/Vcxproj.pm | 11 +- refs.c | 33 +- refs.h | 8 +- refs/refs-internal.h | 6 + refs/reftable-backend.c | 1045 ++++++++++++++ reftable/.gitattributes | 1 + reftable/LICENSE | 31 + reftable/README.md | 11 + reftable/VERSION | 5 + reftable/basics.c | 215 +++ reftable/basics.h | 53 + reftable/block.c | 435 ++++++ reftable/block.h | 129 ++ reftable/constants.h | 21 + reftable/file.c | 99 ++ reftable/iter.c | 240 ++++ reftable/iter.h | 63 + reftable/merged.c | 327 +++++ reftable/merged.h | 38 + reftable/pq.c | 115 ++ reftable/pq.h | 34 + reftable/reader.c | 753 ++++++++++ reftable/reader.h | 65 + reftable/record.c | 1133 +++++++++++++++ reftable/record.h | 121 ++ reftable/refname.c | 215 +++ reftable/refname.h | 38 + reftable/reftable.c | 91 ++ reftable/reftable.h | 563 ++++++++ reftable/slice.c | 225 +++ reftable/slice.h | 76 + reftable/stack.c | 1230 +++++++++++++++++ reftable/stack.h | 45 + reftable/system.h | 54 + reftable/tree.c | 67 + reftable/tree.h | 34 + reftable/update.sh | 24 + reftable/writer.c | 661 +++++++++ reftable/writer.h | 60 + reftable/zlib-compat.c | 92 ++ repository.c | 2 + repository.h | 3 + setup.c | 12 +- t/t0002-gitfile.sh | 2 +- t/t0031-reftable.sh | 109 ++ t/t1400-update-ref.sh | 32 +- t/t1409-avoid-packing-refs.sh | 6 + t/t1506-rev-parse-diagnosis.sh | 2 +- t/t3210-pack-refs.sh | 6 + t/t6050-replace.sh | 2 +- t/t9020-remote-svn.sh | 4 +- t/test-lib.sh | 5 + 60 files changed, 9777 insertions(+), 57 deletions(-) create mode 100644 Documentation/technical/reftable.txt create mode 100644 refs/reftable-backend.c create mode 100644 reftable/.gitattributes create mode 100644 reftable/LICENSE create mode 100644 reftable/README.md create mode 100644 reftable/VERSION create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/constants.h create mode 100644 reftable/file.c create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/refname.c create mode 100644 reftable/refname.h create mode 100644 reftable/reftable.c create mode 100644 reftable/reftable.h create mode 100644 reftable/slice.c create mode 100644 reftable/slice.h create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/system.h create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100755 reftable/update.sh create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h create mode 100644 reftable/zlib-compat.c create mode 100755 t/t0031-reftable.sh
base-commit: b34789c0b0d3b137f0bb516b417bd8d75e0cb306
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-539%2Fhanwen%2Freftable-v12
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-539/hanwen/reftable-v12
Pull-Request: https://github.com/gitgitgadget/git/pull/539
Range-diff vs v11:
1: dfa5fd74f85 = 1: dfa5fd74f85 refs.h: clarify reflog iteration order
2: 340c5c415e1 = 2: 340c5c415e1 Iterate over the "refs/" namespace in for_each_[raw]ref
3: 6553285043b = 3: 6553285043b refs: document how ref_iterator_advance_fn should handle symrefs
4: 7dc47c7756f = 4: 7dc47c7756f Add .gitattributes for the reftable/ directory
5: 06fcb49e903 = 5: 06fcb49e903 reftable: file format documentation
6: 093fa74a3d0 = 6: 093fa74a3d0 reftable: define version 2 of the spec to accomodate SHA256
7: 6d9031372ce = 7: 6d9031372ce reftable: clarify how empty tables should be written
8: 6ee6c44752c ! 8: 57d338c4983 Add reftable library
@@ reftable/README.md (new)
## reftable/VERSION (new) ##
@@
-+commit 3a486f79abcd17e88e3bca62f43188a7c8b80ff3
++commit 06dd91a8377b0f920d9835b9835d0d650f928dce
+Author: Han-Wen Nienhuys <[email protected]>
-+Date: Mon May 4 19:20:21 2020 +0200
++Date: Wed May 6 21:27:32 2020 +0200
+
-+ C: include "cache.h" in git-core sleep_millisec()
++ C: fix small memory leak in stack.c error path
## reftable/basics.c (new) ##
@@
@@ reftable/block.c (new)
+
+#include "system.h"
+
-+#include "blocksource.h"
+#include "constants.h"
+#include "record.h"
+#include "reftable.h"
@@ reftable/block.c (new)
+ }
+
+ /* We're done with the input data. */
-+ block_source_return_block(block->source, block);
++ reftable_block_done(block);
+ block->data = uncompressed.buf;
+ block->len = sz;
+ block->source = malloc_block_source();
@@ reftable/block.h (new)
+/* size of file footer, depending on format version */
+int footer_size(int version);
+
-+#endif
-
- ## reftable/blocksource.h (new) ##
-@@
-+/*
-+Copyright 2020 Google LLC
-+
-+Use of this source code is governed by a BSD-style
-+license that can be found in the LICENSE file or at
-+https://developers.google.com/open-source/licenses/bsd
-+*/
-+
-+#ifndef BLOCKSOURCE_H
-+#define BLOCKSOURCE_H
-+
-+#include "reftable.h"
-+
-+uint64_t block_source_size(struct reftable_block_source source);
-+int block_source_read_block(struct reftable_block_source source,
-+ struct reftable_block *dest, uint64_t off,
-+ uint32_t size);
-+void block_source_return_block(struct reftable_block_source source,
-+ struct reftable_block *ret);
-+void block_source_close(struct reftable_block_source source);
++/* returns a block to its source. */
++void reftable_block_done(struct reftable_block *ret);
+
+#endif
@@ reftable/iter.c (new)
+ if (fri->double_check) {
+ struct reftable_iterator it = { 0 };
+
-+ err = reftable_reader_seek_ref(fri->r, &it,
-+ ref->ref_name);
++ err = reftable_table_seek_ref(fri->tab, &it,
++ ref->ref_name);
+ if (err == 0) {
+ err = reftable_iterator_next_ref(it, ref);
+ }
@@ reftable/iter.c (new)
+{
+ struct indexed_table_ref_iter *it = (struct indexed_table_ref_iter *)p;
+ block_iter_close(&it->cur);
-+ reader_return_block(it->r, &it->block_reader.block);
++ reftable_block_done(&it->block_reader.block);
+ slice_clear(&it->oid);
+}
+
@@ reftable/iter.c (new)
+ return 1;
+ }
+
-+ reader_return_block(it->r, &it->block_reader.block);
++ reftable_block_done(&it->block_reader.block);
+
+ {
+ uint64_t off = it->offsets[it->offset_idx++];
@@ reftable/iter.h (new)
+/* iterator that produces only ref records that point to `oid` */
+struct filtering_ref_iterator {
+ bool double_check;
-+ struct reftable_reader *r;
++ struct reftable_table tab;
+ struct slice oid;
+ struct reftable_iterator it;
+};
@@ reftable/reader.c (new)
+#include "reader.h"
+
+#include "system.h"
-+
+#include "block.h"
+#include "constants.h"
+#include "iter.h"
@@ reftable/reader.c (new)
+ return result;
+}
+
-+void block_source_return_block(struct reftable_block_source source,
-+ struct reftable_block *blockp)
++void reftable_block_done(struct reftable_block *blockp)
+{
-+ source.ops->return_block(source.arg, blockp);
++ struct reftable_block_source source = blockp->source;
++ if (blockp != NULL && source.ops != NULL)
++ source.ops->return_block(source.arg, blockp);
+ blockp->data = NULL;
+ blockp->len = 0;
+ blockp->source.ops = NULL;
@@ reftable/reader.c (new)
+ return block_source_read_block(r->source, dest, off, sz);
+}
+
-+void reader_return_block(struct reftable_reader *r, struct reftable_block *p)
-+{
-+ block_source_return_block(r->source, p);
-+}
-+
+uint32_t reftable_reader_hash_id(struct reftable_reader *r)
+{
+ return r->hash_id;
@@ reftable/reader.c (new)
+
+ err = parse_footer(r, footer.data, header.data);
+exit:
-+ block_source_return_block(r->source, &footer);
-+ block_source_return_block(r->source, &header);
++ reftable_block_done(&footer);
++ reftable_block_done(&header);
+ return err;
+}
+
@@ reftable/reader.c (new)
+ if (ti->bi.br == NULL) {
+ return;
+ }
-+ reader_return_block(ti->r, &ti->bi.br->block);
++ reftable_block_done(&ti->bi.br->block);
+ FREE_AND_NULL(ti->bi.br);
+
+ ti->bi.last_key.len = 0;
@@ reftable/reader.c (new)
+ }
+
+ if (want_typ != BLOCK_TYPE_ANY && block_typ != want_typ) {
-+ reader_return_block(r, &block);
++ reftable_block_done(&block);
+ return 1;
+ }
+
+ if (block_size > guess_block_size) {
-+ reader_return_block(r, &block);
++ reftable_block_done(&block);
+ err = reader_get_block(r, &block, next_off, block_size);
+ if (err < 0) {
+ return err;
@@ reftable/reader.c (new)
+
+static int reftable_reader_refs_for_unindexed(struct reftable_reader *r,
+ struct reftable_iterator *it,
-+ byte *oid, int oid_len)
++ byte *oid)
+{
+ struct table_iter *ti = reftable_calloc(sizeof(struct table_iter));
+ struct filtering_ref_iterator *filter = NULL;
++ int oid_len = hash_size(r->hash_id);
+ int err = reader_start(r, ti, BLOCK_TYPE_REF, false);
+ if (err < 0) {
+ reftable_free(ti);
@@ reftable/reader.c (new)
+ filter = reftable_calloc(sizeof(struct filtering_ref_iterator));
+ slice_resize(&filter->oid, oid_len);
+ memcpy(filter->oid.buf, oid, oid_len);
-+ filter->r = r;
++ reftable_table_from_reader(&filter->tab, r);
+ filter->double_check = false;
+ iterator_from_table_iter(&filter->it, ti);
+
@@ reftable/reader.c (new)
+}
+
+int reftable_reader_refs_for(struct reftable_reader *r,
-+ struct reftable_iterator *it, byte *oid,
-+ int oid_len)
++ struct reftable_iterator *it, byte *oid)
+{
+ if (r->obj_offsets.present) {
+ return reftable_reader_refs_for_indexed(r, it, oid);
+ }
-+ return reftable_reader_refs_for_unindexed(r, it, oid, oid_len);
++ return reftable_reader_refs_for_unindexed(r, it, oid);
+}
+
+uint64_t reftable_reader_max_update_index(struct reftable_reader *r)
@@ reftable/reader.h (new)
+int block_source_read_block(struct reftable_block_source source,
+ struct reftable_block *dest, uint64_t off,
+ uint32_t size);
-+void block_source_return_block(struct reftable_block_source source,
-+ struct reftable_block *ret);
+void block_source_close(struct reftable_block_source *source);
+
+/* metadata for a block type */
@@ reftable/reader.h (new)
+ struct record rec);
+void reader_close(struct reftable_reader *r);
+const char *reader_name(struct reftable_reader *r);
-+void reader_return_block(struct reftable_reader *r, struct reftable_block *p);
+
+/* initialize a block reader to read from `r` */
+int reader_init_block_reader(struct reftable_reader *r, struct block_reader *br,
@@ reftable/record.c (new)
+ memcpy(dest->buf, rec->hash_prefix, rec->hash_prefix_len);
+}
+
++static void obj_record_clear(void *rec)
++{
++ struct obj_record *obj = (struct obj_record *)rec;
++ FREE_AND_NULL(obj->hash_prefix);
++ FREE_AND_NULL(obj->offsets);
++ memset(obj, 0, sizeof(struct obj_record));
++}
++
+static void obj_record_copy_from(void *rec, const void *src_rec, int hash_size)
+{
-+ struct obj_record *ref = (struct obj_record *)rec;
++ struct obj_record *obj = (struct obj_record *)rec;
+ const struct obj_record *src = (const struct obj_record *)src_rec;
+
-+ *ref = *src;
-+ ref->hash_prefix = reftable_malloc(ref->hash_prefix_len);
-+ memcpy(ref->hash_prefix, src->hash_prefix, ref->hash_prefix_len);
++ obj_record_clear(obj);
++ *obj = *src;
++ obj->hash_prefix = reftable_malloc(obj->hash_prefix_len);
++ memcpy(obj->hash_prefix, src->hash_prefix, obj->hash_prefix_len);
+
+ {
-+ int olen = ref->offset_len * sizeof(uint64_t);
-+ ref->offsets = reftable_malloc(olen);
-+ memcpy(ref->offsets, src->offsets, olen);
++ int olen = obj->offset_len * sizeof(uint64_t);
++ obj->offsets = reftable_malloc(olen);
++ memcpy(obj->offsets, src->offsets, olen);
+ }
+}
+
-+static void obj_record_clear(void *rec)
-+{
-+ struct obj_record *ref = (struct obj_record *)rec;
-+ FREE_AND_NULL(ref->hash_prefix);
-+ FREE_AND_NULL(ref->offsets);
-+ memset(ref, 0, sizeof(struct obj_record));
-+}
-+
+static byte obj_record_val_type(const void *rec)
+{
+ struct obj_record *r = (struct obj_record *)rec;
@@ reftable/record.c (new)
+ const struct reftable_log_record *src =
+ (const struct reftable_log_record *)src_rec;
+
++ reftable_log_record_clear(dst);
+ *dst = *src;
+ if (dst->ref_name != NULL) {
+ dst->ref_name = xstrdup(dst->ref_name);
@@ reftable/refname.h (new)
+#ifndef REFNAME_H
+#define REFNAME_H
+
++#include "reftable.h"
++
+struct modification {
+ struct reftable_table tab;
+
@@ reftable/refname.h (new)
+
+int modification_validate(struct modification *mod);
+
-+/* illegal name, or dir/file conflict */
-+#define REFTABLE_REFNAME_ERROR -9
-+
+#endif
## reftable/reftable.c (new) ##
@@ reftable/reftable.h (new)
+/* closes and deallocates a reader. */
+void reftable_reader_free(struct reftable_reader *);
+
-+/* return an iterator for the refs pointing to oid */
++/* return an iterator for the refs pointing to `oid`. */
+int reftable_reader_refs_for(struct reftable_reader *r,
-+ struct reftable_iterator *it, uint8_t *oid,
-+ int oid_len);
++ struct reftable_iterator *it, uint8_t *oid);
+
+/* return the max_update_index for a table */
+uint64_t reftable_reader_max_update_index(struct reftable_reader *r);
@@ reftable/stack.c (new)
+ int i = 0;
+ for (i = 0; i < new_tables_len; i++) {
+ reader_close(new_tables[i]);
++ reftable_reader_free(new_tables[i]);
+ }
+ }
+ reftable_free(new_tables); 9: d731e1669b7 ! 9: f3bb9410038 Reftable support for git-core
@@ Commit message
For background, see the previous commit introducing the library.
- TODO:
+ This introduces the refs/reftable-backend.c containing reftable powered ref
+ storage backend.
+
+ It can be activated by passing --ref-storage=reftable to "git init".
- * Resolve spots marked with XXX
+ TODO:
- * Support worktrees (t0002-gitfile "linked repo" testcase)
+ * Resolve spots marked with XXX
Example use: see t/t0031-reftable.sh
10: 513d585f0f8 = 10: 94fcc6dca6a vcxproj: adjust for the reftable changes 12: a7d1d2e721c = 11: 2abcbd1af99 t: use update-ref and show-ref to reading/writing refs 11: 846fe29fa4b ! 12: fe9407d10b1 Add some reftable testing infrastructure
@@ Commit message
* Add test_prerequisite REFTABLE. Skip t/t3210-pack-refs.sh for REFTABLE.
+ Major test failures:
+
+ * t9903-bash-prompt - The bash mode reads .git/HEAD directly
+ * t1400-update-ref.sh - Reads from .git/{refs,logs} directly
+ * t1404-update-ref-errors.sh - Manipulates .git/refs/ directly
+ * t1405 - inspecs .git/ directly.
+ * t1450-fsck.sh - manipulates .git/ directly to create invalid state
+ * Rebase, cherry-pick: pseudo refs aren't written through the refs backend.
+
+ Other tests by decreasing brokenness:
+
+ t1407-worktree-ref-store.sh - 5 of 5
+ t1413-reflog-detach.sh - 7 of 7
+ t1415-worktree-refs.sh - 11 of 11
+ t3908-stash-in-worktree.sh - 2 of 2
+ t4207-log-decoration-colors.sh - 2 of 2
+ t5515-fetch-merge-logic.sh - 17 of 17
+ t5900-repo-selection.sh - 8 of 8
+ t6016-rev-list-graph-simplify-history.sh - 12 of 12
+ t5573-pull-verify-signatures.sh - 15 of 16
+ t5612-clone-refspec.sh - 12 of 13
+ t5514-fetch-multiple.sh - 11 of 12
+ t6030-bisect-porcelain.sh - 64 of 71
+ t5533-push-cas.sh - 15 of 17
+ t5539-fetch-http-shallow.sh - 7 of 8
+ t7413-submodule-is-active.sh - 7 of 8
+ t2400-worktree-add.sh - 59 of 69
+ t0100-previous.sh - 5 of 6
+ t7419-submodule-set-branch.sh - 5 of 6
+ t1404-update-ref-errors.sh - 44 of 53
+ t6003-rev-list-topo-order.sh - 29 of 35
+ t1409-avoid-packing-refs.sh - 9 of 11
+ t5541-http-push-smart.sh - 31 of 38
+ t5407-post-rewrite-hook.sh - 13 of 16
+ t9903-bash-prompt.sh - 52 of 66
+ t1414-reflog-walk.sh - 9 of 12
+ t1507-rev-parse-upstream.sh - 21 of 28
+ t2404-worktree-config.sh - 9 of 12
+ t1505-rev-parse-last.sh - 5 of 7
+ t7510-signed-commit.sh - 16 of 23
+ t2018-checkout-branch.sh - 15 of 22
+ (..etc)
+
+
+
## builtin/clone.c ##
@@ refs.c: struct ref_store *get_submodule_ref_store(const char *submodule)
const char *id;
+ ## t/t1409-avoid-packing-refs.sh ##
+@@ t/t1409-avoid-packing-refs.sh: test_description='avoid rewriting packed-refs unnecessarily' +
+ . ./test-lib.sh
+
++if test_have_prereq REFTABLE
++then
++ skip_all='skipping pack-refs tests; incompatible with reftable'
++ test_done
++fi
++
+ # Add an identifying mark to the packed-refs file header line. This
+ # shouldn't upset readers, and it should be omitted if the file is
+ # ever rewritten.
+
## t/t3210-pack-refs.sh ##
@@ t/t3210-pack-refs.sh: semantic is still the same.
'
Signature · 1 line
gitgitgadget
PATCH v12 1/12 refs.h: clarify reflog iteration order
From: Han-Wen Nienhuys <[email protected]>
--- refs.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
refs.h +4 -1
@@ -432,18 +432,21 @@ int delete_refs(const char *msg, struct string_list *refnames,int refs_delete_reflog (struct ref_store *refs,const char *refname);int delete_reflog (const char *refname); -/* iterate over reflog entries */ +/* Iterate over reflog entries. */ typedef int each_reflog_ent_fn (struct object_id *old_oid,struct object_id *new_oid,const char *committer,timestamp_t timestamp,int tz,const char *msg,void *cb_data); +/* Iterate in over reflog entries, oldest entry first. */ int refs_for_each_reflog_ent (struct ref_store *refs,const char *refname,each_reflog_ent_fn fn,void *cb_data);int refs_for_each_reflog_ent_reverse (struct ref_store *refs,const char *refname,each_reflog_ent_fn fn,void *cb_data); + +/* Call a function for each reflog entry, oldest entry first. */ int for_each_reflog_ent (const char *refname,each_reflog_ent_fn fn,void *cb_data);int for_each_reflog_ent_reverse (const char *refname,each_reflog_ent_fn fn,void *cb_data);
Signature · 1 line
gitgitgadget
Re: [PATCH v12 01/12] refs.h: clarify reflog iteration order
Quoted 12 lines · Han-Wen Nienhuys via GitGitGadget
From: Han-Wen Nienhuys <[email protected]>
Signed-off-by Han-Wen Nienhuys <[email protected]>--- refs.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)refs.h +0
-0
refs.h int delete_refs(const char *msg, struct string_list *refnames, · line 435
Quoted 5 lines
refs.h +1
-1@@ -432,18 +432,21 @@ int delete_refs(const char *msg, struct string_list *refnames,int refs_delete_reflog (struct ref_store *refs,const char *refname);int delete_reflog (const char *refname); -/* iterate over reflog entries */ +/* Iterate over reflog entries. */
It is not wrong per-se, but it is more problematic that this description is very similar to the functions that take callback functions of this type than it begins with lowercase.
/* Callback to process a reflog entry found by the iterators (see below) */
perhaps?
refs.h int delete_refs(const char *msg, struct string_list *refnames, · line 441
Quoted 6 lines
refs.h +1
-0typedef int each_reflog_ent_fn (struct object_id *old_oid,struct object_id *new_oid,const char *committer,timestamp_t timestamp,int tz,const char *msg,void *cb_data); +/* Iterate in over reflog entries, oldest entry first. */
/* Iterate over reflog entries of refname in refs */
/* oldest entry first */
refs.h int delete_refs(const char *msg, struct string_list *refnames, · line 443
Quoted 2 lines
refs.h +0
-0int refs_for_each_reflog_ent (struct ref_store *refs,const char *refname,each_reflog_ent_fn fn,void *cb_data);
/* youngest entry first */
refs.h int delete_refs(const char *msg, struct string_list *refnames, · line 449
Quoted 6 lines
refs.h +2
-0int refs_for_each_reflog_ent_reverse (struct ref_store *refs,const char *refname,each_reflog_ent_fn fn,void *cb_data); + +/* Call a function for each reflog entry, oldest entry first. */
Likewise.
Quoted 2 lines
refs.h +0
-0int for_each_reflog_ent (const char *refname,each_reflog_ent_fn fn,void *cb_data);int for_each_reflog_ent_reverse (const char *refname,each_reflog_ent_fn fn,void *cb_data);
PATCH v12 2/12 Iterate over the "refs/" namespace in for_each_[raw]ref
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This happens implicitly in the files/packed ref backend; making it explicit simplifies adding alternate ref storage backends, such as reftable.
--- refs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
refs.c +3 -3
@@ -1525,7 +1525,7 @@ static int do_for_each_ref(struct ref_store *refs, const char *prefix,int refs_for_each_ref (struct ref_store *refs,each_ref_fn fn,void *cb_data) { -return do_for_each_ref (refs,"" , fn,0 ,0 , cb_data); +return do_for_each_ref (refs,"refs/" , fn,0 ,0 , cb_data); }int for_each_ref (each_ref_fn fn,void *cb_data)
@@ -1585,8 +1585,8 @@ int for_each_namespaced_ref(each_ref_fn fn, void *cb_data)int refs_for_each_rawref (struct ref_store *refs,each_ref_fn fn,void *cb_data) { -return do_for_each_ref (refs,"" , fn,0 , -DO_FOR_EACH_INCLUDE_BROKEN , cb_data); +return do_for_each_ref (refs,"refs/" , fn,0 ,DO_FOR_EACH_INCLUDE_BROKEN , + cb_data); }int for_each_rawref (each_ref_fn fn,void *cb_data)
Signature · 1 line
gitgitgadget
Re: [PATCH v12 02/12] Iterate over the "refs/" namespace in for_each_[raw]ref
Quoted 5 lines · Han-Wen Nienhuys via GitGitGadget
From: Han-Wen Nienhuys <[email protected]>
This happens implicitly in the files/packed ref backend; making it explicit simplifies adding alternate ref storage backends, such as reftable.
Makes sense.
Quoted 30 lines
Signed-off-by Han-Wen Nienhuys <[email protected]>--- refs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)refs.c +3
-3@@ -1525,7 +1525,7 @@ static int do_for_each_ref(struct ref_store *refs, const char *prefix,int refs_for_each_ref (struct ref_store *refs,each_ref_fn fn,void *cb_data) { -return do_for_each_ref (refs,"" , fn,0 ,0 , cb_data); +return do_for_each_ref (refs,"refs/" , fn,0 ,0 , cb_data); }int for_each_ref (each_ref_fn fn,void *cb_data)@@ -1585,8 +1585,8 @@ int for_each_namespaced_ref(each_ref_fn fn, void *cb_data)int refs_for_each_rawref (struct ref_store *refs,each_ref_fn fn,void *cb_data) { -return do_for_each_ref (refs,"" , fn,0 , -DO_FOR_EACH_INCLUDE_BROKEN , cb_data); +return do_for_each_ref (refs,"refs/" , fn,0 ,DO_FOR_EACH_INCLUDE_BROKEN , + cb_data); }int for_each_rawref (each_ref_fn fn,void *cb_data)
Re: [PATCH v12 02/12] Iterate over the "refs/" namespace in for_each_[raw]ref
Quoted 10 lines · Junio C Hamano
Quoted 5 lines · Han-Wen Nienhuys via GitGitGadget
From: Han-Wen Nienhuys <[email protected]>
This happens implicitly in the files/packed ref backend; making it explicit simplifies adding alternate ref storage backends, such as reftable.
Makes sense.
It makes sense, but I'm not happy with this yet. I think the more consistent thing would be to have pseudo refs be integrated in the ref backend completely. Then, there could be a INCLUDE_PSEUDO_REFS flag that could be supplied to the iteration to decide whether or not to produce these refs.
Signature · 11 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
PATCH v12 3/12 refs: document how ref_iterator_advance_fn should handle symrefs
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
--- refs/refs-internal.h | 5 +++++ 1 file changed, 5 insertions(+)
refs/refs-internal.h +5 -0
@@ -438,6 +438,11 @@ void base_ref_iterator_free(struct ref_iterator *iter);/* Virtual function declarations for ref_iterators: */ +/* +* backend-specific implementation of ref_iterator_advance. +* For symrefs, the function should set REF_ISSYMREF, and it should also +* dereference the symref to provide the OID referent. +*/ typedef int ref_iterator_advance_fn (struct ref_iterator *ref_iterator); typedef int ref_iterator_peel_fn(struct ref_iterator *ref_iterator,
Signature · 1 line
gitgitgadget
Re: [PATCH v12 03/12] refs: document how ref_iterator_advance_fn should handle symrefs
Quoted 13 lines · Han-Wen Nienhuys via GitGitGadget
From: Han-Wen Nienhuys <[email protected]>
Signed-off-by Han-Wen Nienhuys <[email protected]>--- refs/refs-internal.h | 5 +++++ 1 file changed, 5 insertions(+)refs/refs-internal.h +0
-0@@ -438,6 +438,11 @@ void base_ref_iterator_free(struct ref_iterator *iter);
refs/refs-internal.h void base_ref_iterator_free(struct ref_iterator *iter); · line 448
Quoted 10 lines
refs/refs-internal.h +5
-0/* Virtual function declarations for ref_iterators: */ +/* +* backend-specific implementation of ref_iterator_advance. +* For symrefs, the function should set REF_ISSYMREF, and it should also +* dereference the symref to provide the OID referent. +*/ typedef int ref_iterator_advance_fn (struct ref_iterator *ref_iterator); typedef int ref_iterator_peel_fn(struct ref_iterator *ref_iterator,
Makes sense.
I wonder if I should take 1-3/12 as a separate "clean-up" series and merge it before everything else down to 'master'? That would reduce the churn somewhat.
I also wonder if there are similar "clean-up" buried later in the series. For example, would it make sense to also move 11/12 and have it graduate early together with 1-3/12?
Thanks.
Re: [PATCH v12 03/12] refs: document how ref_iterator_advance_fn should handle symrefs
Quoted 5 lines · Junio C Hamano
Makes sense.
I wonder if I should take 1-3/12 as a separate "clean-up" series and merge it before everything else down to 'master'? That would reduce the churn somewhat.
That would be great. Do I need to send them separately, or do can you cherry-pick the changes out of this series?
Quoted 3 lines
I also wonder if there are similar "clean-up" buried later in the series. For example, would it make sense to also move 11/12 and have it graduate early together with 1-3/12?
yes, I'll reorder my patches to reflect this.
Signature · 11 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Re: [PATCH v12 03/12] refs: document how ref_iterator_advance_fn should handle symrefs
Quoted 9 lines · Han-Wen Nienhuys
Quoted 5 lines · Junio C Hamano
Makes sense.
I wonder if I should take 1-3/12 as a separate "clean-up" series and merge it before everything else down to 'master'? That would reduce the churn somewhat.
That would be great. Do I need to send them separately, or do can you cherry-pick the changes out of this series?
For the past several days, my tree had two topics, hn/refs-cleanup (4 patches---1, 2, 3 and 11 from this series) and hn/reftable (the rest) queued separately. If everybody (including you) is happy with the former, we can just treat it as a separate "preliminary clean-up" topic and merge it down thru 'next' to 'master' and hopefully we can do so before the end of this cycle, as I do not immediately see anything controversial in them. The other topic builds on top of it, and it may have to get into a reviewable state, but at least the early "good bits" split out into a separate topic shouldn't have to wait. Then we only need to iterate on the latter part.
I am wondering if we can also throw the file format documentation into the "more or less uncontroversial" pile. There may be a lot to dislike in the current "implementation" in reftable/*.c, especially when viewed in the context of this project, that makes it less appealing to review, but my understanding is that the feature as designed by Shawn and described in the format document is already in use in JGit, so even if the code may need a major update only to become viewable from some reviewers' point of view, the design it aims to implement, at least a major part of it, is more or less cast in stone. Unless we collectively decide that we will never support reftable in git-core, we need to adopt the format documentation and the way the subsystem is supposed to work as-is to be compatible, which makes it "more or less uncontroversial". It may need some copyediting, though. Also it is not clear to me if the base part (without "hash id" and incremented format version) is the only thing in such a "the other system already implements it, so compatibility concerns leave little room to change the design at this point" state, or the updated variant that allows us to support SHA-256 and other hashes is also in that state.
Re: [PATCH v12 03/12] refs: document how ref_iterator_advance_fn should handle symrefs
Quoted 7 lines · Junio C Hamano
Quoted 7 lines · Junio C Hamano
Quoted 5 lines
Makes sense.
I wonder if I should take 1-3/12 as a separate "clean-up" series and merge it before everything else down to 'master'? That would reduce the churn somewhat.
Quoted 6 lines
Quoted 2 lines
That would be great. Do I need to send them separately, or do can you cherry-pick the changes out of this series?
For the past several days, my tree had two topics, hn/refs-cleanup (4 patches---1, 2, 3 and 11 from this series) and hn/reftable (the rest) queued separately. If everybody (including you) is happy with
I think the following patches should be uncontroversial:
* refs.h: clarify reflog iteration order - I tweaked it to reflect your comments
* refs: document how ref_iterator_advance_fn should handle symrefs
* reftable: file format documentation - I tweaked it to remove nbsp, and fixed one phrasing nit.
* reftable: clarify how empty tables should be written
* t: use update-ref and show-ref to reading/writing refs
The patch "Iterate over the "refs/" namespace in for_each_[raw]ref" is something I want to mull over for a bit. It may be unnecessary (stay tuned).
Quoted 10 lines
I am wondering if we can also throw the file format documentation into the "more or less uncontroversial" pile. There may be a lot to dislike in the current "implementation" in reftable/*.c, especially when viewed in the context of this project, that makes it less appealing to review, but my understanding is that the feature as designed by Shawn and described in the format document is already in use in JGit, so even if the code may need a major update only to become viewable from some reviewers' point of view, the design it aims to implement, at least a major part of it, is more or less cast in stone.
Correct; I support this.
Quoted 10 lines
Unless we collectively decide that we will never support
reftable in git-core, we need to adopt the format documentation and the way the subsystem is supposed to work as-is to be compatible, which makes it "more or less uncontroversial". It may need some copyediting, though. Also it is not clear to me if the base part (without "hash id" and incremented format version) is the only thing in such a "the other system already implements it, so compatibility concerns leave little room to change the design at this point" state, or the updated variant that allows us to support SHA-256 and other hashes is also in that state.
JGit supports version 1 of reftable only, and JGit doesn't do SHA-256 at all, so we have a little wiggle room with SHA-256, but the proposal I make here is the simplest way to also support SHA-256 that is in line with how it has been decided on in general (with a 4-byte hash ID).
The serialization format is set in stone from Google's perspective (we have been using it in production for 2.5 years now). The repo layout on disk is recent (Code submitted last November, and updated in February to reflect Peff's compatibility advice), so we could get away with minor tweaks if we want.
Signature · 11 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
PATCH v12 4/12 Add .gitattributes for the reftable/ directory
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
--- reftable/.gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 reftable/.gitattributes
reftable/.gitattributes new file +1 -0
@@ -0,0 +1 @@ +/zlib-compat.c whitespace=-indent-with-non-tab,-trailing-space
Signature · 1 line
gitgitgadget
PATCH v12 5/12 reftable: file format documentation
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Jonathan Nieder <[email protected]>
Shawn Pearce explains:
Some repositories contain a lot of references (e.g. android at 866k, rails at 31k). The reftable format provides:
- Near constant time lookup for any single reference, even when the repository is cold and not in process or kernel cache.
- Near constant time verification a SHA-1 is referred to by at least one reference (for allow-tip-sha1-in-want).
- Efficient lookup of an entire namespace, such as refs/tags/.
- Support atomic push O(size_of_update) operations.
- Combine reflog storage with ref storage.
This file format spec was originally written in July, 2017 by Shawn Pearce. Some refinements since then were made by Shawn and by Han-Wen Nienhuys based on experiences implementing and experimenting with the format. (All of this was in the context of our work at Google and Google is happy to contribute the result to the Git project.)
Imported from JGit[1]'s current version (c217d33ff, "Documentation/technical/reftable: improve repo layout", 2020-02-04) of Documentation/technical/reftable.md and converted to asciidoc by running
pandoc -t asciidoc -f markdown reftable.md >reftable.txt
using pandoc 2.2.1. The result required the following additional minor changes:
- removed the [TOC] directive to add a table of contents, since asciidoc does not support it
- replaced git-scm.com/docs links with linkgit: directives that link to other pages within Git's documentation
[1] https://eclipse.googlesource.com/jgit/jgit
--- Documentation/Makefile | 1 + Documentation/technical/reftable.txt | 1067 ++++++++++++++++++++++++++ 2 files changed, 1068 insertions(+) create mode 100644 Documentation/technical/reftable.txt
Documentation/Makefile +1 -0
@@ -93,6 +93,7 @@ TECH_DOCS += technical/protocol-capabilities TECH_DOCS += technical/protocol-common TECH_DOCS += technical/protocol-v2 TECH_DOCS += technical/racy-git +TECH_DOCS += technical/reftable TECH_DOCS += technical/send-pack-pipeline TECH_DOCS += technical/shallow TECH_DOCS += technical/signature-format
Documentation/technical/reftable.txt new file +1067 -0
1,067 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v12 6/12 reftable: define version 2 of the spec to accomodate SHA256
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
--- Documentation/technical/reftable.txt | 50 ++++++++++++++++------------ 1 file changed, 28 insertions(+), 22 deletions(-)
Documentation/technical/reftable.txt +28 -22
@@ -193,8 +193,8 @@ and non-aligned files. Very small files (e.g. 1 only ref block) may omit `padding` and the ref index to reduce total file size. -Header -^^^^^^ +Header (version 1) +^^^^^^^^^^^^^^^^^^ A 24-byte header appears at the beginning of the file:
@@ -215,6 +215,24 @@ used in a stack for link:#Update-transactions[transactions], these fields can order the files such that the prior file’s `max_update_index + 1` is the next file’s `min_update_index`. +Header (version 2) +^^^^^^^^^^^^^^^^^^ + +A 28-byte header appears at the beginning of the file: + +.... +'REFT' +uint8( version_number = 1 ) +uint24( block_size ) +uint64( min_update_index ) +uint64( max_update_index ) +uint32( hash_id ) +.... + +The header is identical to `version_number=1`, with the 4-byte hash ID +("sha1" for SHA1 and "s256" for SHA-256) append to the header. + + First ref block ^^^^^^^^^^^^^^^
@@ -671,14 +689,8 @@ Footer After the last block of the file, a file footer is written. It begins like the file header, but is extended with additional data. -A 68-byte footer appears at the end: - .... - 'REFT' - uint8( version_number = 1 ) - uint24( block_size ) - uint64( min_update_index ) - uint64( max_update_index ) + HEADER uint64( ref_index_position ) uint64( (obj_position << 5) | obj_id_len )
@@ -701,12 +713,16 @@ obj blocks. * `obj_index_position`: byte position for the start of the obj index. * `log_index_position`: byte position for the start of the log index. +The size of the footer is 68 bytes for version 1, and 72 bytes for +version 2. + Reading the footer ++++++++++++++++++ -Readers must seek to `file_length - 68` to access the footer. A trusted -external source (such as `stat(2)`) is necessary to obtain -`file_length`. When reading the footer, readers must verify: +Readers must first read the file start to determine the version +number. Then they seek to `file_length - FOOTER_LENGTH` to access the +footer. A trusted external source (such as `stat(2)`) is necessary to +obtain `file_length`. When reading the footer, readers must verify: * 4-byte magic is correct * 1-byte version number is recognized
@@ -1055,13 +1071,3 @@ impossible. A common format that can be supported by all major Git implementations (git-core, JGit, libgit2) is strongly preferred. - -Future -~~~~~~ - -Longer hashes -^^^^^^^^^^^^^ - -Version will bump (e.g. 2) to indicate `value` uses a different object -id length other than 20. The length could be stored in an expanded file -header, or hardcoded as part of the version.
Signature · 1 line
gitgitgadget
Re: [PATCH v12 06/12] reftable: define version 2 of the spec to accomodate SHA256
Documentation/technical/reftable.txt and non-aligned files. · line 193
Quoted 13 lines · Han-Wen Nienhuys via GitGitGadget
From: Han-Wen Nienhuys <[email protected]>
Signed-off-by Han-Wen Nienhuys <[email protected]>--- Documentation/technical/reftable.txt | 50 ++++++++++++++++------------ 1 file changed, 28 insertions(+), 22 deletions(-)Documentation/technical/reftable.txt +0
-0@@ -193,8 +193,8 @@ and non-aligned files. Very small files (e.g. 1 only ref block) may omit `padding` and the ref
Hmph, I am seeing nbsp before '1' and am wondering where it came from.
Documentation/technical/reftable.txt used in a stack for link:#Update-transactions[transactions], these · line 216
Quoted 12 lines
Documentation/technical/reftable.txt +2
-2index to reduce total file size. -Header -^^^^^^ +Header (version 1) +^^^^^^^^^^^^^^^^^^ A 24-byte header appears at the beginning of the file:@@ -215,6 +215,24 @@ used in a stack for link:#Update-transactions[transactions], these fields can order the files such that the prior file’s `max_update_index + 1` is the next file’s `min_update_index`.
Am I correct to assume that we do not plan to support a repository with mixed set of refs, some referring to a commit with its SHA-1 object name while others using SHA-256 object name?
Documentation/technical/reftable.txt used in a stack for link:#Update-transactions[transactions], these · line 225
Quoted 8 lines
Documentation/technical/reftable.txt +8
-0+Header (version 2) +^^^^^^^^^^^^^^^^^^ + +A 28-byte header appears at the beginning of the file: + +.... +'REFT' +uint8( version_number = 1 )
Shouldn't this be 2 instead, as v1 lacked the Hash-id field?
Documentation/technical/reftable.txt used in a stack for link:#Update-transactions[transactions], these · line 233
Quoted 8 lines
Documentation/technical/reftable.txt +8
-0+uint24( block_size ) +uint64( min_update_index ) +uint64( max_update_index ) +uint32( hash_id ) +.... + +The header is identical to `version_number=1`, with the 4-byte hash ID +("sha1" for SHA1 and "s256" for SHA-256) append to the header.
Am I correct to assume that SHA-1 repositories are encouraged to use version 2 when the code becomes available?
Quoted 26 lines
Documentation/technical/reftable.txt +4
-7First ref block ^^^^^^^^^^^^^^^@@ -671,14 +689,8 @@ Footer After the last block of the file, a file footer is written. It begins like the file header, but is extended with additional data. -A 68-byte footer appears at the end: - - 'REFT' - uint8( version_number = 1 ) - uint24( block_size ) - uint64( min_update_index ) - uint64( max_update_index ) + HEADER uint64( ref_index_position ) uint64( (obj_position << 5) | obj_id_len )@@ -701,12 +713,16 @@ obj blocks. * `obj_index_position`: byte position for the start of the obj index. * `log_index_position`: byte position for the start of the log index. +The size of the footer is 68 bytes for version 1, and 72 bytes for +version 2. +
Documentation/technical/reftable.txt obj blocks. · line 725
Quoted 10 lines
Documentation/technical/reftable.txt +4
-3Reading the footer ++++++++++++++++++ -Readers must seek to `file_length - 68` to access the footer. A trusted -external source (such as `stat(2)`) is necessary to obtain -`file_length`. When reading the footer, readers must verify: +Readers must first read the file start to determine the version +number. Then they seek to `file_length - FOOTER_LENGTH` to access the +footer. A trusted external source (such as `stat(2)`) is necessary to +obtain `file_length`. When reading the footer, readers must verify:
In any case, the size of this patch is pleasant to see---it must be a sign that the previous step was done well not to hardcode the "hash size is 20 bytes" assumption all over the place and instead used "this field holds N+m bytes where N is the size of the hash described in the REFT header" consistently.
Nicely done.
PATCH v12 7/12 reftable: clarify how empty tables should be written
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The format allows for some ambiguity, as a lone footer also starts with a valid file header. However, the current JGit code will barf on this. This commit codifies this behavior into the standard.
--- Documentation/technical/reftable.txt | 7 +++++++ 1 file changed, 7 insertions(+)
Documentation/technical/reftable.txt +7 -0
@@ -731,6 +731,13 @@ version) Once verified, the other fields of the footer can be accessed. +Empty tables +++++++++++++ + +A reftable may be empty. In this case, the file starts with a header +and is immediately followed by a footer. + + Varint encoding ^^^^^^^^^^^^^^^
Signature · 1 line
gitgitgadget
PATCH v12 8/12 Add reftable library
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Reftable is a format for storing the ref database. Its rationale and specification is in the preceding commit.
This imports the upstream library as one big commit. For understanding the code, it is suggested to read the code in the following order:
* The specification under Documentation/technical/reftable.txt
* reftable.h - the public API
* record.{c,h} - reading and writing records
* block.{c,h} - reading and writing blocks.
* writer.{c,h} - writing a complete reftable file.
* merged.{c,h} and pq.{c,h} - reading a stack of reftables
* stack.{c,h} - writing and compacting stacks of reftable on the filesystem.
--- reftable/LICENSE | 31 + reftable/README.md | 11 + reftable/VERSION | 5 + reftable/basics.c | 215 +++++++ reftable/basics.h | 53 ++ reftable/block.c | 435 ++++++++++++++ reftable/block.h | 129 +++++ reftable/constants.h | 21 + reftable/file.c | 99 ++++ reftable/iter.c | 240 ++++++++ reftable/iter.h | 63 ++ reftable/merged.c | 327 +++++++++++ reftable/merged.h | 38 ++ reftable/pq.c | 115 ++++ reftable/pq.h | 34 ++ reftable/reader.c | 753 ++++++++++++++++++++++++ reftable/reader.h | 65 +++ reftable/record.c | 1133 ++++++++++++++++++++++++++++++++++++ reftable/record.h | 121 ++++ reftable/refname.c | 215 +++++++ reftable/refname.h | 38 ++ reftable/reftable.c | 91 +++ reftable/reftable.h | 563 ++++++++++++++++++ reftable/slice.c | 225 ++++++++ reftable/slice.h | 76 +++ reftable/stack.c | 1230 ++++++++++++++++++++++++++++++++++++++++ reftable/stack.h | 45 ++ reftable/system.h | 54 ++ reftable/tree.c | 67 +++ reftable/tree.h | 34 ++ reftable/update.sh | 24 + reftable/writer.c | 661 +++++++++++++++++++++ reftable/writer.h | 60 ++ reftable/zlib-compat.c | 92 +++ 34 files changed, 7363 insertions(+) create mode 100644 reftable/LICENSE create mode 100644 reftable/README.md create mode 100644 reftable/VERSION create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/constants.h create mode 100644 reftable/file.c create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/refname.c create mode 100644 reftable/refname.h create mode 100644 reftable/reftable.c create mode 100644 reftable/reftable.h create mode 100644 reftable/slice.c create mode 100644 reftable/slice.h create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/system.h create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100755 reftable/update.sh create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h create mode 100644 reftable/zlib-compat.c
reftable/LICENSE new file +31 -0
@@ -0,0 +1,31 @@ +BSD License + +Copyright (c) 2020, Google LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of Google LLC nor the names of its contributors may +be used to endorse or promote products derived from this software +without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
reftable/README.md new file +11 -0
@@ -0,0 +1,11 @@ + +The source code in this directory comes from https://github.com/google/reftable. + +The VERSION file keeps track of the current version of the reftable library. + +To update the library, do: + + sh reftable/update.sh + +Bugfixes should be accompanied by a test and applied to upstream project at +https://github.com/google/reftable.
reftable/VERSION new file +5 -0
@@ -0,0 +1,5 @@ +commit 06dd91a8377b0f920d9835b9835d0d650f928dce +Author: Han-Wen Nienhuys <[email protected]> +Date: Wed May 6 21:27:32 2020 +0200 + + C: fix small memory leak in stack.c error path
reftable/basics.c new file +215 -0
215 lines not on this page · raw message
reftable/basics.h new file +53 -0
53 lines not on this page · raw message
reftable/block.c new file +435 -0
435 lines not on this page · raw message
reftable/block.h new file +129 -0
129 lines not on this page · raw message
reftable/constants.h new file +21 -0
@@ -0,0 +1,21 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef CONSTANTS_H +#define CONSTANTS_H + +#define BLOCK_TYPE_LOG 'g' +#define BLOCK_TYPE_INDEX 'i' +#define BLOCK_TYPE_REF 'r' +#define BLOCK_TYPE_OBJ 'o' +#define BLOCK_TYPE_ANY 0 + +#define MAX_RESTARTS ((1 << 16) - 1) +#define DEFAULT_BLOCK_SIZE 4096 + +#endif
reftable/file.c new file +99 -0
99 lines not on this page · raw message
reftable/iter.c new file +240 -0
240 lines not on this page · raw message
reftable/iter.h new file +63 -0
63 lines not on this page · raw message
reftable/merged.c new file +327 -0
327 lines not on this page · raw message
reftable/merged.h new file +38 -0
@@ -0,0 +1,38 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef MERGED_H +#define MERGED_H + +#include "pq.h" +#include "reftable.h" + +struct reftable_merged_table { +struct reftable_reader **stack; +int stack_len; +uint32_t hash_id; +bool suppress_deletions; + +uint64_t min; +uint64_t max; +}; + +struct merged_iter { +struct reftable_iterator *stack; +uint32_t hash_id; +int stack_len; +byte typ; +bool suppress_deletions; +struct merged_iter_pqueue pq; +}; + +void merged_table_clear (struct reftable_merged_table *mt); +int merged_table_seek_record (struct reftable_merged_table *mt, +struct reftable_iterator *it,struct record rec); + +#endif
reftable/pq.c new file +115 -0
115 lines not on this page · raw message
reftable/pq.h new file +34 -0
@@ -0,0 +1,34 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef PQ_H +#define PQ_H + +#include "record.h" + +struct pq_entry { +int index; +struct record rec; +}; + +int pq_less (struct pq_entry a,struct pq_entry b); + +struct merged_iter_pqueue { +struct pq_entry *heap; +int len; +int cap; +}; + +struct pq_entry merged_iter_pqueue_top (struct merged_iter_pqueue pq); +bool merged_iter_pqueue_is_empty (struct merged_iter_pqueue pq); +void merged_iter_pqueue_check (struct merged_iter_pqueue pq); +struct pq_entry merged_iter_pqueue_remove (struct merged_iter_pqueue *pq); +void merged_iter_pqueue_add (struct merged_iter_pqueue *pq,struct pq_entry e); +void merged_iter_pqueue_clear (struct merged_iter_pqueue *pq); + +#endif
reftable/reader.c new file +753 -0
753 lines not on this page · raw message
reftable/reader.h new file +65 -0
65 lines not on this page · raw message
reftable/record.c new file +1133 -0
366 lines not on this page · raw message
reftable/record.h new file +121 -0
121 lines not on this page · raw message
reftable/refname.c new file +215 -0
215 lines not on this page · raw message
reftable/reftable.c new file +91 -0
91 lines not on this page · raw message
reftable/reftable.h new file +563 -0
563 lines not on this page · raw message
reftable/slice.c new file +225 -0
225 lines not on this page · raw message
reftable/slice.h new file +76 -0
76 lines not on this page · raw message
reftable/stack.c new file +1230 -0
1,230 lines not on this page · raw message
reftable/stack.h new file +45 -0
45 lines not on this page · raw message
reftable/system.h new file +54 -0
54 lines not on this page · raw message
reftable/tree.c new file +67 -0
67 lines not on this page · raw message
reftable/writer.c new file +661 -0
661 lines not on this page · raw message
reftable/writer.h new file +60 -0
60 lines not on this page · raw message
reftable/zlib-compat.c new file +92 -0
92 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
gitgitgadget
Part of the diff is not shown here. View the raw message.
PATCH v12 9/12 Reftable support for git-core
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
For background, see the previous commit introducing the library.
This introduces the refs/reftable-backend.c containing reftable powered ref storage backend.
It can be activated by passing --ref-storage=reftable to "git init".
TODO:
* Resolve spots marked with XXX
Example use: see t/t0031-reftable.sh
--- .../technical/repository-version.txt | 7 + Makefile | 27 +- builtin/clone.c | 3 +- builtin/init-db.c | 56 +- cache.h | 6 +- refs.c | 27 +- refs.h | 3 + refs/refs-internal.h | 1 + refs/reftable-backend.c | 1045 +++++++++++++++++ repository.c | 2 + repository.h | 3 + setup.c | 12 +- t/t0031-reftable.sh | 109 ++ 13 files changed, 1271 insertions(+), 30 deletions(-) create mode 100644 refs/reftable-backend.c create mode 100755 t/t0031-reftable.sh
Documentation/technical/repository-version.txt +7 -0
@@ -100,3 +100,10 @@ If set, by default "git config" reads from both "config" and multiple working directory mode, "config" file is shared while "config.worktree" is per-working directory (i.e., it's in GIT_COMMON_DIR/worktrees/<id>/config.worktree) + +==== `refStorage` + +Specifies the file format for the ref database. Values are `files` +(for the traditional packed + loose ref format) and `reftable` for the +binary reftable format. See https://github.com/google/reftable for +more information.
Makefile +25 -2
@@ -810,6 +810,7 @@ TEST_SHELL_PATH = $(SHELL_PATH) LIB_FILE = libgit.a XDIFF_LIB = xdiff/lib.a VCSSVN_LIB = vcs-svn/lib.a +REFTABLE_LIB = reftable/libreftable.a GENERATED_H += config-list.h GENERATED_H += command-list.h
@@ -962,6 +963,7 @@ LIB_OBJS += ref-filter.o LIB_OBJS += reflog-walk.o LIB_OBJS += refs.o LIB_OBJS += refs/files-backend.o +LIB_OBJS += refs/reftable-backend.o LIB_OBJS += refs/iterator.o LIB_OBJS += refs/packed-backend.o LIB_OBJS += refs/ref-cache.o
@@ -1165,7 +1167,7 @@ THIRD_PARTY_SOURCES += compat/regex/% THIRD_PARTY_SOURCES += sha1collisiondetection/% THIRD_PARTY_SOURCES += sha1dc/% -GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) +GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) EXTLIBS = GIT_USER_AGENT = git/$(GIT_VERSION)
@@ -2360,11 +2362,29 @@ VCSSVN_OBJS += vcs-svn/sliding_window.o VCSSVN_OBJS += vcs-svn/svndiff.o VCSSVN_OBJS += vcs-svn/svndump.o +REFTABLE_OBJS += reftable/basics.o +REFTABLE_OBJS += reftable/block.o +REFTABLE_OBJS += reftable/file.o +REFTABLE_OBJS += reftable/iter.o +REFTABLE_OBJS += reftable/merged.o +REFTABLE_OBJS += reftable/pq.o +REFTABLE_OBJS += reftable/reader.o +REFTABLE_OBJS += reftable/record.o +REFTABLE_OBJS += reftable/refname.o +REFTABLE_OBJS += reftable/reftable.o +REFTABLE_OBJS += reftable/slice.o +REFTABLE_OBJS += reftable/stack.o +REFTABLE_OBJS += reftable/tree.o +REFTABLE_OBJS += reftable/writer.o +REFTABLE_OBJS += reftable/zlib-compat.o + + TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) OBJECTS := $(LIB_OBJS) $(BUILTIN_OBJS) $(PROGRAM_OBJS) $(TEST_OBJS) \ $(XDIFF_OBJS) \ $(VCSSVN_OBJS) \ $(FUZZ_OBJS) \ + $(REFTABLE_OBJS) \ common-main.o \ git.o ifndef NO_CURL
@@ -2505,6 +2525,9 @@ $(XDIFF_LIB): $(XDIFF_OBJS) $(VCSSVN_LIB): $(VCSSVN_OBJS) $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ +$(REFTABLE_LIB): $(REFTABLE_OBJS) + $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ + export DEFAULT_EDITOR DEFAULT_PAGER Documentation/GIT-EXCLUDED-PROGRAMS: FORCE
@@ -3120,7 +3143,7 @@ cocciclean: clean: profile-clean coverage-clean cocciclean $(RM) *.res $(RM) $(OBJECTS) - $(RM) $(LIB_FILE) $(XDIFF_LIB) $(VCSSVN_LIB) + $(RM) $(LIB_FILE) $(XDIFF_LIB) $(VCSSVN_LIB) $(REFTABLE_LIB) $(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) git$X $(RM) $(TEST_PROGRAMS) $(RM) $(FUZZ_PROGRAMS)
builtin/clone.c +2 -1
@@ -1108,7 +1108,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix) } } -init_db (git_dir, real_git_dir, option_template,GIT_HASH_UNKNOWN ,INIT_DB_QUIET ); +init_db (git_dir, real_git_dir, option_template,GIT_HASH_UNKNOWN , +DEFAULT_REF_STORAGE ,INIT_DB_QUIET );if (real_git_dir) git_dir = real_git_dir;
builtin/init-db.c +42 -14
132 lines not on this page · raw message
cache.h +4 -2
@@ -628,8 +628,9 @@ int path_inside_repo(const char *prefix, const char *path);int init_db (const char *git_dir,const char *real_git_dir,const char *template_dir,int hash_algo, -unsigned int flags); -void initialize_repository_version (int hash_algo); +const char *ref_storage_format,unsigned int flags); +void initialize_repository_version (int hash_algo, +const char *ref_storage_format);void sanitize_stdfds (void );int daemonize (void );
@@ -1043,6 +1044,7 @@ struct repository_format {int is_bare;int hash_algo;char *work_tree; +char *ref_storage;struct string_list unknown_extensions; };
refs.c +19 -8
@@ -17,10 +17,16 @@#include "argv-array.h" #include "repository.h" +const char *default_ref_storage (void ) +{ +const char *test =getenv ("GIT_TEST_REFTABLE" ); +return test ?"reftable" :"files" ; +} +/* * List of all available backends */ -static struct ref_storage_be *refs_backends = &refs_be_files; +static struct ref_storage_be *refs_backends = &refs_be_reftable;static struct ref_storage_be *find_ref_storage_backend (const char *name) {
@@ -1792,13 +1798,13 @@ static struct ref_store *lookup_ref_store_map(struct hashmap *map,* Create, record, and return a ref_store instance for the specified * gitdir. */ -static struct ref_store *ref_store_init (const char *gitdir, +static struct ref_store *ref_store_init (const char *gitdir,const char *be_name,unsigned int flags) { -const char *be_name ="files" ; -struct ref_storage_be *be =find_ref_storage_backend (be_name); +struct ref_storage_be *be;struct ref_store *refs; + be =find_ref_storage_backend (be_name);if (!be)BUG ("reference backend %s is unknown" , be_name);
@@ -1814,7 +1820,11 @@ struct ref_store *get_main_ref_store(struct repository *r)if (!r->gitdir)BUG ("attempting to get main_ref_store outside of repository" ); - r->refs_private =ref_store_init (r->gitdir,REF_STORE_ALL_CAPS ); + r->refs_private =ref_store_init (r->gitdir, + r->ref_storage_format ? + r->ref_storage_format : +DEFAULT_REF_STORAGE , +REF_STORE_ALL_CAPS );return r->refs_private; }
@@ -1869,7 +1879,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule) goto done;/* assume that add_submodule_odb() has been called */ - refs =ref_store_init (submodule_sb.buf, + refs =ref_store_init (submodule_sb.buf,DEFAULT_REF_STORAGE ,/* XXX */ REF_STORE_READ |REF_STORE_ODB );register_ref_store_map (&submodule_ref_stores,"submodule" , refs, submodule);
@@ -1883,6 +1893,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule)struct ref_store *get_worktree_ref_store (const struct worktree *wt) { +const char *format =DEFAULT_REF_STORAGE ;/* XXX */ struct ref_store *refs;const char *id;
@@ -1896,9 +1907,9 @@ struct ref_store *get_worktree_ref_store(const struct worktree *wt)if (wt->id) refs =ref_store_init (git_common_path ("worktrees/%s" , wt->id), -REF_STORE_ALL_CAPS ); + format,REF_STORE_ALL_CAPS );else - refs =ref_store_init (get_git_common_dir (), + refs =ref_store_init (get_git_common_dir (), format,REF_STORE_ALL_CAPS );if (refs)
refs.h +3 -0
@@ -9,6 +9,9 @@ struct string_list;struct string_list_item ;struct worktree ; +/* Returns the ref storage backend to use by default. */ +const char *default_ref_storage (void ); +/* * Resolve a reference, recursively following symbolic refererences. *
refs/refs-internal.h +1 -0
@@ -661,6 +661,7 @@ struct ref_storage_be { };extern struct ref_storage_be refs_be_files; +extern struct ref_storage_be refs_be_reftable;extern struct ref_storage_be refs_be_packed;/*
refs/reftable-backend.c new file +1045 -0
1,045 lines not on this page · raw message
repository.c +2 -0
@@ -178,6 +178,8 @@ int repo_init(struct repository *repo,if (worktree)repo_set_worktree (repo, worktree); + repo->ref_storage_format =xstrdup_or_null (format.ref_storage); +clear_repository_format (&format);return 0 ;
repository.h +3 -0
@@ -74,6 +74,9 @@ struct repository {*/ struct ref_store *refs_private; +/* The format to use for the ref database. */ +char *ref_storage_format; +/* * Contains path to often used file names. */
setup.c +9 -3
@@ -468,9 +468,11 @@ static int check_repo_format(const char *var, const char *value, void *vdata)if (!value)return config_error_nonbool (var); data->partial_clone =xstrdup (value); - }else if (!strcmp (ext,"worktreeconfig" )) + }else if (!strcmp (ext,"worktreeconfig" )) { data->worktree_config =git_config_bool (var, value); -else + }else if (!strcmp (ext,"refstorage" )) { + data->ref_storage =xstrdup (value); + }else string_list_append (&data->unknown_extensions, ext); }
@@ -559,6 +561,7 @@ void clear_repository_format(struct repository_format *format)string_list_clear (&format->unknown_extensions,0 );free (format->work_tree);free (format->partial_clone); +free (format->ref_storage);init_repository_format (format); }
@@ -1204,8 +1207,11 @@ const char *setup_git_directory_gently(int *nongit_ok) gitdir =DEFAULT_GIT_DIR_ENVIRONMENT ;setup_git_env (gitdir); } -if (startup_info->have_repository) +if (startup_info->have_repository) {repo_set_hash_algo (the_repository, repo_fmt.hash_algo); + the_repository->ref_storage_format = +xstrdup_or_null (repo_fmt.ref_storage); + } }strbuf_release (&dir);
t/t0031-reftable.sh new file +109 -0
109 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v12 10/12 vcxproj: adjust for the reftable changes
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Johannes Schindelin <[email protected]>
This allows Git to be compiled via Visual Studio again after integrating the hn/reftable branch.
--- config.mak.uname | 2 +- contrib/buildsystems/Generators/Vcxproj.pm | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-)
config.mak.uname +1 -1
@@ -710,7 +710,7 @@ vcxproj: # Make .vcxproj files and add them unset QUIET_GEN QUIET_BUILT_IN; \ perl contrib/buildsystems/generate -g Vcxproj - git add -f git.sln {*,*/lib,t/helper/*}/*.vcxproj + git add -f git.sln {*,*/lib,*/libreftable,t/helper/*}/*.vcxproj # Generate the LinkOrCopyBuiltins.targets and LinkOrCopyRemoteHttp.targets file (echo '<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">' && \
contrib/buildsystems/Generators/Vcxproj.pm +10 -1
@@ -77,7 +77,7 @@ sub createProject { my $libs_release = "\n "; my $libs_debug = "\n "; if (!$static_library) { - $libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); + $libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib|reftable\/libreftable\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); $libs_debug = $libs_release; $libs_debug =~ s/zlib\.lib/zlibd\.lib/g; $libs_debug =~ s/libcurl\.lib/libcurl-d\.lib/g;
@@ -231,6 +231,7 @@ sub createProject { EOM if (!$static_library || $target =~ 'vcs-svn' || $target =~ 'xdiff') { my $uuid_libgit = $$build_structure{"LIBS_libgit_GUID"}; + my $uuid_libreftable = $$build_structure{"LIBS_reftable/libreftable_GUID"}; my $uuid_xdiff_lib = $$build_structure{"LIBS_xdiff/lib_GUID"}; print F << "EOM";
@@ -240,6 +241,14 @@ sub createProject { <ReferenceOutputAssembly>false</ReferenceOutputAssembly> </ProjectReference> EOM + if (!($name =~ /xdiff|libreftable/)) { + print F << "EOM"; + <ProjectReference Include="$cdup\\reftable\\libreftable\\libreftable.vcxproj"> + <Project>$uuid_libreftable</Project> + <ReferenceOutputAssembly>false</ReferenceOutputAssembly> + </ProjectReference> +EOM + } if (!($name =~ 'xdiff')) { print F << "EOM"; <ProjectReference Include="$cdup\\xdiff\\lib\\xdiff_lib.vcxproj">
Signature · 1 line
gitgitgadget
PATCH v12 11/12 t: use update-ref and show-ref to reading/writing refs
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Reading and writing .git/refs/* assumes that refs are stored in the 'files' ref backend.
--- t/t0002-gitfile.sh | 2 +- t/t1400-update-ref.sh | 32 ++++++++++++++++---------------- t/t1506-rev-parse-diagnosis.sh | 2 +- t/t6050-replace.sh | 2 +- t/t9020-remote-svn.sh | 4 ++-- 5 files changed, 21 insertions(+), 21 deletions(-)
t/t0002-gitfile.sh +1 -1
@@ -62,7 +62,7 @@ test_expect_success 'check commit-tree' ' ' test_expect_success 'check rev-list' ' - echo $SHA >"$REAL/HEAD" && + git update-ref "HEAD" "$SHA" && test "$SHA" = "$(git rev-list HEAD)" '
t/t1400-update-ref.sh +16 -16
@@ -37,15 +37,15 @@ test_expect_success setup ' test_expect_success "create $m" ' git update-ref $m $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success "create $m with oldvalue verification" ' git update-ref $m $B $A && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "fail to delete $m with stale ref" ' test_must_fail git update-ref -d $m $A && - test $B = "$(cat .git/$m)" + test $B = "$(git show-ref -s --verify $m)" ' test_expect_success "delete $m" ' test_when_finished "rm -f .git/$m" &&
@@ -56,7 +56,7 @@ test_expect_success "delete $m" ' test_expect_success "delete $m without oldvalue verification" ' test_when_finished "rm -f .git/$m" && git update-ref $m $A && - test $A = $(cat .git/$m) && + test $A = $(git show-ref -s --verify $m) && git update-ref -d $m && test_path_is_missing .git/$m '
@@ -69,15 +69,15 @@ test_expect_success "fail to create $n" ' test_expect_success "create $m (by HEAD)" ' git update-ref HEAD $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success "create $m (by HEAD) with oldvalue verification" ' git update-ref HEAD $B $A && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "fail to delete $m (by HEAD) with stale ref" ' test_must_fail git update-ref -d HEAD $A && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "delete $m (by HEAD)" ' test_when_finished "rm -f .git/$m" &&
@@ -178,14 +178,14 @@ test_expect_success '--no-create-reflog overrides core.logAllRefUpdates=always' test_expect_success "create $m (by HEAD)" ' git update-ref HEAD $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success 'pack refs' ' git pack-refs --all ' test_expect_success "move $m (by HEAD)" ' git update-ref HEAD $B $A && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "delete $m (by HEAD) should remove both packed and loose $m" ' test_when_finished "rm -f .git/$m" &&
@@ -255,7 +255,7 @@ test_expect_success '(not) change HEAD with wrong SHA1' ' ' test_expect_success "(not) changed .git/$m" ' test_when_finished "rm -f .git/$m" && - ! test $B = $(cat .git/$m) + ! test $B = $(git show-ref -s --verify $m) ' rm -f .git/logs/refs/heads/master
@@ -263,19 +263,19 @@ test_expect_success "create $m (logged by touch)" ' test_config core.logAllRefUpdates false && GIT_COMMITTER_DATE="2005-05-26 23:30" \ git update-ref --create-reflog HEAD $A -m "Initial Creation" && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success "update $m (logged by touch)" ' test_config core.logAllRefUpdates false && GIT_COMMITTER_DATE="2005-05-26 23:31" \ git update-ref HEAD $B $A -m "Switch" && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "set $m (logged by touch)" ' test_config core.logAllRefUpdates false && GIT_COMMITTER_DATE="2005-05-26 23:41" \ git update-ref HEAD $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success 'empty directory removal' '
@@ -319,19 +319,19 @@ test_expect_success "create $m (logged by config)" ' test_config core.logAllRefUpdates true && GIT_COMMITTER_DATE="2005-05-26 23:32" \ git update-ref HEAD $A -m "Initial Creation" && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success "update $m (logged by config)" ' test_config core.logAllRefUpdates true && GIT_COMMITTER_DATE="2005-05-26 23:33" \ git update-ref HEAD'" $B $A "'-m "Switch" && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "set $m (logged by config)" ' test_config core.logAllRefUpdates true && GIT_COMMITTER_DATE="2005-05-26 23:43" \ git update-ref HEAD $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' cat >expect <<EOF
t/t1506-rev-parse-diagnosis.sh +1 -1
@@ -207,7 +207,7 @@ test_expect_success 'arg before dashdash must be a revision (ambiguous)' ' { # we do not want to use rev-parse here, because # we are testing it - cat .git/refs/heads/foobar && + git show-ref -s refs/heads/foobar && printf "%s\n" -- } >expect && git rev-parse foobar -- >actual &&
t/t6050-replace.sh +1 -1
@@ -135,7 +135,7 @@ test_expect_success 'tag replaced commit' ' test_expect_success '"git fsck" works' ' git fsck master >fsck_master.out && test_i18ngrep "dangling commit $R" fsck_master.out && - test_i18ngrep "dangling tag $(cat .git/refs/tags/mytag)" fsck_master.out && + test_i18ngrep "dangling tag $(git show-ref -s refs/tags/mytag)" fsck_master.out && test -z "$(git fsck)" '
t/t9020-remote-svn.sh +2 -2
@@ -48,8 +48,8 @@ test_expect_success REMOTE_SVN 'simple fetch' ' ' test_debug ' - cat .git/refs/svn/svnsim/master - cat .git/refs/remotes/svnsim/master + git show-ref -s refs/svn/svnsim/master + git show-ref -s refs/remotes/svnsim/master ' test_expect_success REMOTE_SVN 'repeated fetch, nothing shall change' '
Signature · 1 line
gitgitgadget
PATCH v12 12/12 Add some reftable testing infrastructure
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
* Add GIT_TEST_REFTABLE environment var to control default ref storage
* Add test_prerequisite REFTABLE. Skip t/t3210-pack-refs.sh for REFTABLE.
Major test failures:
* t9903-bash-prompt - The bash mode reads .git/HEAD directly
* t1400-update-ref.sh - Reads from .git/{refs,logs} directly
* t1404-update-ref-errors.sh - Manipulates .git/refs/ directly
* t1405 - inspecs .git/ directly.
* t1450-fsck.sh - manipulates .git/ directly to create invalid state
* Rebase, cherry-pick: pseudo refs aren't written through the refs backend.
Other tests by decreasing brokenness:
t1407-worktree-ref-store.sh - 5 of 5 t1413-reflog-detach.sh - 7 of 7 t1415-worktree-refs.sh - 11 of 11 t3908-stash-in-worktree.sh - 2 of 2 t4207-log-decoration-colors.sh - 2 of 2 t5515-fetch-merge-logic.sh - 17 of 17 t5900-repo-selection.sh - 8 of 8 t6016-rev-list-graph-simplify-history.sh - 12 of 12 t5573-pull-verify-signatures.sh - 15 of 16 t5612-clone-refspec.sh - 12 of 13 t5514-fetch-multiple.sh - 11 of 12 t6030-bisect-porcelain.sh - 64 of 71 t5533-push-cas.sh - 15 of 17 t5539-fetch-http-shallow.sh - 7 of 8 t7413-submodule-is-active.sh - 7 of 8 t2400-worktree-add.sh - 59 of 69 t0100-previous.sh - 5 of 6 t7419-submodule-set-branch.sh - 5 of 6 t1404-update-ref-errors.sh - 44 of 53 t6003-rev-list-topo-order.sh - 29 of 35 t1409-avoid-packing-refs.sh - 9 of 11 t5541-http-push-smart.sh - 31 of 38 t5407-post-rewrite-hook.sh - 13 of 16 t9903-bash-prompt.sh - 52 of 66 t1414-reflog-walk.sh - 9 of 12 t1507-rev-parse-upstream.sh - 21 of 28 t2404-worktree-config.sh - 9 of 12 t1505-rev-parse-last.sh - 5 of 7 t7510-signed-commit.sh - 16 of 23 t2018-checkout-branch.sh - 15 of 22
(..etc)
--- builtin/clone.c | 2 +- builtin/init-db.c | 2 +- refs.c | 6 +++--- t/t1409-avoid-packing-refs.sh | 6 ++++++ t/t3210-pack-refs.sh | 6 ++++++ t/test-lib.sh | 5 +++++ 6 files changed, 22 insertions(+), 5 deletions(-)
builtin/clone.c +1 -1
@@ -1109,7 +1109,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix) }init_db (git_dir, real_git_dir, option_template,GIT_HASH_UNKNOWN , -DEFAULT_REF_STORAGE ,INIT_DB_QUIET ); +default_ref_storage (),INIT_DB_QUIET );if (real_git_dir) git_dir = real_git_dir;
builtin/init-db.c +1 -1
@@ -545,7 +545,7 @@ static const char *const init_db_usage[] = {int cmd_init_db (int argc,const char **argv,const char *prefix) {const char *git_dir; -const char *ref_storage_format =DEFAULT_REF_STORAGE ; +const char *ref_storage_format =default_ref_storage ();const char *real_git_dir =NULL ;const char *work_tree;const char *template_dir =NULL ;
refs.c +3 -3
@@ -1823,7 +1823,7 @@ struct ref_store *get_main_ref_store(struct repository *r) r->refs_private =ref_store_init (r->gitdir, r->ref_storage_format ? r->ref_storage_format : -DEFAULT_REF_STORAGE , +default_ref_storage (),REF_STORE_ALL_CAPS );return r->refs_private; }
@@ -1879,7 +1879,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule) goto done;/* assume that add_submodule_odb() has been called */ - refs =ref_store_init (submodule_sb.buf,DEFAULT_REF_STORAGE ,/* XXX */ + refs =ref_store_init (submodule_sb.buf,default_ref_storage (),REF_STORE_READ |REF_STORE_ODB );register_ref_store_map (&submodule_ref_stores,"submodule" , refs, submodule);
@@ -1893,7 +1893,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule)struct ref_store *get_worktree_ref_store (const struct worktree *wt) { -const char *format =DEFAULT_REF_STORAGE ;/* XXX */ +const char *format =default_ref_storage ();struct ref_store *refs;const char *id;
t/t1409-avoid-packing-refs.sh +6 -0
@@ -4,6 +4,12 @@ test_description='avoid rewriting packed-refs unnecessarily' . ./test-lib.sh +if test_have_prereq REFTABLE +then + skip_all='skipping pack-refs tests; incompatible with reftable' + test_done +fi + # Add an identifying mark to the packed-refs file header line. This # shouldn't upset readers, and it should be omitted if the file is # ever rewritten.
t/t3210-pack-refs.sh +6 -0
@@ -11,6 +11,12 @@ semantic is still the same. ' . ./test-lib.sh +if test_have_prereq REFTABLE +then + skip_all='skipping pack-refs tests; incompatible with reftable' + test_done +fi + test_expect_success 'enable reflogs' ' git config core.logallrefupdates true '
t/test-lib.sh +5 -0
@@ -1514,6 +1514,11 @@ FreeBSD) ;; esac +if test -n "$GIT_TEST_REFTABLE" +then + test_set_prereq REFTABLE +fi + ( COLUMNS=1 && test $COLUMNS = 1 ) && test_set_prereq COLUMNS_CAN_BE_1 test -z "$NO_PERL" && test_set_prereq PERL test -z "$NO_PTHREADS" && test_set_prereq PTHREADS
Signature · 1 line
gitgitgadget
v13 cover Reftable support git-core
In reply to Han-Wen Nienhuys via GitGitGadgetThis adds the reftable library, and hooks it up as a ref backend.
Includes testing support, to test: make -C t/ GIT_TEST_REFTABLE=1
Summary 19055 tests pass 2765 tests fail
Some issues:
* many tests inspect .git/{logs,heads}/ directly.
* worktrees broken.
* Rebase/cherry-pick still largely broken
v16
* handle pseudo refs.
* various bugfixes and fixes for mem leaks.
Han-Wen Nienhuys (11):
refs.h: clarify reflog iteration order
t: use update-ref and show-ref to reading/writing refs
refs: document how ref_iterator_advance_fn should handle symrefs
reftable: clarify how empty tables should be written
reftable: define version 2 of the spec to accomodate SHA256 Write pseudorefs through ref backends.
Iterate over the "refs/" namespace in for_each_[raw]ref Add .gitattributes for the reftable/ directory
Add reftable library
Reftable support for git-core
Add some reftable testing infrastructure
Johannes Schindelin (1):
vcxproj: adjust for the reftable changes
Jonathan Nieder (1):
reftable: file format documentation
Documentation/Makefile | 1 + Documentation/technical/reftable.txt | 1083 ++++++++++++++ .../technical/repository-version.txt | 7 + Makefile | 27 +- builtin/clone.c | 3 +- builtin/init-db.c | 56 +- cache.h | 6 +- config.mak.uname | 2 +- contrib/buildsystems/Generators/Vcxproj.pm | 11 +- refs.c | 148 +- refs.h | 28 +- refs/files-backend.c | 164 ++- refs/packed-backend.c | 40 +- refs/refs-internal.h | 18 + refs/reftable-backend.c | 1218 ++++++++++++++++ reftable/.gitattributes | 1 + reftable/LICENSE | 31 + reftable/README.md | 11 + reftable/VERSION | 14 + reftable/basics.c | 215 +++ reftable/basics.h | 53 + reftable/block.c | 434 ++++++ reftable/block.h | 129 ++ reftable/constants.h | 21 + reftable/file.c | 99 ++ reftable/iter.c | 240 ++++ reftable/iter.h | 63 + reftable/merged.c | 325 +++++ reftable/merged.h | 38 + reftable/pq.c | 114 ++ reftable/pq.h | 34 + reftable/reader.c | 753 ++++++++++ reftable/reader.h | 65 + reftable/record.c | 1141 +++++++++++++++ reftable/record.h | 121 ++ reftable/refname.c | 215 +++ reftable/refname.h | 38 + reftable/reftable.c | 91 ++ reftable/reftable.h | 564 ++++++++ reftable/slice.c | 225 +++ reftable/slice.h | 76 + reftable/stack.c | 1245 +++++++++++++++++ reftable/stack.h | 48 + reftable/system.h | 54 + reftable/tree.c | 67 + reftable/tree.h | 34 + reftable/update.sh | 24 + reftable/writer.c | 665 +++++++++ reftable/writer.h | 60 + reftable/zlib-compat.c | 92 ++ repository.c | 2 + repository.h | 3 + setup.c | 12 +- t/t0002-gitfile.sh | 2 +- t/t0031-reftable.sh | 120 ++ t/t1400-update-ref.sh | 32 +- t/t1409-avoid-packing-refs.sh | 6 + t/t1506-rev-parse-diagnosis.sh | 2 +- t/t3210-pack-refs.sh | 6 + t/t6050-replace.sh | 2 +- t/t9020-remote-svn.sh | 4 +- t/test-lib.sh | 5 + 62 files changed, 10201 insertions(+), 207 deletions(-) create mode 100644 Documentation/technical/reftable.txt create mode 100644 refs/reftable-backend.c create mode 100644 reftable/.gitattributes create mode 100644 reftable/LICENSE create mode 100644 reftable/README.md create mode 100644 reftable/VERSION create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/constants.h create mode 100644 reftable/file.c create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/refname.c create mode 100644 reftable/refname.h create mode 100644 reftable/reftable.c create mode 100644 reftable/reftable.h create mode 100644 reftable/slice.c create mode 100644 reftable/slice.h create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/system.h create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100755 reftable/update.sh create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h create mode 100644 reftable/zlib-compat.c create mode 100755 t/t0031-reftable.sh
base-commit: b994622632154fc3b17fb40a38819ad954a5fb88
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-539%2Fhanwen%2Freftable-v13
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-539/hanwen/reftable-v13
Pull-Request: https://github.com/gitgitgadget/git/pull/539
Range-diff vs v12:
1: dfa5fd74f85 ! 1: 8394c156eb4 refs.h: clarify reflog iteration order
@@ refs.h: int delete_refs(const char *msg, struct string_list *refnames,
int delete_reflog(const char *refname);
-/* iterate over reflog entries */
-+/* Iterate over reflog entries. */
++/* Callback to process a reflog entry found by the iteration functions (see ++ * below) */
typedef int each_reflog_ent_fn(
struct object_id *old_oid, struct object_id *new_oid, const char *committer, timestamp_t timestamp,
int tz, const char *msg, void *cb_data);
-+/* Iterate in over reflog entries, oldest entry first. */
++/* Iterate in over reflog entries in the log for refname. */
++
++/* oldest entry first */
int refs_for_each_reflog_ent(struct ref_store *refs, const char *refname,
each_reflog_ent_fn fn, void *cb_data);
++
++/* youngest entry first */
int refs_for_each_reflog_ent_reverse(struct ref_store *refs,
const char *refname,
each_reflog_ent_fn fn,
void *cb_data);
+
-+/* Call a function for each reflog entry, oldest entry first. */
++/* Call a function for each reflog entry in the log for refname. */
++
++/* oldest entry first */
int for_each_reflog_ent(const char *refname, each_reflog_ent_fn fn, void *cb_data);
++
++/* youngest entry first */
int for_each_reflog_ent_reverse(const char *refname, each_reflog_ent_fn fn, void *cb_data);
+ /*
11: 2abcbd1af99 = 2: dbf45fe8753 t: use update-ref and show-ref to reading/writing refs
3: 6553285043b = 3: be083a85fb5 refs: document how ref_iterator_advance_fn should handle symrefs
5: 06fcb49e903 ! 4: 96fd9814a67 reftable: file format documentation
@@ Documentation/technical/reftable.txt (new)
+
+Some repositories contain a lot of references (e.g. android at 866k,
+rails at 31k). The existing packed-refs format takes up a lot of space
-+(e.g. 62M), and does not scale with additional references. Lookup of a
++(e.g. 62M), and does not scale with additional references. Lookup of a
+single reference requires linearly scanning the file.
+
+Atomic pushes modifying multiple references require copying the entire
+packed-refs file, which can be a considerable amount of data moved
-+(e.g. 62M in, 62M out) for even small transactions (2 refs modified).
++(e.g. 62M in, 62M out) for even small transactions (2 refs modified).
+
+Repositories with many loose references occupy a large number of disk
+blocks from the local file system, as each reference is its own file
@@ Documentation/technical/reftable.txt (new)
+Performance
+^^^^^^^^^^^
+
-+Space used, packed-refs vs. reftable:
++Space used, packed-refs vs. reftable:
+
+[cols=",>,>,>,>,>",options="header",]
+|===============================================================
@@ Documentation/technical/reftable.txt (new)
+|reftable |hot | |20.2 usec |320.8 usec
+|=========================================================
+
-+Space used for 149,932 log entries for 43,061 refs, reflog vs. reftable:
++Space used for 149,932 log entries for 43,061 refs, reflog vs. reftable:
+
+[cols=",>,>",options="header",]
+|================================
@@ Documentation/technical/reftable.txt (new)
+index] to support fast lookup. Readers must be able to read both aligned
+and non-aligned files.
+
-+Very small files (e.g. 1 only ref block) may omit `padding` and the ref
++Very small files (e.g. a single ref block) may omit `padding` and the ref
+index to reduce total file size.
+
+Header
@@ Documentation/technical/reftable.txt (new)
+
+The index may be organized into a multi-level index, where the 1st level
+index block points to additional ref index blocks (2nd level), which may
-+in turn point to either additional index blocks (e.g. 3rd level) or ref
++in turn point to either additional index blocks (e.g. 3rd level) or ref
+blocks (leaf level). Disk reads required to access a ref go up with
+higher index levels. Multi-level indexes may be required to ensure no
+single index block exceeds the file format’s max block size of
@@ Documentation/technical/reftable.txt (new)
+
+The first `position_delta` is the position from the start of the file.
+Additional `position_delta` entries are sorted ascending and relative to
-+the prior entry, e.g. a reader would perform:
++the prior entry, e.g. a reader would perform:
+
+....
+pos = position_delta[0]
@@ Documentation/technical/reftable.txt (new)
+ uint32( CRC-32 of above )
+....
+
-+If a section is missing (e.g. ref index) the corresponding position
++If a section is missing (e.g. ref index) the corresponding position
+field (e.g. `ref_index_position`) will be 0.
+
+* `obj_position`: byte position for the first obj block.
@@ Documentation/technical/reftable.txt (new)
+Low overhead
+^^^^^^^^^^^^
+
-+A reftable with very few references (e.g. git.git with 5 heads) is 269
-+bytes for reftable, vs. 332 bytes for packed-refs. This supports
++A reftable with very few references (e.g. git.git with 5 heads) is 269
++bytes for reftable, vs. 332 bytes for packed-refs. This supports
+reftable scaling down for transaction logs (below).
+
+Block size
@@ Documentation/technical/reftable.txt (new)
+repository is single-threaded for writers. Writers may have to busy-spin
+(with backoff) around creating `tables.list.lock`, for up to an
+acceptable wait period, aborting if the repository is too busy to
-+mutate. Application servers wrapped around repositories (e.g. Gerrit
++mutate. Application servers wrapped around repositories (e.g. Gerrit
+Code Review) can layer their own lock/wait queue to improve fairness to
+writers.
+
@@ Documentation/technical/reftable.txt (new)
+bzip packed-refs
+^^^^^^^^^^^^^^^^
+
-+`bzip2` can significantly shrink a large packed-refs file (e.g. 62 MiB
++`bzip2` can significantly shrink a large packed-refs file (e.g. 62 MiB
+compresses to 23 MiB, 37%). However the bzip format does not support
+random access to a single reference. Readers must inflate and discard
+while performing a linear scan.
@@ Documentation/technical/reftable.txt (new)
+interleaved between refs.
+
+Performance testing indicates reftable is faster for lookups (51%
-+faster, 11.2 usec vs. 5.4 usec), although reftable produces a slightly
++faster, 11.2 usec vs. 5.4 usec), although reftable produces a slightly
+larger file (+ ~3.2%, 28.3M vs 29.2M):
+
+[cols=">,>,>,>",options="header",]
@@ Documentation/technical/reftable.txt (new)
+The RefTree format adds additional load on the object database storage
+layer (more loose objects, more objects in packs), and relies heavily on
+the packer’s delta compression to save space. Namespaces which are flat
-+(e.g. thousands of tags in refs/tags) initially create very large loose
++(e.g. thousands of tags in refs/tags) initially create very large loose
+objects, and so RefTree does not address the problem of copying many
+references to modify a handful.
+
@@ Documentation/technical/reftable.txt (new)
+Longer hashes
+^^^^^^^^^^^^^
+
-+Version will bump (e.g. 2) to indicate `value` uses a different object
++Version will bump (e.g. 2) to indicate `value` uses a different object
+id length other than 20. The length could be stored in an expanded file
+header, or hardcoded as part of the version.
7: 6d9031372ce = 5: 7aa3f92fca0 reftable: clarify how empty tables should be written
6: 093fa74a3d0 ! 6: 1e3c8f2d3e8 reftable: define version 2 of the spec to accomodate SHA256 @@ Metadata
## Commit message ##
reftable: define version 2 of the spec to accomodate SHA256
+ Version appends a hash ID to the file header, making it slightly larger.
+
+ This commit also changes "SHA-1" into "object ID" in many places.
+
## Documentation/technical/reftable.txt ##
+@@ Documentation/technical/reftable.txt: Objectives
+
+ * Near constant time lookup for any single reference, even when the
+ repository is cold and not in process or kernel cache.
+-* Near constant time verification if a SHA-1 is referred to by at least ++* Near constant time verification if an object ID is referred to by at least
+ one reference (for allow-tip-sha1-in-want).
+ * Efficient lookup of an entire namespace, such as refs/tags/.
+ * Support atomic push with O(size_of_update) operations.
@@ Documentation/technical/reftable.txt: and non-aligned files.
- Very small files (e.g. 1 only ref block) may omit padding and the ref
+ Very small files (e.g. a single ref block) may omit padding and the ref index to reduce total file size.
-Header
@@ Documentation/technical/reftable.txt: used in a stack for link:#Update-transacti
+
+....
+'REFT'
-+uint8( version_number = 1 )
++uint8( version_number = 2 )
+uint24( block_size )
+uint64( min_update_index )
+uint64( max_update_index )
@@ Documentation/technical/reftable.txt: used in a stack for link:#Update-transacti
+The header is identical to version_number=1, with the 4-byte hash ID +("sha1" for SHA1 and "s256" for SHA-256) append to the header.
+
++For maximum backward compatibility, it is recommended to use version 1 when ++writing SHA1 reftables.
++
+
First ref block
^^^^^^^^^^^^^^^
+@@ Documentation/technical/reftable.txt: The value follows. Its format is determined by value_type, one of
+ the following:
+
+ * 0x0: deletion; no value data (see transactions, below)
+-* 0x1: one 20-byte object id; value of the ref
+-* 0x2: two 20-byte object ids; value of the ref, peeled target
++* 0x1: one object id; value of the ref
++* 0x2: two object ids; value of the ref, peeled target
+ * 0x3: symbolic reference: varint( target_len ) target
+
+ Symbolic references use 0x3, followed by the complete name of the
+@@ Documentation/technical/reftable.txt: Obj block format
+ ^^^^^^^^^^^^^^^^
+
+ Object blocks are optional. Writers may choose to omit object blocks,
+-especially if readers will not use the SHA-1 to ref mapping.
++especially if readers will not use the object ID to ref mapping.
+
+-Object blocks use unique, abbreviated 2-20 byte SHA-1 keys, mapping to
++Object blocks use unique, abbreviated 2-32 byte object ID keys, mapping to
+ ref blocks containing references pointing to that object directly, or as
+ the peeled value of an annotated tag. Like ref blocks, object blocks use
+ the file’s standard block size. The abbrevation length is available in
+@@ Documentation/technical/reftable.txt: the footer as obj_id_len.
+ To save space in small files, object blocks may be omitted if the ref
+ index is not present, as brute force search will only need to read a few
+ ref blocks. When missing, readers should brute force a linear search of
+-all references to lookup by SHA-1.
++all references to lookup by object ID.
+
+ An object block is written as:
+
+@@ Documentation/technical/reftable.txt: works the same as in reference blocks.
+
+ Because object identifiers are abbreviated by writers to the shortest
+ unique abbreviation within the reftable, obj key lengths are variable
+-between 2 and 20 bytes. Readers must compare only for common prefix
++between 2 and 32 bytes. Readers must compare only for common prefix
+ match within an obj block or obj index.
+
+ obj record
+@@ Documentation/technical/reftable.txt: for (j = 1; j < position_count; j++) {
+ ....
+
+ With a position in hand, a reader must linearly scan the ref block,
+-starting from the first ref_record, testing each reference’s SHA-1s
++starting from the first ref_record, testing each reference’s object IDs
+ (for value_type = 0x1 or 0x2) for full equality. Faster searching by
+-SHA-1 within a single ref block is not supported by the reftable format.
++object ID within a single ref block is not supported by the reftable format.
+ Smaller block sizes reduce the number of candidates this step must
+ consider.
+
+@@ Documentation/technical/reftable.txt: reflogs must treat this as a deletion.
+ For log_type = 0x1, the log_data section follows
+ linkgit:git-update-ref[1] logging and includes:
+
+-* two 20-byte SHA-1s (old id, new id)
++* two object IDs (old id, new id)
+ * varint string of committer’s name
+ * varint string of committer’s email
+ * varint time in seconds since epoch (Jan 1, 1970)
@@ Documentation/technical/reftable.txt: Footer
After the last block of the file, a file footer is written. It begins
like the file header, but is extended with additional data.
@@ Documentation/technical/reftable.txt: obj blocks.
* 4-byte magic is correct
* 1-byte version number is recognized
+@@ Documentation/technical/reftable.txt: Lightweight refs dominate
+ ^^^^^^^^^^^^^^^^^^^^^^^^^
+
+ The reftable format assumes the vast majority of references are single +-SHA-1 valued with common prefixes, such as Gerrit Code Review’s ++object IDs valued with common prefixes, such as Gerrit Code Review’s
+ refs/changes/ namespace, GitHub’s refs/pulls/ namespace, or many
+ lightweight tags in the refs/tags/ namespace.
+
+-Annotated tags storing the peeled object cost an additional 20 bytes per ++Annotated tags storing the peeled object cost an additional object ID per
+ reference.
+
+ Low overhead
+@@ Documentation/technical/reftable.txt: Scans and lookups dominate +
+ Scanning all references and lookup by name (or namespace such as
+ refs/heads/) are the most common activities performed on repositories. +-SHA-1s are stored directly with references to optimize this use case. ++Object IDs are stored directly with references to optimize this use case. +
+ Logs are infrequently read
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ Documentation/technical/reftable.txt: impossible.
A common format that can be supported by all major Git implementations
@@ Documentation/technical/reftable.txt: impossible.
-Longer hashes
-^^^^^^^^^^^^^
-
--Version will bump (e.g. 2) to indicate value uses a different object +-Version will bump (e.g. 2) to indicate value uses a different object
-id length other than 20. The length could be stored in an expanded file -header, or hardcoded as part of the version.
-: ----------- > 7: 2c2f94ddc0e Write pseudorefs through ref backends.
2: 340c5c415e1 = 8: 3becaaee66a Iterate over the "refs/" namespace in for_each_[raw]ref
4: 7dc47c7756f = 9: a6f77965f84 Add .gitattributes for the reftable/ directory
8: 57d338c4983 ! 10: 8103703c358 Add reftable library
@@ reftable/README.md (new)
## reftable/VERSION (new) ##
@@
-+commit 06dd91a8377b0f920d9835b9835d0d650f928dce
++commit e74c14b66b6c15f6526c485f2e45d3f2735d359d
+Author: Han-Wen Nienhuys <[email protected]>
-+Date: Wed May 6 21:27:32 2020 +0200
-+
-+ C: fix small memory leak in stack.c error path
++Date: Mon May 11 21:02:55 2020 +0200
++
++ C: handle out-of-date reftable stacks
++
++ * Make reftable_stack_reload() check out-of-dateness. This makes it very cheap,
++ allowing it to be called often. This is useful because Git calls itself often,
++ which effectively requires a reload.
++
++ * In reftable_stack_add(), check the return value of stack_uptodate(), leading
++ to erroneously succeeding the transaction.
++
++ * A test that exercises the above.
## reftable/basics.c (new) ##
@@
@@ reftable/block.c (new)
+
+int block_iter_next(struct block_iter *it, struct record rec)
+{
++ struct slice in = {
++ .buf = it->br->block.data + it->next_off,
++ .len = it->br->block_len - it->next_off,
++ };
++ struct slice start = in;
++ struct slice key = { 0 };
++ byte extra = 0;
++ int n = 0;
++
+ if (it->next_off >= it->br->block_len) {
+ return 1;
+ }
+
-+ {
-+ struct slice in = {
-+ .buf = it->br->block.data + it->next_off,
-+ .len = it->br->block_len - it->next_off,
-+ };
-+ struct slice start = in;
-+ struct slice key = { 0 };
-+ byte extra;
-+ int n = decode_key(&key, &extra, it->last_key, in);
-+ if (n < 0) {
-+ return -1;
-+ }
-+
-+ slice_consume(&in, n);
-+ n = record_decode(rec, key, extra, in, it->br->hash_size);
-+ if (n < 0) {
-+ return -1;
-+ }
-+ slice_consume(&in, n);
++ n = decode_key(&key, &extra, it->last_key, in);
++ if (n < 0) {
++ return -1;
++ }
+
-+ slice_copy(&it->last_key, key);
-+ it->next_off += start.len - in.len;
-+ slice_clear(&key);
-+ return 0;
++ slice_consume(&in, n);
++ n = record_decode(rec, key, extra, in, it->br->hash_size);
++ if (n < 0) {
++ return -1;
+ }
++ slice_consume(&in, n);
++
++ slice_copy(&it->last_key, key);
++ it->next_off += start.len - in.len;
++ slice_clear(&key);
++ return 0;
+}
+
+int block_reader_first_key(struct block_reader *br, struct slice *key)
@@ reftable/block.c (new)
+ .key = want,
+ .r = br,
+ };
++ struct record rec = new_record(block_reader_type(br));
++ struct slice key = { 0 };
++ int err = 0;
++ struct block_iter next = { 0 };
+
+ int i = binsearch(br->restart_count, &restart_key_less, &args);
+ if (args.error) {
-+ return -1;
++ err = REFTABLE_FORMAT_ERROR;
++ goto exit;
+ }
+
+ it->br = br;
@@ reftable/block.c (new)
+ it->next_off = br->header_off + 4;
+ }
+
-+ {
-+ struct record rec = new_record(block_reader_type(br));
-+ struct slice key = { 0 };
-+ int result = 0;
-+ int err = 0;
-+ struct block_iter next = { 0 };
-+ while (true) {
-+ block_iter_copy_from(&next, it);
-+
-+ err = block_iter_next(&next, rec);
-+ if (err < 0) {
-+ result = -1;
-+ goto exit;
-+ }
-+
-+ record_key(rec, &key);
-+ if (err > 0 || slice_compare(key, want) >= 0) {
-+ result = 0;
-+ goto exit;
-+ }
-+
-+ block_iter_copy_from(it, &next);
++ /* We're looking for the last entry less/equal than the wanted key, so
++ we have to go one entry too far and then back up.
++ */
++ while (true) {
++ block_iter_copy_from(&next, it);
++ err = block_iter_next(&next, rec);
++ if (err < 0) {
++ goto exit;
+ }
+
-+ exit:
-+ slice_clear(&key);
-+ slice_clear(&next.last_key);
-+ record_destroy(&rec);
++ record_key(rec, &key);
++ if (err > 0 || slice_compare(key, want) >= 0) {
++ err = 0;
++ goto exit;
++ }
+
-+ return result;
++ block_iter_copy_from(it, &next);
+ }
++
++exit:
++ slice_clear(&key);
++ slice_clear(&next.last_key);
++ record_destroy(&rec);
++
++ return err;
+}
+
+void block_writer_clear(struct block_writer *bw)
@@ reftable/merged.c (new)
+ if (err < 0) {
+ return err;
+ }
-+ record_clear(top.rec);
-+ reftable_free(record_yield(&top.rec));
++ record_destroy(&top.rec);
+ }
+
+ record_copy_from(rec, entry.rec, hash_size(mi->hash_id));
-+ record_clear(entry.rec);
-+ reftable_free(record_yield(&entry.rec));
++ record_destroy(&entry.rec);
+ slice_clear(&entry_key);
+ return 0;
+}
@@ reftable/pq.c (new)
+{
+ int i = 0;
+ for (i = 0; i < pq->len; i++) {
-+ record_clear(pq->heap[i].rec);
-+ reftable_free(record_yield(&pq->heap[i].rec));
++ record_destroy(&pq->heap[i].rec);
+ }
+ FREE_AND_NULL(pq->heap);
+ pq->len = pq->cap = 0;
@@ reftable/record.c (new)
+ }
+ slice_consume(&in, n);
+ seen_target = true;
++ if (r->target != NULL) {
++ reftable_free(r->target);
++ }
+ r->target = (char *)slice_as_string(&dest);
+ } break;
+
@@ reftable/record.c (new)
+ r->update_index = (~max) - ts;
+
+ if (val_type == 0) {
++ FREE_AND_NULL(r->old_hash);
++ FREE_AND_NULL(r->new_hash);
++ FREE_AND_NULL(r->message);
++ FREE_AND_NULL(r->email);
++ FREE_AND_NULL(r->name);
+ return 0;
+ }
+
@@ reftable/refname.c (new)
+static int find_name(size_t k, void *arg)
+{
+ struct find_arg *f_arg = (struct find_arg *)arg;
-+
+ return strcmp(f_arg->names[k], f_arg->want) >= 0;
+}
+
@@ reftable/refname.c (new)
+
+static void modification_clear(struct modification *mod)
+{
++ /* don't delete the strings themselves; they're owned by ref records.
++ */
+ FREE_AND_NULL(mod->add);
+ FREE_AND_NULL(mod->del);
+ mod->add_len = 0;
@@ reftable/refname.c (new)
+ goto exit;
+ }
+ }
-+
+ err = reftable_table_seek_ref(mod->tab, &it, prefix);
+ if (err) {
+ goto exit;
@@ reftable/reftable.h (new)
+/* Commits the transaction, releasing the lock. */
+int reftable_addition_commit(struct reftable_addition *add);
+
-+/* Release all non-committed data from the transaction; releases the lock if
-+ * held. */
-+void reftable_addition_close(struct reftable_addition *add);
++/* Release all non-committed data from the transaction, and deallocate the
++ transaction. Releases the lock if held. */
++void reftable_addition_destroy(struct reftable_addition *add);
+
+/* add a new table to the stack. The write_table function must call
+ reftable_writer_set_limits, add refs and return an error value. */
@@ reftable/reftable.h (new)
+/* frees all resources associated with the stack. */
+void reftable_stack_destroy(struct reftable_stack *st);
+
-+/* reloads the stack if necessary. */
++/* Reloads the stack if necessary. This is very cheap to run if the stack was up
++ * to date */
+int reftable_stack_reload(struct reftable_stack *st);
+
+/* Policy for expiring reflog entries. */
@@ reftable/stack.c (new)
+ p->reftable_dir = xstrdup(dir);
+ p->config = config;
+
-+ err = reftable_stack_reload(p);
++ err = reftable_stack_reload_maybe_reuse(p, true);
+ if (err < 0) {
+ reftable_stack_destroy(p);
+ } else {
@@ reftable/stack.c (new)
+/* Close and free the stack */
+void reftable_stack_destroy(struct reftable_stack *st)
+{
-+ if (st->merged == NULL) {
-+ return;
++ if (st->merged != NULL) {
++ reftable_merged_table_close(st->merged);
++ reftable_merged_table_free(st->merged);
++ st->merged = NULL;
+ }
-+
-+ reftable_merged_table_close(st->merged);
-+ reftable_merged_table_free(st->merged);
-+ st->merged = NULL;
-+
+ FREE_AND_NULL(st->list_file);
+ FREE_AND_NULL(st->reftable_dir);
+ reftable_free(st);
@@ reftable/stack.c (new)
+ return udiff;
+}
+
-+static int reftable_stack_reload_maybe_reuse(struct reftable_stack *st,
-+ bool reuse_open)
++int reftable_stack_reload_maybe_reuse(struct reftable_stack *st,
++ bool reuse_open)
+{
+ struct timeval deadline = { 0 };
+ int err = gettimeofday(&deadline, NULL);
@@ reftable/stack.c (new)
+ return 0;
+}
+
-+int reftable_stack_reload(struct reftable_stack *st)
-+{
-+ return reftable_stack_reload_maybe_reuse(st, true);
-+}
-+
+/* -1 = error
+ 0 = up to date
+ 1 = changed. */
@@ reftable/stack.c (new)
+ return err;
+}
+
++int reftable_stack_reload(struct reftable_stack *st)
++{
++ int err = stack_uptodate(st);
++ if (err > 0) {
++ return reftable_stack_reload_maybe_reuse(st, true);
++ }
++ return err;
++}
++
+int reftable_stack_add(struct reftable_stack *st,
+ int (*write)(struct reftable_writer *wr, void *arg),
+ void *arg)
@@ reftable/stack.c (new)
+ slice_clear(&nm);
+}
+
++void reftable_addition_destroy(struct reftable_addition *add)
++{
++ if (add == NULL) {
++ return;
++ }
++ reftable_addition_close(add);
++ reftable_free(add);
++}
++
+int reftable_addition_commit(struct reftable_addition *add)
+{
+ struct slice table_list = { 0 };
@@ reftable/stack.c (new)
+ struct reftable_stack *st)
+{
+ int err = 0;
-+ *dest = reftable_malloc(sizeof(**dest));
++ *dest = reftable_calloc(sizeof(**dest));
+ err = reftable_stack_init_addition(*dest, st);
+ if (err) {
+ reftable_free(*dest);
@@ reftable/stack.c (new)
+ if (err < 0) {
+ goto exit;
+ }
++ if (err > 0) {
++ err = REFTABLE_LOCK_ERROR;
++ goto exit;
++ }
+
+ err = reftable_addition_add(&add, write_table, arg);
+ if (err < 0) {
@@ reftable/stack.c (new)
+ slice_append_string(&tab_file_name, "/");
+ slice_append(&tab_file_name, next_name);
+
++ /* TODO: should check destination out of paranoia */
+ err = rename(slice_as_string(&temp_tab_file_name),
+ slice_as_string(&tab_file_name));
+ if (err < 0) {
@@ reftable/stack.c (new)
+ if (err < 0) {
+ goto exit;
+ }
-+ reftable_writer_free(wr);
+
+ err = close(tab_fd);
+ tab_fd = 0;
+
+exit:
++ reftable_writer_free(wr);
+ if (tab_fd > 0) {
+ close(tab_fd);
+ tab_fd = 0;
@@ reftable/stack.c (new)
+
+ err = validate_ref_record_addition(tab, refs, len);
+
++exit:
+ for (i = 0; i < len; i++) {
+ reftable_ref_record_clear(&refs[i]);
+ }
+
-+exit:
+ free(refs);
+ reftable_iterator_destroy(&it);
+ reftable_reader_free(rd);
@@ reftable/stack.h (new)
+ struct reftable_log_expiry_config *config);
+int fastlog2(uint64_t sz);
+int stack_check_addition(struct reftable_stack *st, const char *new_tab_name);
++void reftable_addition_close(struct reftable_addition *add);
++int reftable_stack_reload_maybe_reuse(struct reftable_stack *st,
++ bool reuse_open);
+
+struct segment {
+ int start, end;
@@ reftable/writer.c (new)
+ writer_reinit_block_writer(w, BLOCK_TYPE_INDEX);
+
+ err = block_writer_add(w->block_writer, rec);
-+ assert(err == 0);
++ if (err != 0) {
++ /* write into fresh block should always succeed
++ */
++ abort();
++ }
+ }
+ for (i = 0; i < idx_len; i++) {
+ slice_clear(&idx[i].last_key);
9: f3bb9410038 ! 11: ace95b6cd88 Reftable support for git-core
@@ refs/reftable-backend.c (new)
+ break;
+ }
+
++ /*
++ We could filter pseudo refs here explicitly, but HEAD is not
++ a PSEUDOREF, but a PER_WORKTREE, b/c each worktree can have
++ its own HEAD.
++ */
+ ri->base.refname = ri->ref.ref_name;
+ if (ri->prefix != NULL &&
+ strncmp(ri->prefix, ri->ref.ref_name, strlen(ri->prefix))) {
@@ refs/reftable-backend.c (new)
+
+static int reftable_transaction_prepare(struct ref_store *ref_store,
+ struct ref_transaction *transaction,
-+ struct strbuf *err)
++ struct strbuf *errbuf)
+{
-+ return 0;
++ /* XXX rewrite using the reftable transaction API. */
++ struct git_reftable_ref_store *refs =
++ (struct git_reftable_ref_store *)ref_store;
++ int err = refs->err;
++ if (err < 0) {
++ goto done;
++ }
++ err = reftable_stack_reload(refs->stack);
++ if (err) {
++ goto done;
++ }
++
++done:
++ return err;
+}
+
+static int reftable_transaction_abort(struct ref_store *ref_store,
@@ refs/reftable-backend.c (new)
+ return reftable_transaction_commit(ref_store, transaction, err);
+}
+
++struct write_pseudoref_arg {
++ struct reftable_stack *stack;
++ const char *pseudoref;
++ const struct object_id *new_oid;
++ const struct object_id *old_oid;
++};
++
++static int write_pseudoref_table(struct reftable_writer *writer, void *argv)
++{
++ struct write_pseudoref_arg *arg = (struct write_pseudoref_arg *)argv;
++ uint64_t ts = reftable_stack_next_update_index(arg->stack);
++ int err = 0;
++ struct reftable_ref_record read_ref = { NULL };
++ struct reftable_ref_record write_ref = { NULL };
++
++ reftable_writer_set_limits(writer, ts, ts);
++ if (arg->old_oid) {
++ struct object_id read_oid;
++ err = reftable_stack_read_ref(arg->stack, arg->pseudoref,
++ &read_ref);
++ if (err < 0)
++ goto done;
++
++ if ((err > 0) != is_null_oid(arg->old_oid)) {
++ err = REFTABLE_LOCK_ERROR;
++ goto done;
++ }
++
++ /* XXX If old_oid is set, and we have a symref? */
++
++ if (err == 0 && read_ref.value == NULL) {
++ err = REFTABLE_LOCK_ERROR;
++ goto done;
++ }
++
++ hashcpy(read_oid.hash, read_ref.value);
++ if (!oideq(arg->old_oid, &read_oid)) {
++ err = REFTABLE_LOCK_ERROR;
++ goto done;
++ }
++ }
++
++ write_ref.ref_name = (char *)arg->pseudoref;
++ write_ref.update_index = ts;
++ if (!is_null_oid(arg->new_oid))
++ write_ref.value = (uint8_t *)arg->new_oid->hash;
++
++ err = reftable_writer_add_ref(writer, &write_ref);
++done:
++ reftable_ref_record_clear(&read_ref);
++ return err;
++}
++
++static int reftable_write_pseudoref(struct ref_store *ref_store,
++ const char *pseudoref,
++ const struct object_id *oid,
++ const struct object_id *old_oid,
++ struct strbuf *errbuf)
++{
++ struct git_reftable_ref_store *refs =
++ (struct git_reftable_ref_store *)ref_store;
++ struct write_pseudoref_arg arg = {
++ .stack = refs->stack,
++ .pseudoref = pseudoref,
++ .new_oid = oid,
++ };
++ struct reftable_addition *add = NULL;
++ int err = refs->err;
++ if (err < 0) {
++ goto done;
++ }
++
++ err = reftable_stack_reload(refs->stack);
++ if (err) {
++ goto done;
++ }
++ err = reftable_stack_new_addition(&add, refs->stack);
++ if (err) {
++ goto done;
++ }
++ if (old_oid) {
++ struct object_id actual_old_oid;
++
++ /* XXX this is cut & paste from files-backend - should factor
++ * out? */
++ if (read_ref(pseudoref, &actual_old_oid)) {
++ if (!is_null_oid(old_oid)) {
++ strbuf_addf(errbuf,
++ _("could not read ref '%s'"),
++ pseudoref);
++ goto done;
++ }
++ } else if (is_null_oid(old_oid)) {
++ strbuf_addf(errbuf, _("ref '%s' already exists"),
++ pseudoref);
++ goto done;
++ } else if (!oideq(&actual_old_oid, old_oid)) {
++ strbuf_addf(errbuf,
++ _("unexpected object ID when writing '%s'"),
++ pseudoref);
++ goto done;
++ }
++ }
++
++ err = reftable_addition_add(add, &write_pseudoref_table, &arg);
++ if (err < 0) {
++ strbuf_addf(errbuf, "reftable: pseudoref update failure: %s",
++ reftable_error_str(err));
++ }
++
++ err = reftable_addition_commit(add);
++ if (err < 0) {
++ strbuf_addf(errbuf, "reftable: pseudoref commit failure: %s",
++ reftable_error_str(err));
++ }
++
++done:
++ reftable_addition_destroy(add);
++ return err;
++}
++
++static int reftable_delete_pseudoref(struct ref_store *ref_store,
++ const char *pseudoref,
++ const struct object_id *old_oid)
++{
++ struct strbuf errbuf = STRBUF_INIT;
++ int ret = reftable_write_pseudoref(ref_store, pseudoref, &null_oid,
++ old_oid, &errbuf);
++ /* XXX what to do with the error message? */
++ strbuf_release(&errbuf);
++ return ret;
++}
++
+struct write_delete_refs_arg {
+ struct reftable_stack *stack;
+ struct string_list *refnames;
@@ refs/reftable-backend.c (new)
+ .logmsg = msg,
+ .flags = flags,
+ };
-+ if (refs->err < 0) {
-+ return refs->err;
++ int err = refs->err;
++ if (err < 0) {
++ goto done;
+ }
-+
-+ return reftable_stack_add(refs->stack, &write_delete_refs_table, &arg);
++ err = reftable_stack_reload(refs->stack);
++ if (err) {
++ goto done;
++ }
++ err = reftable_stack_add(refs->stack, &write_delete_refs_table, &arg);
++done:
++ return err;
+}
+
+static int reftable_pack_refs(struct ref_store *ref_store, unsigned int flags)
@@ refs/reftable-backend.c (new)
+ .refname = refname,
+ .target = target,
+ .logmsg = logmsg };
-+ if (refs->err < 0) {
-+ return refs->err;
++ int err = refs->err;
++ if (err < 0) {
++ goto done;
++ }
++ err = reftable_stack_reload(refs->stack);
++ if (err) {
++ goto done;
+ }
-+ return reftable_stack_add(refs->stack, &write_create_symref_table,
-+ &arg);
++ err = reftable_stack_add(refs->stack, &write_create_symref_table, &arg);
++done:
++ return err;
+}
+
+struct write_rename_arg {
@@ refs/reftable-backend.c (new)
+ .newname = newrefname,
+ .logmsg = logmsg,
+ };
-+ if (refs->err < 0) {
-+ return refs->err;
++ int err = refs->err;
++ if (err < 0) {
++ goto done;
++ }
++ err = reftable_stack_reload(refs->stack);
++ if (err) {
++ goto done;
+ }
+
-+ return reftable_stack_add(refs->stack, &write_rename_table, &arg);
++ err = reftable_stack_add(refs->stack, &write_rename_table, &arg);
++done:
++ return err;
+}
+
+static int reftable_copy_ref(struct ref_store *ref_store,
@@ refs/reftable-backend.c (new)
+ reftable_rename_ref,
+ reftable_copy_ref,
+
++ reftable_write_pseudoref,
++ reftable_delete_pseudoref,
++
+ reftable_ref_iterator_begin,
+ reftable_read_raw_ref,
+
@@ t/t0031-reftable.sh (new)
+ test_cmp expect actual
+'
+
++# cherry-pick uses a pseudo ref.
++test_expect_success 'pseudo refs' '
++ initialize &&
++ test_commit message1 file1 &&
++ test_commit message2 file2 &&
++ git branch source &&
++ git checkout HEAD^ &&
++ test_commit message3 file3 &&
++ git cherry-pick source &&
++ test -f file2
++'
+
+test_done
+
10: 94fcc6dca6a = 12: b96f0712d44 vcxproj: adjust for the reftable changes
12: fe9407d10b1 = 13: 0e732d30b51 Add some reftable testing infrastructureSignature · 1 line
gitgitgadget
PATCH v13 1/13 refs.h: clarify reflog iteration order
From: Han-Wen Nienhuys <[email protected]>
--- refs.h | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-)
refs.h +13 -1
@@ -432,19 +432,31 @@ int delete_refs(const char *msg, struct string_list *refnames,int refs_delete_reflog (struct ref_store *refs,const char *refname);int delete_reflog (const char *refname); -/* iterate over reflog entries */ +/* Callback to process a reflog entry found by the iteration functions (see +* below) */ typedef int each_reflog_ent_fn (struct object_id *old_oid,struct object_id *new_oid,const char *committer,timestamp_t timestamp,int tz,const char *msg,void *cb_data); +/* Iterate in over reflog entries in the log for `refname`. */ + +/* oldest entry first */ int refs_for_each_reflog_ent (struct ref_store *refs,const char *refname,each_reflog_ent_fn fn,void *cb_data); + +/* youngest entry first */ int refs_for_each_reflog_ent_reverse (struct ref_store *refs,const char *refname,each_reflog_ent_fn fn,void *cb_data); + +/* Call a function for each reflog entry in the log for `refname`. */ + +/* oldest entry first */ int for_each_reflog_ent (const char *refname,each_reflog_ent_fn fn,void *cb_data); + +/* youngest entry first */ int for_each_reflog_ent_reverse (const char *refname,each_reflog_ent_fn fn,void *cb_data);/*
Signature · 1 line
gitgitgadget
Re: [PATCH v13 01/13] refs.h: clarify reflog iteration order
Quoted 6 lines · Han-Wen Nienhuys via GitGitGadget
From: Han-Wen Nienhuys <[email protected]>
Signed-off-by Han-Wen Nienhuys <[email protected]>--- refs.h | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-)
I think the split of the topics was a good first move; now we need to make sure that we need no more work on the early part.
refs.h int delete_refs(const char *msg, struct string_list *refnames, · line 436
Quoted 11 lines
refs.h +2
-1@@ -432,19 +432,31 @@ int delete_refs(const char *msg, struct string_list *refnames,int refs_delete_reflog (struct ref_store *refs,const char *refname);int delete_reflog (const char *refname); -/* iterate over reflog entries */ +/* Callback to process a reflog entry found by the iteration functions (see +* below) */
/*
* A multi-line comment in our codebase
* look like this, with the opening slash-asterisk
* and closing asterisk-slash occupying their own
* lines.
*/
refs.h int delete_refs(const char *msg, struct string_list *refnames, · line 442
Quoted 6 lines
refs.h +1
-0typedef int each_reflog_ent_fn (struct object_id *old_oid,struct object_id *new_oid,const char *committer,timestamp_t timestamp,int tz,const char *msg,void *cb_data); +/* Iterate in over reflog entries in the log for `refname`. */
"in over"??? Should this just be "iterate over reflog entries in ..."?
Quoted 8 lines
refs.h +4
-0+ +/* oldest entry first */ int refs_for_each_reflog_ent (struct ref_store *refs,const char *refname,each_reflog_ent_fn fn,void *cb_data); + +/* youngest entry first */ int refs_for_each_reflog_ent_reverse(struct ref_store *refs, const char *refname,
refs.h int delete_refs(const char *msg, struct string_list *refnames, · line 460
Quoted 10 lines
refs.h +6
-0each_reflog_ent_fn fn, void *cb_data); + +/* Call a function for each reflog entry in the log for `refname`. */ + +/* oldest entry first */ int for_each_reflog_ent (const char *refname,each_reflog_ent_fn fn,void *cb_data); + +/* youngest entry first */ int for_each_reflog_ent_reverse (const char *refname,each_reflog_ent_fn fn,void *cb_data);
The only difference betwen refs_for_each_reflog_ent() and for_each_reflog_ent() is that the former takes an arbitrary ref-store while the latter works on the default ref-store, no? They should otherwise be identical, so describing one as "Iterate over reflog entries" while describing the other as "Call a function for each" feel misleading.
PATCH v13 2/13 t: use update-ref and show-ref to reading/writing refs
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Reading and writing .git/refs/* assumes that refs are stored in the 'files' ref backend.
--- t/t0002-gitfile.sh | 2 +- t/t1400-update-ref.sh | 32 ++++++++++++++++---------------- t/t1506-rev-parse-diagnosis.sh | 2 +- t/t6050-replace.sh | 2 +- t/t9020-remote-svn.sh | 4 ++-- 5 files changed, 21 insertions(+), 21 deletions(-)
t/t0002-gitfile.sh +1 -1
@@ -62,7 +62,7 @@ test_expect_success 'check commit-tree' ' ' test_expect_success 'check rev-list' ' - echo $SHA >"$REAL/HEAD" && + git update-ref "HEAD" "$SHA" && test "$SHA" = "$(git rev-list HEAD)" '
t/t1400-update-ref.sh +16 -16
@@ -37,15 +37,15 @@ test_expect_success setup ' test_expect_success "create $m" ' git update-ref $m $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success "create $m with oldvalue verification" ' git update-ref $m $B $A && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "fail to delete $m with stale ref" ' test_must_fail git update-ref -d $m $A && - test $B = "$(cat .git/$m)" + test $B = "$(git show-ref -s --verify $m)" ' test_expect_success "delete $m" ' test_when_finished "rm -f .git/$m" &&
@@ -56,7 +56,7 @@ test_expect_success "delete $m" ' test_expect_success "delete $m without oldvalue verification" ' test_when_finished "rm -f .git/$m" && git update-ref $m $A && - test $A = $(cat .git/$m) && + test $A = $(git show-ref -s --verify $m) && git update-ref -d $m && test_path_is_missing .git/$m '
@@ -69,15 +69,15 @@ test_expect_success "fail to create $n" ' test_expect_success "create $m (by HEAD)" ' git update-ref HEAD $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success "create $m (by HEAD) with oldvalue verification" ' git update-ref HEAD $B $A && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "fail to delete $m (by HEAD) with stale ref" ' test_must_fail git update-ref -d HEAD $A && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "delete $m (by HEAD)" ' test_when_finished "rm -f .git/$m" &&
@@ -178,14 +178,14 @@ test_expect_success '--no-create-reflog overrides core.logAllRefUpdates=always' test_expect_success "create $m (by HEAD)" ' git update-ref HEAD $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success 'pack refs' ' git pack-refs --all ' test_expect_success "move $m (by HEAD)" ' git update-ref HEAD $B $A && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "delete $m (by HEAD) should remove both packed and loose $m" ' test_when_finished "rm -f .git/$m" &&
@@ -255,7 +255,7 @@ test_expect_success '(not) change HEAD with wrong SHA1' ' ' test_expect_success "(not) changed .git/$m" ' test_when_finished "rm -f .git/$m" && - ! test $B = $(cat .git/$m) + ! test $B = $(git show-ref -s --verify $m) ' rm -f .git/logs/refs/heads/master
@@ -263,19 +263,19 @@ test_expect_success "create $m (logged by touch)" ' test_config core.logAllRefUpdates false && GIT_COMMITTER_DATE="2005-05-26 23:30" \ git update-ref --create-reflog HEAD $A -m "Initial Creation" && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success "update $m (logged by touch)" ' test_config core.logAllRefUpdates false && GIT_COMMITTER_DATE="2005-05-26 23:31" \ git update-ref HEAD $B $A -m "Switch" && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "set $m (logged by touch)" ' test_config core.logAllRefUpdates false && GIT_COMMITTER_DATE="2005-05-26 23:41" \ git update-ref HEAD $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success 'empty directory removal' '
@@ -319,19 +319,19 @@ test_expect_success "create $m (logged by config)" ' test_config core.logAllRefUpdates true && GIT_COMMITTER_DATE="2005-05-26 23:32" \ git update-ref HEAD $A -m "Initial Creation" && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' test_expect_success "update $m (logged by config)" ' test_config core.logAllRefUpdates true && GIT_COMMITTER_DATE="2005-05-26 23:33" \ git update-ref HEAD'" $B $A "'-m "Switch" && - test $B = $(cat .git/$m) + test $B = $(git show-ref -s --verify $m) ' test_expect_success "set $m (logged by config)" ' test_config core.logAllRefUpdates true && GIT_COMMITTER_DATE="2005-05-26 23:43" \ git update-ref HEAD $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m) ' cat >expect <<EOF
t/t1506-rev-parse-diagnosis.sh +1 -1
@@ -207,7 +207,7 @@ test_expect_success 'arg before dashdash must be a revision (ambiguous)' ' { # we do not want to use rev-parse here, because # we are testing it - cat .git/refs/heads/foobar && + git show-ref -s refs/heads/foobar && printf "%s\n" -- } >expect && git rev-parse foobar -- >actual &&
t/t6050-replace.sh +1 -1
@@ -135,7 +135,7 @@ test_expect_success 'tag replaced commit' ' test_expect_success '"git fsck" works' ' git fsck master >fsck_master.out && test_i18ngrep "dangling commit $R" fsck_master.out && - test_i18ngrep "dangling tag $(cat .git/refs/tags/mytag)" fsck_master.out && + test_i18ngrep "dangling tag $(git show-ref -s refs/tags/mytag)" fsck_master.out && test -z "$(git fsck)" '
t/t9020-remote-svn.sh +2 -2
@@ -48,8 +48,8 @@ test_expect_success REMOTE_SVN 'simple fetch' ' ' test_debug ' - cat .git/refs/svn/svnsim/master - cat .git/refs/remotes/svnsim/master + git show-ref -s refs/svn/svnsim/master + git show-ref -s refs/remotes/svnsim/master ' test_expect_success REMOTE_SVN 'repeated fetch, nothing shall change' '
Signature · 1 line
gitgitgadget
Re: [PATCH v13 02/13] t: use update-ref and show-ref to reading/writing refs
Quoted 32 lines · Han-Wen Nienhuys via GitGitGadget
From: Han-Wen Nienhuys <[email protected]>
Reading and writing .git/refs/* assumes that refs are stored in the 'files' ref backend.
Signed-off-by Han-Wen Nienhuys <[email protected]>--- t/t0002-gitfile.sh | 2 +- t/t1400-update-ref.sh | 32 ++++++++++++++++---------------- t/t1506-rev-parse-diagnosis.sh | 2 +- t/t6050-replace.sh | 2 +- t/t9020-remote-svn.sh | 4 ++-- 5 files changed, 21 insertions(+), 21 deletions(-)t/t0002-gitfile.sh +1
-1@@ -62,7 +62,7 @@ test_expect_success 'check commit-tree' ' ' test_expect_success 'check rev-list' ' - echo $SHA >"$REAL/HEAD" && + git update-ref "HEAD" "$SHA" && test "$SHA" = "$(git rev-list HEAD)" 't/t1400-update-ref.sh +0
-0
t/t1400-update-ref.sh test_expect_success setup ' · line 40
Quoted 5 lines
t/t1400-update-ref.sh +1
-1@@ -37,15 +37,15 @@ test_expect_success setup ' test_expect_success "create $m" ' git update-ref $m $A && - test $A = $(cat .git/$m) + test $A = $(git show-ref -s --verify $m)
All of the above looks good. These were written long time ago, and we should have cleaned them up much earlier, but we didn't.
t/t1506-rev-parse-diagnosis.sh test_expect_success 'arg before dashdash must be a revision (ambiguous)' ' · line 211
Quoted 11 lines
t/t1506-rev-parse-diagnosis.sh +1
-1@@ -207,7 +207,7 @@ test_expect_success 'arg before dashdash must be a revision (ambiguous)' ' { # we do not want to use rev-parse here, because # we are testing it - cat .git/refs/heads/foobar && + git show-ref -s refs/heads/foobar && printf "%s\n" --
Likewise.
t/t6050-replace.sh test_expect_success 'tag replaced commit' ' · line 138
Quoted 12 lines
t/t1506-rev-parse-diagnosis.sh +0
-0} >expect && git rev-parse foobar -- >actual &&t/t6050-replace.sh +1
-1@@ -135,7 +135,7 @@ test_expect_success 'tag replaced commit' ' test_expect_success '"git fsck" works' ' git fsck master >fsck_master.out && test_i18ngrep "dangling commit $R" fsck_master.out && - test_i18ngrep "dangling tag $(cat .git/refs/tags/mytag)" fsck_master.out && + test_i18ngrep "dangling tag $(git show-ref -s refs/tags/mytag)" fsck_master.out &&
Likewise; this has no excuse of being old code ;-)
Quoted 8 lines
t/t6050-replace.sh +0
-0test -z "$(git fsck)" 't/t9020-remote-svn.sh +0
-0
t/t9020-remote-svn.sh test_expect_success REMOTE_SVN 'simple fetch' ' · line 53
Quoted 8 lines
t/t9020-remote-svn.sh +2
-2@@ -48,8 +48,8 @@ test_expect_success REMOTE_SVN 'simple fetch' ' ' test_debug ' - cat .git/refs/svn/svnsim/master - cat .git/refs/remotes/svnsim/master + git show-ref -s refs/svn/svnsim/master + git show-ref -s refs/remotes/svnsim/master '
Looks good.
Thanks.
PATCH v13 3/13 refs: document how ref_iterator_advance_fn should handle symrefs
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
--- refs/refs-internal.h | 5 +++++ 1 file changed, 5 insertions(+)
refs/refs-internal.h +5 -0
@@ -438,6 +438,11 @@ void base_ref_iterator_free(struct ref_iterator *iter);/* Virtual function declarations for ref_iterators: */ +/* +* backend-specific implementation of ref_iterator_advance. +* For symrefs, the function should set REF_ISSYMREF, and it should also +* dereference the symref to provide the OID referent. +*/ typedef int ref_iterator_advance_fn (struct ref_iterator *ref_iterator); typedef int ref_iterator_peel_fn(struct ref_iterator *ref_iterator,
Signature · 1 line
gitgitgadget
Re: [PATCH v13 03/13] refs: document how ref_iterator_advance_fn should handle symrefs
Quoted 12 lines · Han-Wen Nienhuys via GitGitGadget
From: Han-Wen Nienhuys <[email protected]>
Signed-off-by Han-Wen Nienhuys <[email protected]>--- refs/refs-internal.h | 5 +++++ 1 file changed, 5 insertions(+)refs/refs-internal.h +0
-0
refs/refs-internal.h void base_ref_iterator_free(struct ref_iterator *iter); · line 446
Quoted 9 lines
refs/refs-internal.h +5
-0@@ -438,6 +438,11 @@ void base_ref_iterator_free(struct ref_iterator *iter);/* Virtual function declarations for ref_iterators: */ +/* +* backend-specific implementation of ref_iterator_advance. +* For symrefs, the function should set REF_ISSYMREF, and it should also +* dereference the symref to provide the OID referent. +*/ typedef int ref_iterator_advance_fn (struct ref_iterator *ref_iterator);
Shouldn't we also talk about the need for the backend to yield broken refs with ISBROKEN (iow, you are not allowed to skip---let the caller make the decision to skip) etc.?
Other than that, this is a good additional piece of information to tell the implementors of new backend.
Thanks.
Quoted 2 lines
refs/refs-internal.h +0
-0typedef int ref_iterator_peel_fn(struct ref_iterator *ref_iterator,
PATCH v13 4/13 reftable: file format documentation
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Jonathan Nieder <[email protected]>
Shawn Pearce explains:
Some repositories contain a lot of references (e.g. android at 866k, rails at 31k). The reftable format provides:
- Near constant time lookup for any single reference, even when the repository is cold and not in process or kernel cache.
- Near constant time verification a SHA-1 is referred to by at least one reference (for allow-tip-sha1-in-want).
- Efficient lookup of an entire namespace, such as refs/tags/.
- Support atomic push O(size_of_update) operations.
- Combine reflog storage with ref storage.
This file format spec was originally written in July, 2017 by Shawn Pearce. Some refinements since then were made by Shawn and by Han-Wen Nienhuys based on experiences implementing and experimenting with the format. (All of this was in the context of our work at Google and Google is happy to contribute the result to the Git project.)
Imported from JGit[1]'s current version (c217d33ff, "Documentation/technical/reftable: improve repo layout", 2020-02-04) of Documentation/technical/reftable.md and converted to asciidoc by running
pandoc -t asciidoc -f markdown reftable.md >reftable.txt
using pandoc 2.2.1. The result required the following additional minor changes:
- removed the [TOC] directive to add a table of contents, since asciidoc does not support it
- replaced git-scm.com/docs links with linkgit: directives that link to other pages within Git's documentation
[1] https://eclipse.googlesource.com/jgit/jgit
--- Documentation/Makefile | 1 + Documentation/technical/reftable.txt | 1067 ++++++++++++++++++++++++++ 2 files changed, 1068 insertions(+) create mode 100644 Documentation/technical/reftable.txt
Documentation/Makefile +1 -0
@@ -93,6 +93,7 @@ TECH_DOCS += technical/protocol-capabilities TECH_DOCS += technical/protocol-common TECH_DOCS += technical/protocol-v2 TECH_DOCS += technical/racy-git +TECH_DOCS += technical/reftable TECH_DOCS += technical/send-pack-pipeline TECH_DOCS += technical/shallow TECH_DOCS += technical/signature-format
Documentation/technical/reftable.txt new file +1067 -0
1,067 lines not on this page · raw message
Signature · 1 line
gitgitgadget
Re: [PATCH v13 04/13] reftable: file format documentation
Quoted 11 lines · Jonathan Nieder via GitGitGadget
From: Jonathan Nieder <[email protected]>
Shawn Pearce explains:
Some repositories contain a lot of references (e.g. android at 866k, rails at 31k). The reftable format provides:
- Near constant time lookup for any single reference, even when the repository is cold and not in process or kernel cache.
- Near constant time verification a SHA-1 is referred to by at least one reference (for allow-tip-sha1-in-want).
Not quite grammatical sentence? Perhaps "if" after "verification?
Quoted 17 lines
- Efficient lookup of an entire namespace, such as
refs/tags/.
- Support atomic pushO(size_of_update)operations.
- Combine reflog storage with ref storage.This file format spec was originally written in July, 2017 by Shawn Pearce. Some refinements since then were made by Shawn and by Han-Wen Nienhuys based on experiences implementing and experimenting with the format. (All of this was in the context of our work at Google and Google is happy to contribute the result to the Git project.)
Imported from JGit[1]'s current version (
c217d33ff, "Documentation/technical/reftable: improve repo layout", 2020-02-04) ofDocumentation/technical/reftable.mdand converted to asciidoc by runningpandoc -t asciidoc -f markdown reftable.md >reftable.txt
Quoted 7 lines
using pandoc 2.2.1. The result required the following additional minor changes:
- removed the [TOC] directive to add a table of contents, since asciidoc does not support it
- replaced git-scm.com/docs links with linkgit: directives that link to other pages within Git's documentation
There are many
’
funny-quotes where we would prefer to place vanilla single quotes, which may also need to be corrected in the conversion toolchain.
Typoes pointed out below may probably be from the original where they should be corrected.
Quoted 6 lines
Documentation/technical/reftable.txt new file +0
-0
Documentation/technical/reftable.txt line 10
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0@@ -0,0 +1,1067 @@ +reftable +-------- + +Overview +~~~~~~~~ + +Problem statement +^^^^^^^^^^^^^^^^^ + +Some repositories contain a lot of references (e.g. android at 866k,
Let's not use here after "e.g.". I see a normal space after "e.g." a few lines below.
Quoted 4 lines
Documentation/technical/reftable.txt new file +4
-0+rails at 31k). The existing packed-refs format takes up a lot of space +(e.g. 62M), and does not scale with additional references. Lookup of a +single reference requires linearly scanning the file. +Atomic pushes modifying multiple references require copying the entire
Documentation/technical/reftable.txt line 25
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0+packed-refs file, which can be a considerable amount of data moved +(e.g. 62M in, 62M out) for even small transactions (2 refs modified). + +Repositories with many loose references occupy a large number of disk +blocks from the local file system, as each reference is its own file +storing 41 bytes (and another file for the corresponding reflog). This +negatively affects the number of inodes available when a large number of +repositories are stored on the same filesystem. Readers can be penalized +due to the larger number of syscalls required to traverse and read the +`$GIT_DIR/refs` directory.
Another downside is that we cannot arrange atomic updates to multiple refs over loose refs, even though the "lookup of a single reference does not require linear scan" unlike packed-refs, (as long as the filesystem does its job). Worth mentioning?
Documentation/technical/reftable.txt line 34
Quoted 9 lines
Documentation/technical/reftable.txt new file +9
-0+ +Objectives +^^^^^^^^^^ + +* Near constant time lookup for any single reference, even when the +repository is cold and not in process or kernel cache. +* Near constant time verification if a SHA-1 is referred to by at least +one reference (for allow-tip-sha1-in-want). +* Efficient lookup of an entire namespace, such as `refs/tags/`.
Does this "lookup" refer to "do we have anything in refs/tags/ hierarchy?" or "enumerate all refs under refs/tags/ hierarchy?"
If the latter, perhaps s/lookup of/iteration over/
Documentation/technical/reftable.txt line 93
Quoted 13 lines
Documentation/technical/reftable.txt new file +12
-0+* Support atomic push with `O(size_of_update)` operations. +* Combine reflog storage with ref storage for small transactions. +* Separate reflog storage for base refs and historical logs. +Details +~~~~~~~ + +Peeling +^^^^^^^ + +References stored in a reftable are peeled, a record for an annotated +(or signed) tag records both the tag object, and the object it refers +to.
OK. Peeled results are recorded in packed-refs file because quite often when we use a tag object, what we actually want to access is the commit object it points at. We do so here for the same reason?
Not a rhetorical question, but if it invites a question from a reader, it may deserve to be described before readers ask it.
Documentation/technical/reftable.txt line 99
Quoted 5 lines
Documentation/technical/reftable.txt new file +5
-0+Reference name encoding +^^^^^^^^^^^^^^^^^^^^^^^ + +Reference names are an uninterpreted sequence of bytes that must pass +linkgit:git-check-ref-format[1] as a valid reference name.
OK. We want to be able to express any reference that we allow in the current backends.
Quoted 4 lines
Documentation/technical/reftable.txt new file +4
-0+Key unicity +^^^^^^^^^^^ + +Each entry must have a unique key; repeated keys are disallowed.
Documentation/technical/reftable.txt line 114
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0+ +Network byte order +^^^^^^^^^^^^^^^^^^ + +All multi-byte, fixed width fields are in network byte order. + +Ordering +^^^^^^^^ + +Blocks are lexicographically ordered by their first reference.
Key and Block are not explained until quite later, so these two among the above three are "Huh?" to readers at this point during their first read, but it probably cannot be helped. Let's read on.
Documentation/technical/reftable.txt line 125
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0+Directory/file conflicts +^^^^^^^^^^^^^^^^^^^^^^^^ + +The reftable format accepts both `refs/heads/foo` and +`refs/heads/foo/bar` as distinct references. + +This property is useful for retaining log records in reftable, but may +confuse versions of Git using `$GIT_DIR/refs` directory tree to maintain +references. Users of reftable may choose to continue to reject `foo` and +`foo/bar` type conflicts to prevent problems for peers.
Here "users" refer to things like "git-core", "jgit", "libgit2", etc.?
Let's say we have these two "conflicting" branches and want to interoperate with existing versions of Git (e.g. a "git ls-remote" client requests us to show what we have). We could either show "refs/heads/foo" with its object name, or "refs/heads/foo/bar" with its object name, but not both.
"users ... may choose" implies that it is up to the implementation of reftable user which one to show, so given a single repository, "jgit" may show "refs/heads/foo" while "libgit2" may choose to show the other one.
I am not sure if that is desirable---I suspect that we want to record which one needs to be chosen so that these "D/F conflicts disallowing" users can make consistent choices, but I dunno.
Quoted 4 lines
Documentation/technical/reftable.txt new file +4
-0+Block size +^^^^^^^^^^ + +The file’s block size is arbitrarily determined by the writer, and does
Documentation/technical/reftable.txt line 177
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0+not have to be a power of 2. The block size must be larger than the +longest reference name or log entry used in the repository, as +references cannot span blocks. + +Powers of two that are friendly to the virtual memory system or +filesystem (such as 4k or 8k) are recommended. Larger sizes (64k) can +yield better compression, with a possible increased cost incurred by +readers during access. + +The largest block size is `16777215` bytes (15.99 MiB).
The number being "(2**24) - 1" might be as significant as "15.99 MiB" to readers. As we recommend, and the users would find it natural, to use powers of two, the largest block size in practice would be 8 MiB?
Quoted 21 lines
Documentation/technical/reftable.txt new file +21
-0+Ref block format +^^^^^^^^^^^^^^^^ + +A ref block is written as: + +.... +'r' +uint24( block_len ) +ref_record+ +uint24( restart_offset )+ +uint16( restart_count ) + +padding? +.... + +Blocks begin with `block_type = 'r'` and a 3-byte `block_len` which +encodes the number of bytes in the block up to, but not including the +optional `padding`. This is always less than or equal to the file’s +block size. In the first ref block, `block_len` includes 24 bytes for +the file header. +
Documentation/technical/reftable.txt line 258
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0+The 2-byte `restart_count` stores the number of entries in the +`restart_offset` list, which must not be empty. Readers can use +`restart_count` to binary search between restarts before starting a +linear scan. + +Exactly `restart_count` 3-byte `restart_offset` values precedes the +`restart_count`. Offsets are relative to the start of the block and +refer to the first byte of any `ref_record` whose name has not been +prefix compressed. Entries in the `restart_offset` list must be sorted, +ascending. Readers can start linear scans from any of these records.
So the algorithm to find a record in a single block would be
- see how big the block_len is by reading the first four bytes;
- look at the last 16-bit word to see how many restart_offset entries there are;
- bisect using restart_offset array to see where to start a linear scan in the ref_record array
- linear scan the range between two adjacent offsets in restart_offset array.
(this is a mental exercise to make sure the information given here is sufficient, which I think it is).
Documentation/technical/reftable.txt line 266
Quoted 7 lines
Documentation/technical/reftable.txt new file +7
-0+A variable number of `ref_record` fill the middle of the block, +describing reference names and values. The format is described below. + +As the first ref block shares the first file block with the file header, +all `restart_offset` in the first block are relative to the start of the +file (position 0), and include the file header. This forces the first +`restart_offset` to be `28`.
OK.
Quoted 9 lines
Documentation/technical/reftable.txt new file +9
-0+ref record +++++++++++ + +A `ref_record` describes a single reference, storing both the name and +its value(s). Records are formatted as: + +.... +varint( prefix_length ) +varint( (suffix_length << 3) | value_type )
Documentation/technical/reftable.txt line 286
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0+suffix +varint( update_index_delta ) +value? +.... + +The `prefix_length` field specifies how many leading bytes of the prior +reference record’s name should be copied to obtain this reference’s +name. This must be 0 for the first reference in any block, and also must +be 0 for any `ref_record` whose offset is listed in the `restart_offset` +table at the end of the block.
OK. That's quite similar to how v4 index format shortens pathnames. We have encode_varint() helper (in varint.c), that is used from the index v4 code (in read-cache.c) and also untracked cache code (in dir.c).
The OFS_DELTA codepaths, both for encoding (in packfile.c) and for decoding (in builtin/pack-objects.c), uses the same algorithm but open codes it without using helper functions from varint.c (could this become leftoverbit? I dunno).
The document clarifies that the chosen variant of varint representation much later, but it may want to be moved up close to where "we use network byte order" etc. are declared.
Documentation/technical/reftable.txt line 295
Quoted 8 lines
Documentation/technical/reftable.txt new file +8
-0+Recovering a reference name from any `ref_record` is a simple concat: + +.... +this_name = prior_name[0..prefix_length] + suffix +.... + +The `suffix_length` value provides the number of bytes available in +`suffix` to copy from `suffix` to complete the reference name.
It is interesting that suffix is *not* a simple NUL terminated string. This DOES allow a NUL byte in a refname, but because we upfront declared that only the refnames allowed are the ones that pass check-ref-format, that would not be an advantage the use of varint to encode the suffix seeks. Then what is it?
I guess the answer is that a lot of time, the suffix length is shorter than 16 bytes (= 128/8), so we can store 3-bit value_type for free in such a case. Is that worth a mention?
Documentation/technical/reftable.txt line 299
Quoted 3 lines
Documentation/technical/reftable.txt new file +3
-0+The `update_index` that last modified the reference can be obtained by +adding `update_index_delta` to the `min_update_index` from the file +header: `min_update_index + update_index_delta`.
At this point, we can infer the following from we have learned so far by reading the document:
- there is a quantity called "update index" attached to each ref record;
- each file knows the range of update indices used in it; and
- when files are chained together, the range of update indices do not overlap---all indices in one file are strictly larger or smaller than those in another file.
But we haven't learned at all what "update index" is and how it is used, which makes it a frustrating read. We probably should give a mention of what it is here (a brief description at the same level of detail as say "monotonically increasing counter that is used as a transaction id when any ref update is made"), even though we will go into much deeper details in a later section.
Documentation/technical/reftable.txt line 307
Quoted 7 lines
Documentation/technical/reftable.txt new file +7
-0+The `value` follows. Its format is determined by `value_type`, one of +the following: + +* `0x0`: deletion; no value data (see transactions, below) +* `0x1`: one 20-byte object id; value of the ref +* `0x2`: two 20-byte object ids; value of the ref, peeled target +* `0x3`: symbolic reference: `varint( target_len ) target`
We probably should write these as "one (binary) object name", and "two (binary) object names", without hardwiring the number of bytes needed to represent an object name.
Documentation/technical/reftable.txt line 310
Quoted 2 lines
Documentation/technical/reftable.txt new file +2
-0+Symbolic references use `0x3`, followed by the complete name of the +reference target. No compression is applied to the target name.
Is there a place in the file format where an incomplete name can be stored? If not, I think it makes it easier to read if we drop "complete" from the sentence.
Quoted 7 lines
Documentation/technical/reftable.txt new file +7
-0+Types `0x4..0x7` are reserved for future use. + +Ref index +^^^^^^^^^ + +The ref index stores the name of the last reference from every ref block +in the file, enabling reduced disk seeks for lookups. Any reference can
Documentation/technical/reftable.txt line 328
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0+be found by searching the index, identifying the containing block, and +searching within that block. + +The index may be organized into a multi-level index, where the 1st level +index block points to additional ref index blocks (2nd level), which may +in turn point to either additional index blocks (e.g. 3rd level) or ref +blocks (leaf level). Disk reads required to access a ref go up with +higher index levels. Multi-level indexes may be required to ensure no +single index block exceeds the file format’s max block size of +`16777215` bytes (15.99 MiB). To acheive constant O(1) disk seeks for
achieve
Documentation/technical/reftable.txt line 407
Quoted 10 lines
Documentation/technical/reftable.txt new file +8
-0+lookups the index must be a single level, which is permitted to exceed +the file’s configured block size, but not the format’s max block size of +15.99 MiB. +Obj block format +^^^^^^^^^^^^^^^^ + +Object blocks are optional. Writers may choose to omit object blocks, +especially if readers will not use the SHA-1 to ref mapping.
"the object name to ref mapping".
Quoted 1 line
+Object blocks use unique, abbreviated 2-20 byte SHA-1 keys, mapping to
Likewise. "unique prefix of object names no less than 2 bytes" or somesuch to futureproof "2-20 byte SHA-1".
Documentation/technical/reftable.txt line 412
Quoted 3 lines
Documentation/technical/reftable.txt new file +3
-0+ref blocks containing references pointing to that object directly, or as +the peeled value of an annotated tag. Like ref blocks, object blocks use +the file’s standard block size. The abbrevation length is available in
abbreviation
Quoted 15 lines
Documentation/technical/reftable.txt new file +15
-0+the footer as `obj_id_len`. + +To save space in small files, object blocks may be omitted if the ref +index is not present, as brute force search will only need to read a few +ref blocks. When missing, readers should brute force a linear search of +all references to lookup by SHA-1. + +An object block is written as: + +.... +'o' +uint24( block_len ) +obj_record+ +uint24( restart_offset )+ +uint16( restart_count )
Documentation/technical/reftable.txt line 437
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0+ +padding? +.... + +Fields are identical to ref block. Binary search using the restart table +works the same as in reference blocks. + +Because object identifiers are abbreviated by writers to the shortest +unique abbreviation within the reftable, obj key lengths are variable +between 2 and 20 bytes. Readers must compare only for common prefix
Futureproof "2 and 20" similarly.
Documentation/technical/reftable.txt line 517
Quoted 11 lines
Documentation/technical/reftable.txt new file +10
-0+match within an obj block or obj index. +Log block format +^^^^^^^^^^^^^^^^ + +Unlike ref and obj blocks, log blocks are always unaligned. + +Log blocks are variable in size, and do not match the `block_size` +specified in the file header or footer. Writers should choose an +appropriate buffer size to prepare a log block for deflation, such as +`2 * block_size`.
I can guess the reason behind this design decision, but the readers may not be able to. Should we write it down here, or would it make too much irrelevant details?
Quoted 37 lines
Documentation/technical/reftable.txt new file +37
-0+A log block is written as: + +.... +'g' +uint24( block_len ) +zlib_deflate { + log_record+ + uint24( restart_offset )+ + uint16( restart_count ) +} +.... + +Log blocks look similar to ref blocks, except `block_type = 'g'`. + +The 4-byte block header is followed by the deflated block contents using +zlib deflate. The `block_len` in the header is the inflated size +(including 4-byte block header), and should be used by readers to +preallocate the inflation output buffer. A log block’s `block_len` may +exceed the file’s block size. + +Offsets within the log block (e.g. `restart_offset`) still include the +4-byte header. Readers may prefer prefixing the inflation output buffer +with the 4-byte header. + +Within the deflate container, a variable number of `log_record` describe +reference changes. The log record format is described below. See ref +block format (above) for a description of `restart_offset` and +`restart_count`. + +Because log blocks have no alignment or padding between blocks, readers +must keep track of the bytes consumed by the inflater to know where the +next log block begins. + +log record +++++++++++ + +Log record keys are structured as:
Documentation/technical/reftable.txt line 565
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0+ +.... +ref_name '\0' reverse_int64( update_index ) +.... + +where `update_index` is the unique transaction identifier. The +`update_index` field must be unique within the scope of a `ref_name`. +See the update transactions section below for further details. + +The `reverse_int64` function inverses the value so lexographical
lexicographical
Documentation/technical/reftable.txt line 573
Quoted 8 lines
Documentation/technical/reftable.txt new file +8
-0+ordering the network byte order encoding sorts the more recent records +with higher `update_index` values first: + +.... +reverse_int64(int64 t) { + return 0xffffffffffffffff - t; +} +....
Rationale? It may be to ease the iteration over reflog i.e. "log -g" that wants to go from the youngest to older---isn't it worth mentioning?
Quoted 23 lines
Documentation/technical/reftable.txt new file +23
-0+Log records have a similar starting structure to ref and index records, +utilizing the same prefix compression scheme applied to the log record +key described above. + +.... + varint( prefix_length ) + varint( (suffix_length << 3) | log_type ) + suffix + log_data { + old_id + new_id + varint( name_length ) name + varint( email_length ) email + varint( time_seconds ) + sint16( tz_offset ) + varint( message_length ) message + }? +.... + +Log record entries use `log_type` to indicate what follows: + +* `0x0`: deletion; no log data. +* `0x1`: standard git reflog data using `log_data` above.
Documentation/technical/reftable.txt line 607
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0+ +The `log_type = 0x0` is mostly useful for `git stash drop`, removing an +entry from the reflog of `refs/stash` in a transaction file (below), +without needing to rewrite larger files. Readers reading a stack of +reflogs must treat this as a deletion. + +For `log_type = 0x1`, the `log_data` section follows +linkgit:git-update-ref[1] logging and includes: + +* two 20-byte SHA-1s (old id, new id)
"two (binary) object names (old name, new name)" for futureproof out of SHA-1 world.
Documentation/technical/reftable.txt line 611
Quoted 4 lines
Documentation/technical/reftable.txt new file +4
-0+* varint string of committer’s name +* varint string of committer’s email +* varint time in seconds since epoch (Jan 1, 1970) +* 2-byte timezone offset in minutes (signed)
We use minus eight hundred for "GMT-0800" internally, but this would use -480, which makes more sense ;-)
Documentation/technical/reftable.txt line 619
Quoted 8 lines
Documentation/technical/reftable.txt new file +8
-0+* varint string of message + +`tz_offset` is the absolute number of minutes from GMT the committer was +at the time of the update. For example `GMT-0800` is encoded in reftable +as `sint16(-480)` and `GMT+0230` is `sint16(150)`. + +The committer email does not contain `<` or `>`, it’s the value normally +found between the `<>` in a git commit object header.
Saving two precious bytes?
This is a tangent but in a repository at hosting provider, whose primary (and often the only) source of updates are by end-user pushing into it, if reflogs are enabled, whose name and email are recorded in the logs? The committer or tagger of the object that sits at the tip of the ref after the update? What happens when a blob is pushed to update a ref? Or would it be just a single "user" that represents the "server operator"?
We know in a non-bare repository an individual contributor works on typically records only one <name, email> in the reflog: the user who works in it.
What I am trying to get at is if it makes more sense to have a small table of unique <name, email> pairs used in the file and have log_data record a single varint that is the index into that "committer ident" table. I would suspect that it would give us significantly more gain than mere <> two bytes per log_data entry.
Documentation/technical/reftable.txt line 625
Quoted 5 lines
Documentation/technical/reftable.txt new file +5
-0+The `message_length` may be 0, in which case there was no message +supplied for the update. + +Contrary to traditional reflog (which is a file), renames are encoded as +a combination of ref deletion and ref creation.
Yay? How does the deletion record look like? The new object name being 0*hashlength? I didn't see it defined in the description (and I am guessing that log_type of 0x0 is *NOT* used for that purpose).
So, NEEDSWORK: describe how "creation of a ref" and "deletion of a ref" appears in a log as a log record entry.
Quoted 21 lines
Documentation/technical/reftable.txt new file +21
-0+Footer +^^^^^^ + +After the last block of the file, a file footer is written. It begins +like the file header, but is extended with additional data. + +A 68-byte footer appears at the end: + +.... + 'REFT' + uint8( version_number = 1 ) + uint24( block_size ) + uint64( min_update_index ) + uint64( max_update_index ) + + uint64( ref_index_position ) + uint64( (obj_position << 5) | obj_id_len ) + uint64( obj_index_position ) + + uint64( log_position ) + uint64( log_index_position )
Documentation/technical/reftable.txt line 698
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0+ + uint32( CRC-32 of above ) +.... + +If a section is missing (e.g. ref index) the corresponding position +field (e.g. `ref_index_position`) will be 0. + +* `obj_position`: byte position for the first obj block. +* `obj_id_len`: number of bytes used to abbreviate object identifiers in +obj blocks.
Should we write "this can be up to 31" somewhere? It is more than enough for SHA-1 and not quite sufficient for SHA-256 (unless we say "we store obj_id_len-1 here")?
Quoted 11 lines
Documentation/technical/reftable.txt new file +9
-0+* `log_position`: byte position for the first log block. +* `ref_index_position`: byte position for the start of the ref index. +* `obj_index_position`: byte position for the start of the obj index. +* `log_index_position`: byte position for the start of the log index. +Varint encoding +^^^^^^^^^^^^^^^ + +Varint encoding is identical to the ofs-delta encoding method used +within pack files.
Documentation/technical/reftable.txt line 732
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0+ +Decoder works such as: + +.... +val = buf[ptr] & 0x7f +while (buf[ptr] & 0x80) { + ptr++ + val = ((val + 1) << 7) | (buf[ptr] & 0x7f) +} +....
As already said, I think this should be given upfront next to where we declare that we use network byte order.
Documentation/technical/reftable.txt line 763
Quoted 13 lines
Documentation/technical/reftable.txt new file +13
-0+Restart point selection ++++++++++++++++++++++++ + +Writers determine the restart points at file creation. The process is +arbitrary, but every 16 or 64 records is recommended. Every 16 may be +more suitable for smaller block sizes (4k or 8k), every 64 for larger +block sizes (64k). + +More frequent restart points reduces prefix compression and increases +space consumed by the restart table, both of which increase file size. + +Less frequent restart points makes prefix compression more effective, +decreasing overall file size, with increased penalities for readers
penalties
Quoted 5 lines
Documentation/technical/reftable.txt new file +3
-0+walking through more records after the binary search step. + +A maximum of `65535` restart points per block is supported.
Documentation/technical/reftable.txt line 1052
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0+LMDB +^^^^ + +David Turner proposed +https://public-inbox.org/git/[email protected]/[using +LMDB], as LMDB is lightweight (64k of runtime code) and GPL-compatible +license. + +A downside of LMDB is its reliance on a single C implementation. This +makes embedding inside JGit (a popular reimplemenation of Git)
reimplementation
Quoted 15 lines
Documentation/technical/reftable.txt new file +15
-0+difficult, and hoisting onto virtual storage (for JGit DFS) virtually +impossible. + +A common format that can be supported by all major Git implementations +(git-core, JGit, libgit2) is strongly preferred. + +Future +~~~~~~ + +Longer hashes +^^^^^^^^^^^^^ + +Version will bump (e.g. 2) to indicate `value` uses a different object +id length other than 20. The length could be stored in an expanded file +header, or hardcoded as part of the version.
Re: [PATCH v13 04/13] reftable: file format documentation
Quoted 9 lines · Junio C Hamano
Quoted 7 lines
Some repositories contain a lot of references (e.g. android at 866k, rails at 31k). The reftable format provides:
- Near constant time lookup for any single reference, even when the repository is cold and not in process or kernel cache.
- Near constant time verification a SHA-1 is referred to by at least one reference (for allow-tip-sha1-in-want).Not quite grammatical sentence? Perhaps "if" after "verification?
Done.
Quoted 20 lines
Quoted 20 lines
- Efficient lookup of an entire namespace, such as
refs/tags/.
- Support atomic pushO(size_of_update)operations.
- Combine reflog storage with ref storage.This file format spec was originally written in July, 2017 by Shawn Pearce. Some refinements since then were made by Shawn and by Han-Wen Nienhuys based on experiences implementing and experimenting with the format. (All of this was in the context of our work at Google and Google is happy to contribute the result to the Git project.)
Imported from JGit[1]'s current version (
c217d33ff, "Documentation/technical/reftable: improve repo layout", 2020-02-04) ofDocumentation/technical/reftable.mdand converted to asciidoc by runningpandoc -t asciidoc -f markdown reftable.md >reftable.txt
using pandoc 2.2.1. The result required the following additional minor changes:
Quoted 8 lines
Quoted 4 lines
- removed the [TOC] directive to add a table of contents, since asciidoc does not support it
- replaced git-scm.com/docs links with linkgit: directives that link to other pages within Git's documentationThere are many
’
fixed.
Quoted 2 lines
Typoes pointed out below may probably be from the original where they should be corrected.
fixed.
Quoted 2 lines
Let's not use here after "e.g.". I see a normal space after "e.g." a few lines below.
fixed.
Quoted 7 lines
Documentation/technical/reftable.txt line 17
Quoted 7 lines
Documentation/technical/reftable.txt new file +7
-0+rails at 31k). The existing packed-refs format takes up a lot of space +(e.g. 62M), and does not scale with additional references. Lookup of a +single reference requires linearly scanning the file. + +Atomic pushes modifying multiple references require copying the entire +packed-refs file, which can be a considerable amount of data moved +(e.g. 62M in, 62M out) for even small transactions (2 refs modified).
(*)
Quoted 4 lines
Quoted 4 lines
Documentation/technical/reftable.txt new file +4
-0+Repositories with many loose references occupy a large number of disk +blocks from the local file system, as each reference is its own file +storing 41 bytes (and another file for the corresponding reflog). This +negatively affects the number of inodes available when a large number of
Quoted 8 lines
Documentation/technical/reftable.txt line 25
Quoted 3 lines
Documentation/technical/reftable.txt new file +3
-0+repositories are stored on the same filesystem. Readers can be penalized +due to the larger number of syscalls required to traverse and read the +`$GIT_DIR/refs` directory.Another downside is that we cannot arrange atomic updates to multiple refs over loose refs, even though the "lookup of a single reference does not require linear scan" unlike packed-refs, (as long as the filesystem does its job). Worth mentioning?
that's already there at (*), no?
Quoted 10 lines
Documentation/technical/reftable.txt line 34
Quoted 5 lines
Documentation/technical/reftable.txt new file +5
-0+* Near constant time lookup for any single reference, even when the +repository is cold and not in process or kernel cache. +* Near constant time verification if a SHA-1 is referred to by at least +one reference (for allow-tip-sha1-in-want). +* Efficient lookup of an entire namespace, such as `refs/tags/`.Does this "lookup" refer to "do we have anything in refs/tags/ hierarchy?" or "enumerate all refs under refs/tags/ hierarchy?"
If the latter, perhaps s/lookup of/iteration over/
changed to "efficient enumeration".
Quoted 10 lines
Documentation/technical/reftable.txt line 93
Quoted 6 lines
Documentation/technical/reftable.txt new file +6
-0+Peeling +^^^^^^^ + +References stored in a reftable are peeled, a record for an annotated +(or signed) tag records both the tag object, and the object it refers +to.OK. Peeled results are recorded in packed-refs file because quite often when we use a tag object, what we actually want to access is the commit object it points at. We do so here for the same reason?
I think so. Added a reference to packed-refs
Quoted 19 lines
Documentation/technical/reftable.txt line 125
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0+Directory/file conflicts +^^^^^^^^^^^^^^^^^^^^^^^^ + +The reftable format accepts both `refs/heads/foo` and +`refs/heads/foo/bar` as distinct references. + +This property is useful for retaining log records in reftable, but may +confuse versions of Git using `$GIT_DIR/refs` directory tree to maintain +references. Users of reftable may choose to continue to reject `foo` and +`foo/bar` type conflicts to prevent problems for peers.Here "users" refer to things like "git-core", "jgit", "libgit2", etc.?
Let's say we have these two "conflicting" branches and want to interoperate with existing versions of Git (e.g. a "git ls-remote" client requests us to show what we have). We could either show "refs/heads/foo" with its object name, or "refs/heads/foo/bar" with its object name, but not both.
Quoted 8 lines
"users ... may choose" implies that it is up to the implementation of reftable user which one to show, so given a single repository, "jgit" may show "refs/heads/foo" while "libgit2" may choose to show the other one.
I am not sure if that is desirable---I suspect that we want to record which one needs to be chosen so that these "D/F conflicts disallowing" users can make consistent choices, but I dunno.
I think it's beyond the scope of the document. In reftable the library, I added an option to validate for file/dir conflicts.
(Google's deployment doesn't do the validation completely correctly. We had a ref ending in '/' for example.)
Quoted 12 lines
Quoted 12 lines
Documentation/technical/reftable.txt new file +11
-0+Block size +^^^^^^^^^^ + +The file’s block size is arbitrarily determined by the writer, and does +not have to be a power of 2. The block size must be larger than the +longest reference name or log entry used in the repository, as +references cannot span blocks. + +Powers of two that are friendly to the virtual memory system or +filesystem (such as 4k or 8k) are recommended. Larger sizes (64k) can +yield better compression, with a possible increased cost incurred by
Quoted 8 lines
Documentation/technical/reftable.txt line 177
Quoted 3 lines
Documentation/technical/reftable.txt new file +3
-0+readers during access. + +The largest block size is `16777215` bytes (15.99 MiB).The number being "(2**24) - 1" might be as significant as "15.99 MiB" to readers. As we recommend, and the users would find it natural, to use powers of two, the largest block size in practice would be 8 MiB?
Beyond 4096 bytes (size of a page on x86), I don't think there is a need for powers of two. What makes for a good block size would also be decided by the characteristics of the underlying storage system. At google, we use 64k, IIRC.
Quoted 36 lines
Quoted 21 lines
Documentation/technical/reftable.txt new file +21
-0+Ref block format +^^^^^^^^^^^^^^^^ + +A ref block is written as: + +.... +'r' +uint24( block_len ) +ref_record+ +uint24( restart_offset )+ +uint16( restart_count ) + +padding? +.... + +Blocks begin with `block_type = 'r'` and a 3-byte `block_len` which +encodes the number of bytes in the block up to, but not including the +optional `padding`. This is always less than or equal to the file’s +block size. In the first ref block, `block_len` includes 24 bytes for +the file header. +Documentation/technical/reftable.txt line 258
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0+The 2-byte `restart_count` stores the number of entries in the +`restart_offset` list, which must not be empty. Readers can use +`restart_count` to binary search between restarts before starting a +linear scan. + +Exactly `restart_count` 3-byte `restart_offset` values precedes the +`restart_count`. Offsets are relative to the start of the block and +refer to the first byte of any `ref_record` whose name has not been +prefix compressed. Entries in the `restart_offset` list must be sorted, +ascending. Readers can start linear scans from any of these records.So the algorithm to find a record in a single block would be
- see how big the
block_lenis by reading the first four bytes;
Quoted 8 lines
- look at the last 16-bit word to see how many
restart_offsetentries there are;- bisect using
restart_offsetarray to see where to start a linear scan in theref_recordarray- linear scan the range between two adjacent offsets in
restart_offsetarray.
yes, that works. In practice, you find the last restart before the key, and then iterate until you find a key that is >= the wanted key. There is no real need to find the next restart.
Quoted 3 lines
The document clarifies that the chosen variant of varint representation much later, but it may want to be moved up close to where "we use network byte order" etc. are declared.
moved.
Quoted 6 lines
Quoted 6 lines
Documentation/technical/reftable.txt new file +6
-0+Recovering a reference name from any `ref_record` is a simple concat: + +.... +this_name = prior_name[0..prefix_length] + suffix +.... +
Quoted 8 lines
Quoted 2 lines
+The
suffix_lengthvalue provides the number of bytes available in +suffixto copy fromsuffixto complete the reference name.It is interesting that suffix is *not* a simple NUL terminated string. This DOES allow a NUL byte in a refname, but because we upfront declared that only the refnames allowed are the ones that pass check-ref-format, that would not be an advantage the use of varint to encode the suffix seeks. Then what is it?
reflog records encode update_index in the key, which may lead to keys with zero bytes. So you can't rely on zero termination.
Quoted 3 lines
I guess the answer is that a lot of time, the suffix length is shorter than 16 bytes (= 128/8), so we can store 3-bit
value_typefor free in such a case. Is that worth a mention?
Here, and in other places, I would like to avoid annotating rationales in detail. First of all, this was Shawn's spec; if I go and rewrite whole sections, the authorship becomes muddled. Second, a lot of the "why" will invite questions how to do it differently to achieve the same objectives (see below, where you muse about '<' and '>'), and that's a past station given that it's in JGit. Finally, all these edits cause extra work with conflict resolution in follow-up changes in the series. If we want to editorialize more, I'd rather do it in a follow up series.
Quoted 13 lines
Documentation/technical/reftable.txt line 299
Quoted 3 lines
Documentation/technical/reftable.txt new file +3
-0+The `update_index` that last modified the reference can be obtained by +adding `update_index_delta` to the `min_update_index` from the file +header: `min_update_index + update_index_delta`.At this point, we can infer the following from we have learned so far by reading the document:
- there is a quantity called "update index" attached to each ref record;
- each file knows the range of update indices used in it; and
Quoted 6 lines
But we haven't learned at all what "update index" is and how it is used, which makes it a frustrating read. We probably should give a mention of what it is here (a brief description at the same level of detail as say "monotonically increasing counter that is used as a transaction id when any ref update is made"), even though we will go into much deeper details in a later section.
I think update_index is poorly named. I think it's supposed to be the index into an ordered array of updates.
A better name would be 'logical timestamp'. It is a timestamp, because we use it to determine which update came first, but it is 'logical' because it can't depend on actual clocks, because of synchronization problems. Inside Google, I think we use microsecond Spanner timestamps of the database transaction on the global ref database.
In a local setting, it's good enough to use a sequential number, which is determined by incrementing the last update_index.
Quoted 3 lines
- when files are chained together, the range of update indices do not overlap---all indices in one file are strictly larger or smaller than those in another file.
That's what I thought initially too , but it is a dangerous assumption. See reftable/merged.c, merged_iter_next_entry() for a comment.
At Google, in the datacenter deployment, update indices of different reftable can overlap. This can happen when compaction and replication delay interact.
Quoted 3 lines
We probably should write these as "one (binary) object name", and "two (binary) object names", without hardwiring the number of bytes needed to represent an object name.
This is addressed in the follow-up change for SHA256. Let's leave this alone.
Quoted 6 lines
Documentation/technical/reftable.txt line 310
Quoted 2 lines
Documentation/technical/reftable.txt new file +2
-0+Symbolic references use `0x3`, followed by the complete name of the +reference target. No compression is applied to the target name.Is there a place in the file format where an incomplete name can be stored? If not, I think it makes it easier to read if we drop "complete" from the sentence.
All of the keys prefix compressed, so most refnames are incomplete.
Quoted 74 lines
Quoted 7 lines
Documentation/technical/reftable.txt new file +7
-0+Types `0x4..0x7` are reserved for future use. + +Ref index +^^^^^^^^^ + +The ref index stores the name of the last reference from every ref block +in the file, enabling reduced disk seeks for lookups. Any reference canDocumentation/technical/reftable.txt line 328
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0+be found by searching the index, identifying the containing block, and +searching within that block. + +The index may be organized into a multi-level index, where the 1st level +index block points to additional ref index blocks (2nd level), which may +in turn point to either additional index blocks (e.g. 3rd level) or ref +blocks (leaf level). Disk reads required to access a ref go up with +higher index levels. Multi-level indexes may be required to ensure no +single index block exceeds the file format’s max block size of +`16777215` bytes (15.99 MiB). To acheive constant O(1) disk seeks forachieve
Documentation/technical/reftable.txt line 407
Quoted 10 lines
Documentation/technical/reftable.txt new file +8
-0+lookups the index must be a single level, which is permitted to exceed +the file’s configured block size, but not the format’s max block size of +15.99 MiB. +Obj block format +^^^^^^^^^^^^^^^^ + +Object blocks are optional. Writers may choose to omit object blocks, +especially if readers will not use the SHA-1 to ref mapping."the object name to ref mapping".
Quoted 1 line
+Object blocks use unique, abbreviated 2-20 byte SHA-1 keys, mapping to
Likewise. "unique prefix of object names no less than 2 bytes" or somesuch to futureproof "2-20 byte SHA-1".
Documentation/technical/reftable.txt line 412
Quoted 3 lines
Documentation/technical/reftable.txt new file +3
-0+ref blocks containing references pointing to that object directly, or as +the peeled value of an annotated tag. Like ref blocks, object blocks use +the file’s standard block size. The abbrevation length is available inabbreviation
Quoted 15 lines
Documentation/technical/reftable.txt new file +15
-0+the footer as `obj_id_len`. + +To save space in small files, object blocks may be omitted if the ref +index is not present, as brute force search will only need to read a few +ref blocks. When missing, readers should brute force a linear search of +all references to lookup by SHA-1. + +An object block is written as: + +.... +'o' +uint24( block_len ) +obj_record+ +uint24( restart_offset )+ +uint16( restart_count )Documentation/technical/reftable.txt line 437
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0+ +padding? +.... + +Fields are identical to ref block. Binary search using the restart table +works the same as in reference blocks. + +Because object identifiers are abbreviated by writers to the shortest +unique abbreviation within the reftable, obj key lengths are variable +between 2 and 20 bytes. Readers must compare only for common prefixFutureproof "2 and 20" similarly.
Quoted 1 line
+match within an obj block or obj index.
Quoted 13 lines
Documentation/technical/reftable.txt line 517
Quoted 9 lines
Documentation/technical/reftable.txt new file +9
-0+Log block format +^^^^^^^^^^^^^^^^ + +Unlike ref and obj blocks, log blocks are always unaligned. + +Log blocks are variable in size, and do not match the `block_size` +specified in the file header or footer. Writers should choose an +appropriate buffer size to prepare a log block for deflation, such as +`2 * block_size`.I can guess the reason behind this design decision, but the readers may not be able to. Should we write it down here, or would it make too much irrelevant details?
I honestly don't know about the reason. reflogs will compress well (because there is lots of ascii text), but it makes the format needlessly complex IMO. But it's there, so we can't do much about it now.
Quoted 108 lines
Quoted 37 lines
Documentation/technical/reftable.txt new file +37
-0+A log block is written as: + +.... +'g' +uint24( block_len ) +zlib_deflate { + log_record+ + uint24( restart_offset )+ + uint16( restart_count ) +} +.... + +Log blocks look similar to ref blocks, except `block_type = 'g'`. + +The 4-byte block header is followed by the deflated block contents using +zlib deflate. The `block_len` in the header is the inflated size +(including 4-byte block header), and should be used by readers to +preallocate the inflation output buffer. A log block’s `block_len` may +exceed the file’s block size. + +Offsets within the log block (e.g. `restart_offset`) still include the +4-byte header. Readers may prefer prefixing the inflation output buffer +with the 4-byte header. + +Within the deflate container, a variable number of `log_record` describe +reference changes. The log record format is described below. See ref +block format (above) for a description of `restart_offset` and +`restart_count`. + +Because log blocks have no alignment or padding between blocks, readers +must keep track of the bytes consumed by the inflater to know where the +next log block begins. + +log record +++++++++++ + +Log record keys are structured as:Documentation/technical/reftable.txt line 565
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0+ +.... +ref_name '\0' reverse_int64( update_index ) +.... + +where `update_index` is the unique transaction identifier. The +`update_index` field must be unique within the scope of a `ref_name`. +See the update transactions section below for further details. + +The `reverse_int64` function inverses the value so lexographicallexicographical
Documentation/technical/reftable.txt line 573
Quoted 8 lines
Documentation/technical/reftable.txt new file +8
-0+ordering the network byte order encoding sorts the more recent records +with higher `update_index` values first: + +.... +reverse_int64(int64 t) { + return 0xffffffffffffffff - t; +} +....Rationale? It may be to ease the iteration over reflog i.e. "log -g" that wants to go from the youngest to older---isn't it worth mentioning?
Quoted 23 lines
Documentation/technical/reftable.txt new file +23
-0+Log records have a similar starting structure to ref and index records, +utilizing the same prefix compression scheme applied to the log record +key described above. + +.... + varint( prefix_length ) + varint( (suffix_length << 3) | log_type ) + suffix + log_data { + old_id + new_id + varint( name_length ) name + varint( email_length ) email + varint( time_seconds ) + sint16( tz_offset ) + varint( message_length ) message + }? +.... + +Log record entries use `log_type` to indicate what follows: + +* `0x0`: deletion; no log data. +* `0x1`: standard git reflog data using `log_data` above.Documentation/technical/reftable.txt line 607
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0+ +The `log_type = 0x0` is mostly useful for `git stash drop`, removing an +entry from the reflog of `refs/stash` in a transaction file (below), +without needing to rewrite larger files. Readers reading a stack of +reflogs must treat this as a deletion. + +For `log_type = 0x1`, the `log_data` section follows +linkgit:git-update-ref[1] logging and includes: + +* two 20-byte SHA-1s (old id, new id)"two (binary) object names (old name, new name)" for futureproof out of SHA-1 world.
Documentation/technical/reftable.txt line 611
Quoted 4 lines
Documentation/technical/reftable.txt new file +4
-0+* varint string of committer’s name +* varint string of committer’s email +* varint time in seconds since epoch (Jan 1, 1970) +* 2-byte timezone offset in minutes (signed)We use minus eight hundred for "GMT-0800" internally, but this would use -480, which makes more sense ;-)
Quoted 10 lines
Documentation/technical/reftable.txt line 619
Quoted 8 lines
Documentation/technical/reftable.txt new file +8
-0+* varint string of message + +`tz_offset` is the absolute number of minutes from GMT the committer was +at the time of the update. For example `GMT-0800` is encoded in reftable +as `sint16(-480)` and `GMT+0230` is `sint16(150)`. + +The committer email does not contain `<` or `>`, it’s the value normally +found between the `<>` in a git commit object header.Saving two precious bytes?
If you store reflogs in ascii files, both for convention, and to separate the name from the email. Internally, it's much more convenient to not have the '<' '>'. For any practical application of the email address (Say, you want to send out an email to the user), you'll need to strip off the < >
Quoted 7 lines
This is a tangent but in a repository at hosting provider, whose primary (and often the only) source of updates are by end-user pushing into it, if reflogs are enabled, whose name and email are recorded in the logs? The committer or tagger of the object that sits at the tip of the ref after the update? What happens when a blob is pushed to update a ref? Or would it be just a single "user" that represents the "server operator"?
That's up to the caller to decide. I think JGit uses the committer identity, but I'd have to check.
Quoted 9 lines
We know in a non-bare repository an individual contributor works on typically records only one <name, email> in the reflog: the user who works in it.
What I am trying to get at is if it makes more sense to have a small table of unique <name, email> pairs used in the file and have
log_datarecord a single varint that is the index into that "committer ident" table. I would suspect that it would give us significantly more gain than mere <> two bytes perlog_dataentry.
gain in what dimension? Space-wise, the zlib compression will remove all slack anyway.
Quoted 9 lines
Documentation/technical/reftable.txt line 625
Quoted 5 lines
Documentation/technical/reftable.txt new file +5
-0+The `message_length` may be 0, in which case there was no message +supplied for the update. + +Contrary to traditional reflog (which is a file), renames are encoded as +a combination of ref deletion and ref creation.Yay? How does the deletion record look like? The new object name being 0*hashlength? I didn't see it defined in the description (and I am guessing that
log_typeof0x0is *NOT* used for that purpose).
quoting the spec:
"Log record entries use log_type to indicate what follows:
* 0x0: deletion; no log data."
Quoted 2 lines
So, NEEDSWORK: describe how "creation of a ref" and "deletion of a ref" appears in a log as a log record entry.
the creation would be the appearance of a reflog record for the ref. You'd have to search back in time to decide if a reflog record it was an update to an existing record or a creation.
Quoted 4 lines
Quoted 1 line
+....
As already said, I think this should be given upfront next to where we declare that we use network byte order.
I moved this
--
Han-Wen Nienhuys - Google Munich
I work 80%. Don't expect answers from me on Fridays. --
Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg
Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Re: [PATCH v13 04/13] reftable: file format documentation
Quoted 14 lines · Han-Wen Nienhuys
Quoted 9 lines
Documentation/technical/reftable.txt line 625
Quoted 5 lines
Documentation/technical/reftable.txt new file +5
-0+The `message_length` may be 0, in which case there was no message +supplied for the update. + +Contrary to traditional reflog (which is a file), renames are encoded as +a combination of ref deletion and ref creation.Yay? How does the deletion record look like? The new object name being 0*hashlength? I didn't see it defined in the description (and I am guessing that
log_typeof0x0is *NOT* used for that purpose).quoting the spec:
"Log record entries use
log_typeto indicate what follows:
Quoted 8 lines
*
0x0: deletion; no log data."Quoted 2 lines
So, NEEDSWORK: describe how "creation of a ref" and "deletion of a ref" appears in a log as a log record entry.
the creation would be the appearance of a reflog record for the ref. You'd have to search back in time to decide if a reflog record it was an update to an existing record or a creation.
Correction. This is one of the things that confused me earlier: reflog entries for creating and deleting branches look like this
000000000000 -> xxxxx (create)
xxxxxx.. xx -> 00000 .. (delete)respectively. When the rename happens, we can signal that the deletion and addition are linked, because they occur at the same update_index.
The deletion records for logs (type=0x0) remove a reflog entry at a specific (earlier used) update_index. So you could have the following situation:
0x0001.ref : reflog "refs/heads/master" @ update_index=0x0001, new=xxx, old=yyy ...
and then a subsequent table could specify
0x0020.ref : reflog "refs/heads/master" @ update_index=0x0001 (type=0x0)
which would hide the earlier reflog entry.
Jonathan Nieder said that this is used for git-stash, but I have never understood why this is necessary, and would love to clarify this better.
I'll clarify the explanation to reflect this.
Signature · 7 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Re: [PATCH v13 04/13] reftable: file format documentation
Quoted 3 lines · Han-Wen Nienhuys
Jonathan Nieder said that this is used for git-stash, but I have never understood why this is necessary, and would love to clarify this better.
The doc says this:
"The log_type = 0x0 is mostly useful for git stash drop, removing an entry from the reflog of refs/stash in a transaction file (below), without needing to rewrite larger files. Readers reading a stack of reflogs must treat this as a deletion."
I should probably look at the code for git-stash to see how this plays out.
Signature · 7 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Re: [PATCH v13 04/13] reftable: file format documentation
Quoted 11 lines · Han-Wen Nienhuys
Quoted 3 lines · Han-Wen Nienhuys
Jonathan Nieder said that this is used for git-stash, but I have never understood why this is necessary, and would love to clarify this better.
The doc says this:
"The
log_type = 0x0is mostly useful forgit stash drop, removing an entry from the reflog ofrefs/stashin a transaction file (below), without needing to rewrite larger files. Readers reading a stack of reflogs must treat this as a deletion."
Yup, I saw that and that is where my "I am guessing that log_type of 0x0 is *NOT* used for that purpose)." in my review comment came from.
The "git stash" itself is an abuse of the reflog mechanism, but its "drop" subcommand is probably the worst offender. It wants to remove any arbitrary reflog entry in the middle of a reflog, not just topmost ones (pop) or bottommost ones (expire).
Quoted 1 line
I should probably look at the code for git-stash to see how this plays out.
Re: [PATCH v13 04/13] reftable: file format documentation
In reply to Junio C HamanoQuoted 9 lines · Junio C Hamano
Quoted 7 lines · Han-wen
From: Jonathan Nieder <[email protected]>
Shawn Pearce explains:
Some repositories contain a lot of references (e.g. android at 866k, rails at 31k). The reftable format provides:
Quoted 6 lines
Quoted 4 lines
- Near constant time lookup for any single reference, even when the repository is cold and not in process or kernel cache.
- Near constant time verification a SHA-1 is referred to by at least one reference (for allow-tip-sha1-in-want).Not quite grammatical sentence? Perhaps "if" after "verification?
Good catch, thanks.
[...]
Quoted 8 lines
Quoted 7 lines
using pandoc 2.2.1. The result required the following additional minor changes:
- removed the [TOC] directive to add a table of contents, since asciidoc does not support it
- replaced git-scm.com/docs links with linkgit: directives that link to other pages within Git's documentation
Quoted 6 lines
There are many
’
funny-quotes where we would prefer to place vanilla single quotes, which may also need to be corrected in the conversion toolchain.
Looks like Han-Wen is taking care of this (thanks!).
Quoted 2 lines
Typoes pointed out below may probably be from the original where they should be corrected.
I'm happy to do one final update the doc in JGit to match what we end up with and then replace it with a pointer to Git's copy once that lands.
[...]
Quoted 4 lines
Quoted 4 lines
Documentation/technical/reftable.txt new file +4
-0+Repositories with many loose references occupy a large number of disk +blocks from the local file system, as each reference is its own file +storing 41 bytes (and another file for the corresponding reflog). This +negatively affects the number of inodes available when a large number of
Quoted 8 lines
Documentation/technical/reftable.txt line 25
Quoted 3 lines
Documentation/technical/reftable.txt new file +3
-0+repositories are stored on the same filesystem. Readers can be penalized +due to the larger number of syscalls required to traverse and read the +`$GIT_DIR/refs` directory.Another downside is that we cannot arrange atomic updates to multiple refs over loose refs, even though the "lookup of a single reference does not require linear scan" unlike packed-refs, (as long as the filesystem does its job). Worth mentioning?
Yes, this was another major part of the motivation (avoiding the complication of the "atomic" multi-ref updates to packed-refs that Git and JGit had to learn).
[...]
Quoted 10 lines
Documentation/technical/reftable.txt line 93
Quoted 3 lines
Documentation/technical/reftable.txt new file +3
-0+References stored in a reftable are peeled, a record for an annotated +(or signed) tag records both the tag object, and the object it refers +to.OK. Peeled results are recorded in packed-refs file because quite often when we use a tag object, what we actually want to access is the commit object it points at. We do so here for the same reason?
Not a rhetorical question, but if it invites a question from a reader, it may deserve to be described before readers ask it.
For a single tag ref, peeling to a commit is not very expensive. But for batch lookups e.g. when serving a response to an ls-remote request, it adds up, and having the peeled results recorded helps.
[...]
Quoted 10 lines
Quoted 10 lines
Documentation/technical/reftable.txt new file +10
-0+Directory/file conflicts +^^^^^^^^^^^^^^^^^^^^^^^^ + +The reftable format accepts both `refs/heads/foo` and +`refs/heads/foo/bar` as distinct references. + +This property is useful for retaining log records in reftable, but may +confuse versions of Git using `$GIT_DIR/refs` directory tree to maintain +references. Users of reftable may choose to continue to reject `foo` and +`foo/bar` type conflicts to prevent problems for peers.
[...]
Quoted 8 lines
"users ... may choose" implies that it is up to the implementation of reftable user which one to show, so given a single repository, "jgit" may show "refs/heads/foo" while "libgit2" may choose to show the other one.
I am not sure if that is desirable---I suspect that we want to record which one needs to be chosen so that these "D/F conflicts disallowing" users can make consistent choices, but I dunno.
Yes, I think it would be better to explicitly say that Git will continue to reject D/F conflicts for refs (*not* reflogs) even though the format can support them in principle.
If we choose to permit them some day in the future, I believe that would be a separate repository format extension and protocol capability to avoid confusing old versions of Git.
[...]
Quoted 6 lines
Documentation/technical/reftable.txt line 310
Quoted 2 lines
Documentation/technical/reftable.txt new file +2
-0+Symbolic references use `0x3`, followed by the complete name of the +reference target. No compression is applied to the target name.Is there a place in the file format where an incomplete name can be stored? If not, I think it makes it easier to read if we drop "complete" from the sentence.
The sentence about "no compression" covers the lack of prefix encoding, so I suppose I agree.
Might make sense to say "full name" to convey that we're talking about rev-parse --symbolic-full-name, not a relative path like symlinks support.
[...]
Quoted 5 lines
Quoted 5 lines
Documentation/technical/reftable.txt new file +5
-0+Log block format +^^^^^^^^^^^^^^^^ + +Unlike ref and obj blocks, log blocks are always unaligned. +
Quoted 8 lines
Documentation/technical/reftable.txt line 517
Quoted 4 lines
Documentation/technical/reftable.txt new file +4
-0+Log blocks are variable in size, and do not match the `block_size` +specified in the file header or footer. Writers should choose an +appropriate buffer size to prepare a log block for deflation, such as +`2 * block_size`.I can guess the reason behind this design decision, but the readers may not be able to. Should we write it down here, or would it make too much irrelevant details?
I don't have a strong opinion. It sounds like Han-Wen sees something to explain there, so I suppose it would be nice to spell out.
(My take: reflog lookups are not on the critical path for most operations; especially, random accesses do not need to be fast. From a performance perspective, the best we can do is to compress them well to decrease I/O cost, hence there's not much value to alignment.)
[...]
Quoted 7 lines
This is a tangent but in a repository at hosting provider, whose primary (and often the only) source of updates are by end-user pushing into it, if reflogs are enabled, whose name and email are recorded in the logs? The committer or tagger of the object that sits at the tip of the ref after the update? What happens when a blob is pushed to update a ref? Or would it be just a single "user" that represents the "server operator"?
The latter, "server operator" (GIT_COMMITTER_IDENT at the server).
Committer in commit objects is forgeable, hence wouldn't be very useful here.
Quoted 9 lines
We know in a non-bare repository an individual contributor works on typically records only one <name, email> in the reflog: the user who works in it.
What I am trying to get at is if it makes more sense to have a small table of unique <name, email> pairs used in the file and have
log_datarecord a single varint that is the index into that "committer ident" table. I would suspect that it would give us significantly more gain than mere <> two bytes perlog_dataentry.
That's true, and a good idea for the next rev of the format.
[...]
Quoted 11 lines
Quoted 11 lines
Documentation/technical/reftable.txt new file +11
-0+A 68-byte footer appears at the end: + +.... + 'REFT' + uint8( version_number = 1 ) + uint24( block_size ) + uint64( min_update_index ) + uint64( max_update_index ) + + uint64( ref_index_position ) + uint64( (obj_position << 5) | obj_id_len )
[...]
Quoted 6 lines
Documentation/technical/reftable.txt line 698
Quoted 2 lines
Documentation/technical/reftable.txt new file +2
-0+* `obj_id_len`: number of bytes used to abbreviate object identifiers in +obj blocks.Should we write "this can be up to 31" somewhere? It is more than enough for SHA-1 and not quite sufficient for SHA-256 (unless we say "we store obj_id_len-1 here")?
Oh! I'll take a closer look and then follow up.
Thanks for looking it over,
Jonathan
PATCH v13 5/13 reftable: clarify how empty tables should be written
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The format allows for some ambiguity, as a lone footer also starts with a valid file header. However, the current JGit code will barf on this. This commit codifies this behavior into the standard.
--- Documentation/technical/reftable.txt | 7 +++++++ 1 file changed, 7 insertions(+)
Documentation/technical/reftable.txt +7 -0
@@ -715,6 +715,13 @@ version) Once verified, the other fields of the footer can be accessed. +Empty tables +++++++++++++ + +A reftable may be empty. In this case, the file starts with a header +and is immediately followed by a footer. + + Varint encoding ^^^^^^^^^^^^^^^
Signature · 1 line
gitgitgadget
Re: [PATCH v13 05/13] reftable: clarify how empty tables should be written
Quoted 5 lines · Han-Wen Nienhuys via GitGitGadget
From: Han-Wen Nienhuys <[email protected]>
The format allows for some ambiguity, as a lone footer also starts with a valid file header. However, the current JGit code will barf on this. This commit codifies this behavior into the standard.
Nice to see the documentation being careful.
Quoted 23 lines
Signed-off-by Han-Wen Nienhuys <[email protected]>--- Documentation/technical/reftable.txt | 7 +++++++ 1 file changed, 7 insertions(+)Documentation/technical/reftable.txt +7
-0@@ -715,6 +715,13 @@ version) Once verified, the other fields of the footer can be accessed. +Empty tables +++++++++++++ + +A reftable may be empty. In this case, the file starts with a header +and is immediately followed by a footer. + + Varint encoding ^^^^^^^^^^^^^^^
PATCH v13 6/13 reftable: define version 2 of the spec to accomodate SHA256
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Version appends a hash ID to the file header, making it slightly larger.
This commit also changes "SHA-1" into "object ID" in many places.
--- Documentation/technical/reftable.txt | 79 ++++++++++++++++------------ 1 file changed, 44 insertions(+), 35 deletions(-)
Documentation/technical/reftable.txt +44 -35
@@ -29,7 +29,7 @@ Objectives * Near constant time lookup for any single reference, even when the repository is cold and not in process or kernel cache. -* Near constant time verification if a SHA-1 is referred to by at least +* Near constant time verification if an object ID is referred to by at least one reference (for allow-tip-sha1-in-want). * Efficient lookup of an entire namespace, such as `refs/tags/`. * Support atomic push with `O(size_of_update)` operations.
@@ -193,8 +193,8 @@ and non-aligned files. Very small files (e.g. a single ref block) may omit `padding` and the ref index to reduce total file size. -Header -^^^^^^ +Header (version 1) +^^^^^^^^^^^^^^^^^^ A 24-byte header appears at the beginning of the file:
@@ -215,6 +215,27 @@ used in a stack for link:#Update-transactions[transactions], these fields can order the files such that the prior file’s `max_update_index + 1` is the next file’s `min_update_index`. +Header (version 2) +^^^^^^^^^^^^^^^^^^ + +A 28-byte header appears at the beginning of the file: + +.... +'REFT' +uint8( version_number = 2 ) +uint24( block_size ) +uint64( min_update_index ) +uint64( max_update_index ) +uint32( hash_id ) +.... + +The header is identical to `version_number=1`, with the 4-byte hash ID +("sha1" for SHA1 and "s256" for SHA-256) append to the header. + +For maximum backward compatibility, it is recommended to use version 1 when +writing SHA1 reftables. + + First ref block ^^^^^^^^^^^^^^^
@@ -302,8 +323,8 @@ The `value` follows. Its format is determined by `value_type`, one of the following: * `0x0`: deletion; no value data (see transactions, below) -* `0x1`: one 20-byte object id; value of the ref -* `0x2`: two 20-byte object ids; value of the ref, peeled target +* `0x1`: one object id; value of the ref +* `0x2`: two object ids; value of the ref, peeled target * `0x3`: symbolic reference: `varint( target_len ) target` Symbolic references use `0x3`, followed by the complete name of the
@@ -404,9 +425,9 @@ Obj block format ^^^^^^^^^^^^^^^^ Object blocks are optional. Writers may choose to omit object blocks, -especially if readers will not use the SHA-1 to ref mapping. +especially if readers will not use the object ID to ref mapping. -Object blocks use unique, abbreviated 2-20 byte SHA-1 keys, mapping to +Object blocks use unique, abbreviated 2-32 byte object ID keys, mapping to ref blocks containing references pointing to that object directly, or as the peeled value of an annotated tag. Like ref blocks, object blocks use the file’s standard block size. The abbrevation length is available in
@@ -415,7 +436,7 @@ the footer as `obj_id_len`. To save space in small files, object blocks may be omitted if the ref index is not present, as brute force search will only need to read a few ref blocks. When missing, readers should brute force a linear search of -all references to lookup by SHA-1. +all references to lookup by object ID. An object block is written as:
@@ -434,7 +455,7 @@ works the same as in reference blocks. Because object identifiers are abbreviated by writers to the shortest unique abbreviation within the reftable, obj key lengths are variable -between 2 and 20 bytes. Readers must compare only for common prefix +between 2 and 32 bytes. Readers must compare only for common prefix match within an obj block or obj index. obj record
@@ -487,9 +508,9 @@ for (j = 1; j < position_count; j++) { .... With a position in hand, a reader must linearly scan the ref block, -starting from the first `ref_record`, testing each reference’s SHA-1s +starting from the first `ref_record`, testing each reference’s object IDs (for `value_type = 0x1` or `0x2`) for full equality. Faster searching by -SHA-1 within a single ref block is not supported by the reftable format. +object ID within a single ref block is not supported by the reftable format. Smaller block sizes reduce the number of candidates this step must consider.
@@ -604,7 +625,7 @@ reflogs must treat this as a deletion. For `log_type = 0x1`, the `log_data` section follows linkgit:git-update-ref[1] logging and includes: -* two 20-byte SHA-1s (old id, new id) +* two object IDs (old id, new id) * varint string of committer’s name * varint string of committer’s email * varint time in seconds since epoch (Jan 1, 1970)
@@ -671,14 +692,8 @@ Footer After the last block of the file, a file footer is written. It begins like the file header, but is extended with additional data. -A 68-byte footer appears at the end: - .... - 'REFT' - uint8( version_number = 1 ) - uint24( block_size ) - uint64( min_update_index ) - uint64( max_update_index ) + HEADER uint64( ref_index_position ) uint64( (obj_position << 5) | obj_id_len )
@@ -701,12 +716,16 @@ obj blocks. * `obj_index_position`: byte position for the start of the obj index. * `log_index_position`: byte position for the start of the log index. +The size of the footer is 68 bytes for version 1, and 72 bytes for +version 2. + Reading the footer ++++++++++++++++++ -Readers must seek to `file_length - 68` to access the footer. A trusted -external source (such as `stat(2)`) is necessary to obtain -`file_length`. When reading the footer, readers must verify: +Readers must first read the file start to determine the version +number. Then they seek to `file_length - FOOTER_LENGTH` to access the +footer. A trusted external source (such as `stat(2)`) is necessary to +obtain `file_length`. When reading the footer, readers must verify: * 4-byte magic is correct * 1-byte version number is recognized
@@ -779,11 +798,11 @@ Lightweight refs dominate ^^^^^^^^^^^^^^^^^^^^^^^^^ The reftable format assumes the vast majority of references are single -SHA-1 valued with common prefixes, such as Gerrit Code Review’s +object IDs valued with common prefixes, such as Gerrit Code Review’s `refs/changes/` namespace, GitHub’s `refs/pulls/` namespace, or many lightweight tags in the `refs/tags/` namespace. -Annotated tags storing the peeled object cost an additional 20 bytes per +Annotated tags storing the peeled object cost an additional object ID per reference. Low overhead
@@ -816,7 +835,7 @@ Scans and lookups dominate Scanning all references and lookup by name (or namespace such as `refs/heads/`) are the most common activities performed on repositories. -SHA-1s are stored directly with references to optimize this use case. +Object IDs are stored directly with references to optimize this use case. Logs are infrequently read ^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -1062,13 +1081,3 @@ impossible. A common format that can be supported by all major Git implementations (git-core, JGit, libgit2) is strongly preferred. - -Future -~~~~~~ - -Longer hashes -^^^^^^^^^^^^^ - -Version will bump (e.g. 2) to indicate `value` uses a different object -id length other than 20. The length could be stored in an expanded file -header, or hardcoded as part of the version.
Signature · 1 line
gitgitgadget
Re: [PATCH v13 06/13] reftable: define version 2 of the spec to accomodate SHA256
Quoted 16 lines · Han-Wen Nienhuys via GitGitGadget
From: Han-Wen Nienhuys <[email protected]>
Version appends a hash ID to the file header, making it slightly larger.
This commit also changes "SHA-1" into "object ID" in many places.
Signed-off-by Han-Wen Nienhuys <[email protected]>--- Documentation/technical/reftable.txt | 79 ++++++++++++++++------------ 1 file changed, 44 insertions(+), 35 deletions(-)Documentation/technical/reftable.txt +0
-0
Documentation/technical/reftable.txt Objectives · line 33
Quoted 6 lines
Documentation/technical/reftable.txt +1
-1@@ -29,7 +29,7 @@ Objectives * Near constant time lookup for any single reference, even when the repository is cold and not in process or kernel cache. -* Near constant time verification if a SHA-1 is referred to by at least +* Near constant time verification if an object ID is referred to by at least one reference (for allow-tip-sha1-in-want).
Good. These are called "object names", though.
Quoted 40 lines
Documentation/technical/reftable.txt +23
-2@@ -193,8 +193,8 @@ and non-aligned files. Very small files (e.g. a single ref block) may omit `padding` and the ref index to reduce total file size. -Header -^^^^^^ +Header (version 1) +^^^^^^^^^^^^^^^^^^ A 24-byte header appears at the beginning of the file:@@ -215,6 +215,27 @@ used in a stack for link:#Update-transactions[transactions], these fields can order the files such that the prior file’s `max_update_index + 1` is the next file’s `min_update_index`. +Header (version 2) +^^^^^^^^^^^^^^^^^^ + +A 28-byte header appears at the beginning of the file: + +.... +'REFT' +uint8( version_number = 2 ) +uint24( block_size ) +uint64( min_update_index ) +uint64( max_update_index ) +uint32( hash_id ) +.... + +The header is identical to `version_number=1`, with the 4-byte hash ID +("sha1" for SHA1 and "s256" for SHA-256) append to the header. + +For maximum backward compatibility, it is recommended to use version 1 when +writing SHA1 reftables. + + First ref block ^^^^^^^^^^^^^^^
Documentation/technical/reftable.txt The `value` follows. Its format is determined by `value_type`, one of · line 327
Quoted 7 lines
Documentation/technical/reftable.txt +2
-2@@ -302,8 +323,8 @@ The `value` follows. Its format is determined by `value_type`, one of the following: * `0x0`: deletion; no value data (see transactions, below) -* `0x1`: one 20-byte object id; value of the ref -* `0x2`: two 20-byte object ids; value of the ref, peeled target +* `0x1`: one object id; value of the ref +* `0x2`: two object ids; value of the ref, peeled target
Ah, OK, I pointed out these future-proofing for the previous step, but as long as the end result is written in a hash-algorithm agnostic way, it is OK. Again these are called "object names", though.
Documentation/technical/reftable.txt works the same as in reference blocks. · line 458
Quoted 8 lines
Documentation/technical/reftable.txt +1
-1* `0x3`: symbolic reference: `varint( target_len ) target`@@ -434,7 +455,7 @@ works the same as in reference blocks. Because object identifiers are abbreviated by writers to the shortest unique abbreviation within the reftable, obj key lengths are variable -between 2 and 20 bytes. Readers must compare only for common prefix +between 2 and 32 bytes. Readers must compare only for common prefix
Is it allowed for a reftable file whose hash_id field says "sha1" to use more than 20 bytes of obj key? Phrasing it like "unique prefix of object name, no shorter than 2 bytes" would avoid the problem, I would think.
This version also adds more
’
apostrophes, where we would prefer to place vanilla single quotes, which may need to be corrected in the conversion toolchain.
I did not see any new typo introduced in this step.
Thanks.
Re: [PATCH v13 06/13] reftable: define version 2 of the spec to accomodate SHA256
Quoted 6 lines · Junio C Hamano
Documentation/technical/reftable.txt Objectives · line 33
Quoted 3 lines
Documentation/technical/reftable.txt +1
-1-* Near constant time verification if a SHA-1 is referred to by at least +* Near constant time verification if an object ID is referred to by at least one reference (for allow-tip-sha1-in-want).Good. These are called "object names", though.
Fixed throughout.
All of the source code (libgit2, Jgit, git itself) calls this object ID (git_oid, ObjectId, struct object_id respectively), so it's a bit surprising.
Quoted 4 lines
Quoted 4 lines
*
0x3: symbolic reference:varint( target_len ) target(quoted diff) +0
-0@@ -434,7 +455,7 @@ works the same as in reference blocks.
Quoted 9 lines
Documentation/technical/reftable.txt works the same as in reference blocks. · line 458
Quoted 4 lines
Documentation/technical/reftable.txt +1
-1Because object identifiers are abbreviated by writers to the shortest unique abbreviation within the reftable, obj key lengths are variable -between 2 and 20 bytes. Readers must compare only for common prefix +between 2 and 32 bytes. Readers must compare only for common prefixIs it allowed for a reftable file whose
hash_idfield says "sha1" to use more than 20 bytes of obj key? Phrasing it like "unique prefix of object name, no shorter than 2 bytes" would avoid the problem, I would think.
rephrased.
Quoted 4 lines
This version also adds more
’
curious. I have no idea where they came from. Fixed now.
Signature · 11 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Re: [PATCH v13 06/13] reftable: define version 2 of the spec to accomodate SHA256
Quoted 6 lines · Han-Wen Nienhuys
Quoted 4 lines
This version also adds more
’
curious. I have no idea where they came from. Fixed now.
The version 1 format in 04/13 has the same; it may already be in the original document, or the conversion process via pandoc transformed a plain single quote intelligently?
PATCH v13 7/13 Write pseudorefs through ref backends.
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Pseudorefs store transient data in in the repository. Examples are HEAD, CHERRY_PICK_HEAD, etc.
These refs have always been read through the ref backends, but they were written in a one-off routine that wrote a object ID or symref directly wrote into .git/<pseudo_ref_name>.
This causes problems when introducing a new ref storage backend. To remedy this, extend the ref backend implementation with a write_pseudoref_fn and update_pseudoref_fn.
--- refs.c | 115 +++++------------------------ refs.h | 11 +++ refs/files-backend.c | 164 +++++++++++++++++++++++++++++++++++------- refs/packed-backend.c | 40 ++++------- refs/refs-internal.h | 12 ++++ 5 files changed, 192 insertions(+), 150 deletions(-)
refs.c +17 -98
@@ -739,101 +739,6 @@ long get_files_ref_lock_timeout_ms(void)return timeout_ms; } -static int write_pseudoref (const char *pseudoref,const struct object_id *oid, -const struct object_id *old_oid,struct strbuf *err) -{ -const char *filename; -int fd; -struct lock_file lock =LOCK_INIT ; -struct strbuf buf =STRBUF_INIT ; -int ret =-1 ; - -if (!oid) -return 0 ; - -strbuf_addf (&buf,"%s\n" ,oid_to_hex (oid)); - - filename =git_path ("%s" , pseudoref); - fd =hold_lock_file_for_update_timeout (&lock, filename,0 , -get_files_ref_lock_timeout_ms ()); -if (fd <0 ) { -strbuf_addf (err,_ ("could not open '%s' for writing: %s" ), - filename,strerror (errno)); - goto done; - } - -if (old_oid) { -struct object_id actual_old_oid; - -if (read_ref (pseudoref, &actual_old_oid)) { -if (!is_null_oid (old_oid)) { -strbuf_addf (err,_ ("could not read ref '%s'" ), - pseudoref); -rollback_lock_file (&lock); - goto done; - } - }else if (is_null_oid (old_oid)) { -strbuf_addf (err,_ ("ref '%s' already exists" ), - pseudoref); -rollback_lock_file (&lock); - goto done; - }else if (!oideq (&actual_old_oid, old_oid)) { -strbuf_addf (err,_ ("unexpected object ID when writing '%s'" ), - pseudoref); -rollback_lock_file (&lock); - goto done; - } - } - -if (write_in_full (fd, buf.buf, buf.len) <0 ) { -strbuf_addf (err,_ ("could not write to '%s'" ), filename); -rollback_lock_file (&lock); - goto done; - } - -commit_lock_file (&lock); - ret =0 ; -done: -strbuf_release (&buf); -return ret; -} - -static int delete_pseudoref (const char *pseudoref,const struct object_id *old_oid) -{ -const char *filename; - - filename =git_path ("%s" , pseudoref); - -if (old_oid && !is_null_oid (old_oid)) { -struct lock_file lock =LOCK_INIT ; -int fd; -struct object_id actual_old_oid; - - fd =hold_lock_file_for_update_timeout ( - &lock, filename,0 , -get_files_ref_lock_timeout_ms ()); -if (fd <0 ) { -error_errno (_ ("could not open '%s' for writing" ), - filename); -return -1 ; - } -if (read_ref (pseudoref, &actual_old_oid)) -die (_ ("could not read ref '%s'" ), pseudoref); -if (!oideq (&actual_old_oid, old_oid)) { -error (_ ("unexpected object ID when deleting '%s'" ), - pseudoref); -rollback_lock_file (&lock); -return -1 ; - } - -unlink (filename); -rollback_lock_file (&lock); - }else { -unlink (filename); - } - -return 0 ; -} int refs_delete_ref(struct ref_store *refs, const char *msg, const char *refname,
@@ -844,8 +749,7 @@ int refs_delete_ref(struct ref_store *refs, const char *msg,struct strbuf err =STRBUF_INIT ;if (ref_type (refname) ==REF_TYPE_PSEUDOREF ) { -assert (refs ==get_main_ref_store (the_repository)); -return delete_pseudoref (refname, old_oid); +return ref_store_delete_pseudoref (refs, refname, old_oid); } transaction =ref_store_transaction_begin (refs, &err);
@@ -1172,7 +1076,8 @@ int refs_update_ref(struct ref_store *refs, const char *msg,if (ref_type (refname) ==REF_TYPE_PSEUDOREF ) {assert (refs ==get_main_ref_store (the_repository)); - ret =write_pseudoref (refname, new_oid, old_oid, &err); + ret =ref_store_write_pseudoref (refs, refname, new_oid, old_oid, + &err); }else { t =ref_store_transaction_begin (refs, &err); if (!t ||
@@ -1441,6 +1346,20 @@ int head_ref(each_ref_fn fn, void *cb_data)return refs_head_ref (get_main_ref_store (the_repository), fn, cb_data); } +int ref_store_write_pseudoref (struct ref_store *refs,const char *pseudoref, +const struct object_id *oid, +const struct object_id *old_oid, +struct strbuf *err) +{ +return refs->be->write_pseudoref (refs, pseudoref, oid, old_oid, err); +} + +int ref_store_delete_pseudoref (struct ref_store *refs,const char *pseudoref, +const struct object_id *old_oid) +{ +return refs->be->delete_pseudoref (refs, pseudoref, old_oid); +} +struct ref_iterator *refs_ref_iterator_begin (struct ref_store *refs,const char *prefix,int trim,int flags)
refs.h +11 -0
@@ -728,6 +728,17 @@ int update_ref(const char *msg, const char *refname, const struct object_id *new_oid, const struct object_id *old_oid, unsigned int flags, enum action_on_err onerr); +/* Pseudorefs (eg. HEAD, CHERRY_PICK_HEAD) have a separate routines for updating +and deletion as they cannot take part in normal transactional updates. +Pseudorefs should only be written for the main repository. +*/ +int ref_store_write_pseudoref (struct ref_store *refs,const char *pseudoref, +const struct object_id *oid, +const struct object_id *old_oid, +struct strbuf *err); +int ref_store_delete_pseudoref (struct ref_store *refs,const char *pseudoref, +const struct object_id *old_oid); +int parse_hide_refs_config (const char *var,const char *value,const char *);/*
refs/files-backend.c +137 -27
173 lines not on this page · raw message
refs/packed-backend.c +15 -25
@@ -1641,29 +1641,19 @@ static int packed_reflog_expire(struct ref_store *ref_store, }struct ref_storage_be refs_be_packed = { -NULL , -"packed" , - packed_ref_store_create, - packed_init_db, - packed_transaction_prepare, - packed_transaction_finish, - packed_transaction_abort, - packed_initial_transaction_commit, - - packed_pack_refs, - packed_create_symref, - packed_delete_refs, - packed_rename_ref, - packed_copy_ref, - - packed_ref_iterator_begin, - packed_read_raw_ref, - - packed_reflog_iterator_begin, - packed_for_each_reflog_ent, - packed_for_each_reflog_ent_reverse, - packed_reflog_exists, - packed_create_reflog, - packed_delete_reflog, - packed_reflog_expire +NULL ,"packed" , packed_ref_store_create, packed_init_db, + packed_transaction_prepare, packed_transaction_finish, + packed_transaction_abort, packed_initial_transaction_commit, + + packed_pack_refs, packed_create_symref, packed_delete_refs, + packed_rename_ref, packed_copy_ref, + +/* XXX */ +NULL ,NULL , + + packed_ref_iterator_begin, packed_read_raw_ref, + + packed_reflog_iterator_begin, packed_for_each_reflog_ent, + packed_for_each_reflog_ent_reverse, packed_reflog_exists, + packed_create_reflog, packed_delete_reflog, packed_reflog_expire };
refs/refs-internal.h +12 -0
@@ -549,6 +549,15 @@ typedef int copy_ref_fn(struct ref_store *ref_store, const char *oldref, const char *newref, const char *logmsg); +typedef int write_pseudoref_fn (struct ref_store *ref_store, +const char *pseudoref, +const struct object_id *oid, +const struct object_id *old_oid, +struct strbuf *err); +typedef int delete_pseudoref_fn (struct ref_store *ref_store, +const char *pseudoref, +const struct object_id *old_oid); +/* * Iterate over the references in `ref_store` whose names start with * `prefix`. `prefix` is matched as a literal string, without regard
@@ -648,6 +657,9 @@ struct ref_storage_be {rename_ref_fn *rename_ref;copy_ref_fn *copy_ref; +write_pseudoref_fn *write_pseudoref; +delete_pseudoref_fn *delete_pseudoref; +ref_iterator_begin_fn *iterator_begin;read_raw_ref_fn *read_raw_ref;
Signature · 1 line
gitgitgadget
Re: [PATCH v13 07/13] Write pseudorefs through ref backends.
Hi Han-Wen
Thanks for working on the reftable implementation. I've had a look at this patch but bear in mind I'm not familiar with the refs code
Quoted 129 lines · Han-Wen Nienhuys via GitGitGadget
From: Han-Wen Nienhuys <[email protected]>
Pseudorefs store transient data in in the repository. Examples are HEAD,
CHERRY_PICK_HEAD, etc.These refs have always been read through the ref backends, but they were written in a one-off routine that wrote a object ID or symref directly wrote into .git/<pseudo_ref_name>.
This causes problems when introducing a new ref storage backend. To remedy this, extend the ref backend implementation with a
write_pseudoref_fnandupdate_pseudoref_fn.Signed-off-by Han-Wen Nienhuys <[email protected]>--- refs.c | 115 +++++------------------------ refs.h | 11 +++ refs/files-backend.c | 164 +++++++++++++++++++++++++++++++++++------- refs/packed-backend.c | 40 ++++------- refs/refs-internal.h | 12 ++++ 5 files changed, 192 insertions(+), 150 deletions(-)refs.c +0
-95@@ -739,101 +739,6 @@ long get_files_ref_lock_timeout_ms(void)return timeout_ms; } -static int write_pseudoref (const char *pseudoref,const struct object_id *oid, -const struct object_id *old_oid,struct strbuf *err) -{ -const char *filename; -int fd; -struct lock_file lock =LOCK_INIT ; -struct strbuf buf =STRBUF_INIT ; -int ret =-1 ; - -if (!oid) -return 0 ; - -strbuf_addf (&buf,"%s\n" ,oid_to_hex (oid)); - - filename =git_path ("%s" , pseudoref); - fd =hold_lock_file_for_update_timeout (&lock, filename,0 , -get_files_ref_lock_timeout_ms ()); -if (fd <0 ) { -strbuf_addf (err,_ ("could not open '%s' for writing: %s" ), - filename,strerror (errno)); - goto done; - } - -if (old_oid) { -struct object_id actual_old_oid; - -if (read_ref (pseudoref, &actual_old_oid)) { -if (!is_null_oid (old_oid)) { -strbuf_addf (err,_ ("could not read ref '%s'" ), - pseudoref); -rollback_lock_file (&lock); - goto done; - } - }else if (is_null_oid (old_oid)) { -strbuf_addf (err,_ ("ref '%s' already exists" ), - pseudoref); -rollback_lock_file (&lock); - goto done; - }else if (!oideq (&actual_old_oid, old_oid)) { -strbuf_addf (err,_ ("unexpected object ID when writing '%s'" ), - pseudoref); -rollback_lock_file (&lock); - goto done; - } - } - -if (write_in_full (fd, buf.buf, buf.len) <0 ) { -strbuf_addf (err,_ ("could not write to '%s'" ), filename); -rollback_lock_file (&lock); - goto done; - } - -commit_lock_file (&lock); - ret =0 ; -done: -strbuf_release (&buf); -return ret; -} - -static int delete_pseudoref (const char *pseudoref,const struct object_id *old_oid) -{ -const char *filename; - - filename =git_path ("%s" , pseudoref); - -if (old_oid && !is_null_oid (old_oid)) { -struct lock_file lock =LOCK_INIT ; -int fd; -struct object_id actual_old_oid; - - fd =hold_lock_file_for_update_timeout ( - &lock, filename,0 , -get_files_ref_lock_timeout_ms ()); -if (fd <0 ) { -error_errno (_ ("could not open '%s' for writing" ), - filename); -return -1 ; - } -if (read_ref (pseudoref, &actual_old_oid)) -die (_ ("could not read ref '%s'" ), pseudoref); -if (!oideq (&actual_old_oid, old_oid)) { -error (_ ("unexpected object ID when deleting '%s'" ), - pseudoref); -rollback_lock_file (&lock); -return -1 ; - } - -unlink (filename); -rollback_lock_file (&lock); - }else { -unlink (filename); - } - -return 0 ; -} int refs_delete_ref(struct ref_store *refs, const char *msg, const char *refname,
refs.c int refs_delete_ref(struct ref_store *refs, const char *msg, · line 847
Quoted 4 lines
refs.c +0
-1@@ -844,8 +749,7 @@ int refs_delete_ref(struct ref_store *refs, const char *msg,struct strbuf err =STRBUF_INIT ;if (ref_type (refname) ==REF_TYPE_PSEUDOREF ) { -assert (refs ==get_main_ref_store (the_repository));
I don't think you meant to delete this line, the equivalent line is left untouched in the next hunk and there are comments in refs.h saying this should only me called on the main repository
Quoted 74 lines
refs.c +17
-2-return delete_pseudoref (refname, old_oid); +return ref_store_delete_pseudoref (refs, refname, old_oid); } transaction =ref_store_transaction_begin (refs, &err);@@ -1172,7 +1076,8 @@ int refs_update_ref(struct ref_store *refs, const char *msg,if (ref_type (refname) ==REF_TYPE_PSEUDOREF ) {assert (refs ==get_main_ref_store (the_repository)); - ret =write_pseudoref (refname, new_oid, old_oid, &err); + ret =ref_store_write_pseudoref (refs, refname, new_oid, old_oid, + &err); }else { t =ref_store_transaction_begin (refs, &err); if (!t ||@@ -1441,6 +1346,20 @@ int head_ref(each_ref_fn fn, void *cb_data)return refs_head_ref (get_main_ref_store (the_repository), fn, cb_data); } +int ref_store_write_pseudoref (struct ref_store *refs,const char *pseudoref, +const struct object_id *oid, +const struct object_id *old_oid, +struct strbuf *err) +{ +return refs->be->write_pseudoref (refs, pseudoref, oid, old_oid, err); +} + +int ref_store_delete_pseudoref (struct ref_store *refs,const char *pseudoref, +const struct object_id *old_oid) +{ +return refs->be->delete_pseudoref (refs, pseudoref, old_oid); +} +struct ref_iterator *refs_ref_iterator_begin (struct ref_store *refs,const char *prefix,int trim,int flags)refs.h +11
-0@@ -728,6 +728,17 @@ int update_ref(const char *msg, const char *refname, const struct object_id *new_oid, const struct object_id *old_oid, unsigned int flags, enum action_on_err onerr); +/* Pseudorefs (eg. HEAD, CHERRY_PICK_HEAD) have a separate routines for updating +and deletion as they cannot take part in normal transactional updates. +Pseudorefs should only be written for the main repository. +*/ +int ref_store_write_pseudoref (struct ref_store *refs,const char *pseudoref, +const struct object_id *oid, +const struct object_id *old_oid, +struct strbuf *err); +int ref_store_delete_pseudoref (struct ref_store *refs,const char *pseudoref, +const struct object_id *old_oid); +int parse_hide_refs_config (const char *var,const char *value,const char *);/* refs/files-backend.c +8
-0@@ -731,6 +731,115 @@ static int lock_raw_ref(struct files_ref_store *refs,return ret; } +static int files_write_pseudoref (struct ref_store *ref_store, +const char *pseudoref, +const struct object_id *oid, +const struct object_id *old_oid, +struct strbuf *err) +{ +struct files_ref_store *refs = +files_downcast (ref_store,REF_STORE_READ ,"write_pseudoref" );
refs/files-backend.c static int lock_raw_ref(struct files_ref_store *refs, · line 751
Quoted 10 lines
refs/files-backend.c +10
-0+int fd; +struct lock_file lock =LOCK_INIT ; +struct strbuf filename =STRBUF_INIT ; +struct strbuf buf =STRBUF_INIT ; +int ret =-1 ; + +if (!oid) +return 0 ; + +strbuf_addf (&filename,"%s/%s" , refs->gitdir, pseudoref);
The original used git_path(), this change looks correct to an uninformed reviewer. As far as I can see the only changes to this function are to accommodate the strbuf which is cleanup up correctly.
Quoted 47 lines
refs/files-backend.c +47
-0+ fd =hold_lock_file_for_update_timeout (&lock, filename.buf,0 , +get_files_ref_lock_timeout_ms ()); +if (fd <0 ) { +strbuf_addf (err,_ ("could not open '%s' for writing: %s" ), + buf.buf,strerror (errno)); + goto done; + } + +if (old_oid) { +struct object_id actual_old_oid; + +if (read_ref (pseudoref, &actual_old_oid)) { +if (!is_null_oid (old_oid)) { +strbuf_addf (err,_ ("could not read ref '%s'" ), + pseudoref); +rollback_lock_file (&lock); + goto done; + } + }else if (is_null_oid (old_oid)) { +strbuf_addf (err,_ ("ref '%s' already exists" ), + pseudoref); +rollback_lock_file (&lock); + goto done; + }else if (!oideq (&actual_old_oid, old_oid)) { +strbuf_addf (err, +_ ("unexpected object ID when writing '%s'" ), + pseudoref); +rollback_lock_file (&lock); + goto done; + } + } + +strbuf_addf (&buf,"%s\n" ,oid_to_hex (oid)); +if (write_in_full (fd, buf.buf, buf.len) <0 ) { +strbuf_addf (err,_ ("could not write to '%s'" ), filename.buf); +rollback_lock_file (&lock); + goto done; + } + +commit_lock_file (&lock); + ret =0 ; +done: +strbuf_release (&buf); +strbuf_release (&filename); +return ret; +} +
refs/files-backend.c static int lock_raw_ref(struct files_ref_store *refs, · line 808
Quoted 10 lines
refs/files-backend.c +10
-0+static int files_delete_pseudoref (struct ref_store *ref_store, +const char *pseudoref, +const struct object_id *old_oid) +{ +struct files_ref_store *refs = +files_downcast (ref_store,REF_STORE_READ ,"delete_pseudoref" ); +struct strbuf filename =STRBUF_INIT ; +int ret =-1 ; + +strbuf_addf (&filename,"%s/%s" , refs->gitdir, pseudoref);
Same comment as above, the memory management looks good.
Quoted 86 lines
refs/files-backend.c +52
-27+ +if (old_oid && !is_null_oid (old_oid)) { +struct lock_file lock =LOCK_INIT ; +int fd; +struct object_id actual_old_oid; + + fd =hold_lock_file_for_update_timeout ( + &lock, filename.buf,0 , +get_files_ref_lock_timeout_ms ()); +if (fd <0 ) { +error_errno (_ ("could not open '%s' for writing" ), + filename.buf); + goto done; + } +if (read_ref (pseudoref, &actual_old_oid)) +die (_ ("could not read ref '%s'" ), pseudoref); +if (!oideq (&actual_old_oid, old_oid)) { +error (_ ("unexpected object ID when deleting '%s'" ), + pseudoref); +rollback_lock_file (&lock); + goto done; + } + +unlink (filename.buf); +rollback_lock_file (&lock); + }else { +unlink (filename.buf); + } + ret =0 ; +done: +strbuf_release (&filename); +return ret; +} +struct files_ref_iterator {struct ref_iterator base;@@ -3173,30 +3282,31 @@ static int files_init_db(struct ref_store *ref_store, struct strbuf *err)return 0 ; } -struct ref_storage_be refs_be_files = { -NULL , -"files" , - files_ref_store_create, - files_init_db, - files_transaction_prepare, - files_transaction_finish, - files_transaction_abort, - files_initial_transaction_commit, - - files_pack_refs, - files_create_symref, - files_delete_refs, - files_rename_ref, - files_copy_ref, - - files_ref_iterator_begin, - files_read_raw_ref, - - files_reflog_iterator_begin, - files_for_each_reflog_ent, - files_for_each_reflog_ent_reverse, - files_reflog_exists, - files_create_reflog, - files_delete_reflog, - files_reflog_expire -}; +struct ref_storage_be refs_be_files = {NULL , +"files" , + files_ref_store_create, + files_init_db, + files_transaction_prepare, + files_transaction_finish, + files_transaction_abort, + files_initial_transaction_commit, + + files_pack_refs, + files_create_symref, + files_delete_refs, + files_rename_ref, + files_copy_ref, + + files_write_pseudoref, + files_delete_pseudoref, +
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, struct strbuf *err) · line 3312
Quoted 10 lines
refs/files-backend.c +10
-0+ files_ref_iterator_begin, + files_read_raw_ref, + + files_reflog_iterator_begin, + files_for_each_reflog_ent, + files_for_each_reflog_ent_reverse, + files_reflog_exists, + files_create_reflog, + files_delete_reflog, + files_reflog_expire };
The formatting has gone haywire
Quoted 32 lines
refs/packed-backend.c +0
-24@@ -1641,29 +1641,19 @@ static int packed_reflog_expire(struct ref_store *ref_store, }struct ref_storage_be refs_be_packed = { -NULL , -"packed" , - packed_ref_store_create, - packed_init_db, - packed_transaction_prepare, - packed_transaction_finish, - packed_transaction_abort, - packed_initial_transaction_commit, - - packed_pack_refs, - packed_create_symref, - packed_delete_refs, - packed_rename_ref, - packed_copy_ref, - - packed_ref_iterator_begin, - packed_read_raw_ref, - - packed_reflog_iterator_begin, - packed_for_each_reflog_ent, - packed_for_each_reflog_ent_reverse, - packed_reflog_exists, - packed_create_reflog, - packed_delete_reflog,
refs/packed-backend.c static int packed_reflog_expire(struct ref_store *ref_store, · line 1652
Quoted 10 lines
refs/packed-backend.c +9
-1- packed_reflog_expire +NULL ,"packed" , packed_ref_store_create, packed_init_db, + packed_transaction_prepare, packed_transaction_finish, + packed_transaction_abort, packed_initial_transaction_commit, + + packed_pack_refs, packed_create_symref, packed_delete_refs, + packed_rename_ref, packed_copy_ref, + +/* XXX */ + NULL, NULL,
Should the wrappers above that invoke these virtual functions check they are non-null before dereferencing them? It would be better to die with BUG() than segfault.
refs/packed-backend.c static int packed_reflog_expire(struct ref_store *ref_store, · line 1658
Quoted 6 lines
refs/packed-backend.c +6
-0+ + packed_ref_iterator_begin, packed_read_raw_ref, + + packed_reflog_iterator_begin, packed_for_each_reflog_ent, + packed_for_each_reflog_ent_reverse, packed_reflog_exists, + packed_create_reflog, packed_delete_reflog, packed_reflog_expire
Formatting again
I think this patch basically works but I'm concerned by the potential NULL pointer dereference. While it's unfair to judge a patch by it's formatting the changes to the formatting of existing code and the dropped assertion rightly or wrongly gave me the impression lack of attention which does make me concerned that there are other more serious unintentional changes in the rest of the series.
Best Wishes
Phillip
Quoted 32 lines
refs/packed-backend.c +0
-0};refs/refs-internal.h +12
-0@@ -549,6 +549,15 @@ typedef int copy_ref_fn(struct ref_store *ref_store, const char *oldref, const char *newref, const char *logmsg); +typedef int write_pseudoref_fn (struct ref_store *ref_store, +const char *pseudoref, +const struct object_id *oid, +const struct object_id *old_oid, +struct strbuf *err); +typedef int delete_pseudoref_fn (struct ref_store *ref_store, +const char *pseudoref, +const struct object_id *old_oid); +/* * Iterate over the references in `ref_store` whose names start with * `prefix`. `prefix` is matched as a literal string, without regard @@ -648,6 +657,9 @@ struct ref_storage_be {rename_ref_fn *rename_ref;copy_ref_fn *copy_ref; +write_pseudoref_fn *write_pseudoref; +delete_pseudoref_fn *delete_pseudoref; +ref_iterator_begin_fn *iterator_begin;read_raw_ref_fn *read_raw_ref;
Re: [PATCH v13 07/13] Write pseudorefs through ref backends.
Quoted 6 lines · Phillip Wood
Quoted 2 lines
if (ref_type(refname) == REF_TYPE_PSEUDOREF) {-
assert(refs == get_main_ref_store(the_repository));I don't think you meant to delete this line, the equivalent line is left untouched in the next hunk and there are comments in
refs.hsaying this should only me called on the main repository
yep. - Whoops. Fixed.
Quoted 8 lines
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, struct strbuf *err) · line 3287
Quoted 6 lines
refs/files-backend.c +3
-3-struct ref_storage_be refs_be_files = { -NULL , -"files" , +struct ref_storage_be refs_be_files = {NULL , +"files" , + files_ref_store_create,The formatting has gone haywire
This was clang-format's automatic reformatting. I've looked a bit closer, and it was reformatting because the initializer list was missing the ',' on the final entry. I've added that now.
Quoted 5 lines
Quoted 1 line
+ NULL, NULL,
Should the wrappers above that invoke these virtual functions check they are non-null before dereferencing them? It would be better to die with
BUG()than segfault.
Done.
Quoted 6 lines
I think this patch basically works but I'm concerned by the potential NULL pointer dereference. While it's unfair to judge a patch by it's formatting the changes to the formatting of existing code and the dropped assertion rightly or wrongly gave me the impression lack of attention which does make me concerned that there are other more serious unintentional changes in the rest of the series.
I prefer leaving formatting up to automated tooling. They're better at following mechanical rules precisely.
Signature · 7 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Re: [PATCH v13 07/13] Write pseudorefs through ref backends.
Quoted 19 lines · Han-Wen Nienhuys
Quoted 6 lines · Phillip Wood
Quoted 2 lines
if (ref_type(refname) == REF_TYPE_PSEUDOREF) {-
assert(refs == get_main_ref_store(the_repository));I don't think you meant to delete this line, the equivalent line is left untouched in the next hunk and there are comments in
refs.hsaying this should only me called on the main repositoryyep. - Whoops. Fixed.
Quoted 8 lines
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, struct strbuf *err) · line 3287
Quoted 6 lines
refs/files-backend.c +3
-3-struct ref_storage_be refs_be_files = { -NULL , -"files" , +struct ref_storage_be refs_be_files = {NULL , +"files" , + files_ref_store_create,The formatting has gone haywire
Quoted 3 lines
This was clang-format's automatic reformatting. I've looked a bit closer, and it was reformatting because the initializer list was missing the ',' on the final entry. I've added that now.
It sounds like we need to look at the clang format rules we use, I think the original code is correctly formatted in this case (we've only allowed trailing commas relatively recently), it shouldn't need changing (which clutters up the patch with unrelated changes) just to satisfy clang-format.
Quoted 7 lines
Quoted 5 lines
Quoted 1 line
+ NULL, NULL,
Should the wrappers above that invoke these virtual functions check they are non-null before dereferencing them? It would be better to die with
BUG()than segfault.Done.
Great, I did wonder after I had sent the email if it would be better to implement the BUG() in the virtual functions in the packed-refs backend to avoid the check overhead in the other backends but it's probably not worth worrying about.
Quoted 10 lines
Quoted 7 lines
I think this patch basically works but I'm concerned by the potential NULL pointer dereference. While it's unfair to judge a patch by it's formatting the changes to the formatting of existing code and the dropped assertion rightly or wrongly gave me the impression lack of attention which does make me concerned that there are other more serious unintentional changes in the rest of the series.
I prefer leaving formatting up to automated tooling. They're better at following mechanical rules precisely.
Right but in this case the changes completely obscured the important changes. What really raised a flag for we was that two definitions of the same structure where reformatted in completely different ways - I do use clang format but I also sanity check the results before submitting a patch.
I was looking at our use of git_path_cherry_pick_head() in sequencer.c (I mostly work on rebase) the other day, there are quite a few places we rely on CHERRY_PICK_HEAD/REVERT being a file, I suspect we probably do the same in wt-status.c and builtin/commit.c. The uses are generally
file_exists(git_path_cherry_pick_head())
which I think we could just change to use get_oid()
We also unlink() it in several places which we could replace with delete_ref() but we blindly call unlink() in some places so the ref might not exist.
In the sequencer we always use the refs api when handling REBASE_HEAD, I haven't got round to checking builtin/rebase.c and builtin/am.c for that yet.
Is there a plan to handle FETCH_HEAD with reftable? git rev-parse will parse the first oid in the file which is handy if you've only fetched a single ref, but the file itself contains several lines like
2407200be2a37bfca57ea1a9474318822fec49b0 branch 'git-alias' of
ssh://pi/home/pi/git
It might be special cased in the refs code already I haven't checked
MERGE_HEAD also contains multiple lines for octopus merges but I'm not sure if people really need to run rev-parse on that.
Apologies if the FETCH_HEAD and MERGE_HEAD points have already been covered elsewhere, this thread has got quite long.
Best Wishes
Phillip
Re: [PATCH v13 07/13] Write pseudorefs through ref backends.
Quoted 1 line · Phillip Wood
On 12/05/2020 17:48, Han-Wen Nienhuys wrote:
[...]
Quoted 13 lines
Quoted 9 lines
Quoted 8 lines
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, struct strbuf *err) · line 3287
Quoted 6 lines
refs/files-backend.c +3
-3-struct ref_storage_be refs_be_files = { -NULL , -"files" , +struct ref_storage_be refs_be_files = {NULL , +"files" , + files_ref_store_create,The formatting has gone haywire
Quoted 3 lines
This was clang-format's automatic reformatting. I've looked a bit closer, and it was reformatting because the initializer list was missing the ',' on the final entry. I've added that now.
Quoted 5 lines
It sounds like we need to look at the clang format rules we use, I think the original code is correctly formatted in this case (we've only allowed trailing commas relatively recently), it shouldn't need changing (which clutters up the patch with unrelated changes) just to satisfy clang-format.
I spent some time trying to fix the clang-format rules this afternoon and got nowhere. Searching the web revealed plenty of questions but no useful answers beyond "add a trailing comma" and it's not even clear to me if that works all time. It is a shame clang-format does not seem to support a fairly common way of formatting structure initializers. That the formatting changes so much based on the presence or absence of the trailing comma is really confusing and unhelpful.
Best Wishes
Phillip
Quoted 64 lines
Quoted 8 lines
Quoted 6 lines
Quoted 2 lines
+ NULL, NULL,
Should the wrappers above that invoke these virtual functions check they are non-null before dereferencing them? It would be better to die with
BUG()than segfault.Done.
Great, I did wonder after I had sent the email if it would be better to implement the
BUG()in the virtual functions in the packed-refs backend to avoid the check overhead in the other backends but it's probably not worth worrying about.Quoted 10 lines
Quoted 7 lines
I think this patch basically works but I'm concerned by the potential NULL pointer dereference. While it's unfair to judge a patch by it's formatting the changes to the formatting of existing code and the dropped assertion rightly or wrongly gave me the impression lack of attention which does make me concerned that there are other more serious unintentional changes in the rest of the series.
I prefer leaving formatting up to automated tooling. They're better at following mechanical rules precisely.
Right but in this case the changes completely obscured the important changes. What really raised a flag for we was that two definitions of the same structure where reformatted in completely different ways - I do use clang format but I also sanity check the results before submitting a patch.
I was looking at our use of
git_path_cherry_pick_head()insequencer.c(I mostly work on rebase) the other day, there are quite a few places we rely on CHERRY_PICK_HEAD/REVERT being a file, I suspect we probably do the same inwt-status.candbuiltin/commit.c. The uses are generally
file_exists(git_path_cherry_pick_head())
which I think we could just change to useget_oid()
We alsounlink()it in several places which we could replace withdelete_ref()but we blindly callunlink()in some places so the ref might not exist.In the sequencer we always use the refs api when handling
REBASE_HEAD, I haven't got round to checkingbuiltin/rebase.candbuiltin/am.cfor that yet.Is there a plan to handle
FETCH_HEADwith reftable? git rev-parse will parse the first oid in the file which is handy if you've only fetched a single ref, but the file itself contains several lines like
2407200be2a37bfca57ea1a9474318822fec49b0branch 'git-alias' of
ssh://pi/home/pi/gitIt might be special cased in the refs code already I haven't checked
MERGE_HEADalso contains multiple lines for octopus merges but I'm not sure if people really need to run rev-parse on that.Apologies if the
FETCH_HEADandMERGE_HEADpoints have already been covered elsewhere, this thread has got quite long.Best Wishes
Phillip
PATCH v13 8/13 Iterate over the "refs/" namespace in for_each_[raw]ref
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This happens implicitly in the files/packed ref backend; making it explicit simplifies adding alternate ref storage backends, such as reftable.
--- refs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
refs.c +3 -3
@@ -1444,7 +1444,7 @@ static int do_for_each_ref(struct ref_store *refs, const char *prefix,int refs_for_each_ref (struct ref_store *refs,each_ref_fn fn,void *cb_data) { -return do_for_each_ref (refs,"" , fn,0 ,0 , cb_data); +return do_for_each_ref (refs,"refs/" , fn,0 ,0 , cb_data); }int for_each_ref (each_ref_fn fn,void *cb_data)
@@ -1504,8 +1504,8 @@ int for_each_namespaced_ref(each_ref_fn fn, void *cb_data)int refs_for_each_rawref (struct ref_store *refs,each_ref_fn fn,void *cb_data) { -return do_for_each_ref (refs,"" , fn,0 , -DO_FOR_EACH_INCLUDE_BROKEN , cb_data); +return do_for_each_ref (refs,"refs/" , fn,0 ,DO_FOR_EACH_INCLUDE_BROKEN , + cb_data); }int for_each_rawref (each_ref_fn fn,void *cb_data)
Signature · 1 line
gitgitgadget
PATCH v13 9/13 Add .gitattributes for the reftable/ directory
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
--- reftable/.gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 reftable/.gitattributes
reftable/.gitattributes new file +1 -0
@@ -0,0 +1 @@ +/zlib-compat.c whitespace=-indent-with-non-tab,-trailing-space
Signature · 1 line
gitgitgadget
PATCH v13 10/13 Add reftable library
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
Reftable is a format for storing the ref database. Its rationale and specification is in the preceding commit.
This imports the upstream library as one big commit. For understanding the code, it is suggested to read the code in the following order:
* The specification under Documentation/technical/reftable.txt
* reftable.h - the public API
* record.{c,h} - reading and writing records
* block.{c,h} - reading and writing blocks.
* writer.{c,h} - writing a complete reftable file.
* merged.{c,h} and pq.{c,h} - reading a stack of reftables
* stack.{c,h} - writing and compacting stacks of reftable on the filesystem.
--- reftable/LICENSE | 31 + reftable/README.md | 11 + reftable/VERSION | 14 + reftable/basics.c | 215 +++++++ reftable/basics.h | 53 ++ reftable/block.c | 434 ++++++++++++++ reftable/block.h | 129 +++++ reftable/constants.h | 21 + reftable/file.c | 99 ++++ reftable/iter.c | 240 ++++++++ reftable/iter.h | 63 ++ reftable/merged.c | 325 +++++++++++ reftable/merged.h | 38 ++ reftable/pq.c | 114 ++++ reftable/pq.h | 34 ++ reftable/reader.c | 753 ++++++++++++++++++++++++ reftable/reader.h | 65 +++ reftable/record.c | 1141 ++++++++++++++++++++++++++++++++++++ reftable/record.h | 121 ++++ reftable/refname.c | 215 +++++++ reftable/refname.h | 38 ++ reftable/reftable.c | 91 +++ reftable/reftable.h | 564 ++++++++++++++++++ reftable/slice.c | 225 ++++++++ reftable/slice.h | 76 +++ reftable/stack.c | 1245 ++++++++++++++++++++++++++++++++++++++++ reftable/stack.h | 48 ++ reftable/system.h | 54 ++ reftable/tree.c | 67 +++ reftable/tree.h | 34 ++ reftable/update.sh | 24 + reftable/writer.c | 665 +++++++++++++++++++++ reftable/writer.h | 60 ++ reftable/zlib-compat.c | 92 +++ 34 files changed, 7399 insertions(+) create mode 100644 reftable/LICENSE create mode 100644 reftable/README.md create mode 100644 reftable/VERSION create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/constants.h create mode 100644 reftable/file.c create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/refname.c create mode 100644 reftable/refname.h create mode 100644 reftable/reftable.c create mode 100644 reftable/reftable.h create mode 100644 reftable/slice.c create mode 100644 reftable/slice.h create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/system.h create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100755 reftable/update.sh create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h create mode 100644 reftable/zlib-compat.c
reftable/LICENSE new file +31 -0
@@ -0,0 +1,31 @@ +BSD License + +Copyright (c) 2020, Google LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of Google LLC nor the names of its contributors may +be used to endorse or promote products derived from this software +without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
reftable/README.md new file +11 -0
@@ -0,0 +1,11 @@ + +The source code in this directory comes from https://github.com/google/reftable. + +The VERSION file keeps track of the current version of the reftable library. + +To update the library, do: + + sh reftable/update.sh + +Bugfixes should be accompanied by a test and applied to upstream project at +https://github.com/google/reftable.
reftable/VERSION new file +14 -0
@@ -0,0 +1,14 @@ +commit e74c14b66b6c15f6526c485f2e45d3f2735d359d +Author: Han-Wen Nienhuys <[email protected]> +Date: Mon May 11 21:02:55 2020 +0200 + + C: handle out-of-date reftable stacks + + * Make reftable_stack_reload() check out-of-dateness. This makes it very cheap, + allowing it to be called often. This is useful because Git calls itself often, + which effectively requires a reload. + + * In reftable_stack_add(), check the return value of stack_uptodate(), leading + to erroneously succeeding the transaction. + + * A test that exercises the above.
reftable/basics.c new file +215 -0
215 lines not on this page · raw message
reftable/basics.h new file +53 -0
53 lines not on this page · raw message
reftable/block.c new file +434 -0
434 lines not on this page · raw message
reftable/block.h new file +129 -0
129 lines not on this page · raw message
reftable/constants.h new file +21 -0
@@ -0,0 +1,21 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef CONSTANTS_H +#define CONSTANTS_H + +#define BLOCK_TYPE_LOG 'g' +#define BLOCK_TYPE_INDEX 'i' +#define BLOCK_TYPE_REF 'r' +#define BLOCK_TYPE_OBJ 'o' +#define BLOCK_TYPE_ANY 0 + +#define MAX_RESTARTS ((1 << 16) - 1) +#define DEFAULT_BLOCK_SIZE 4096 + +#endif
reftable/file.c new file +99 -0
99 lines not on this page · raw message
reftable/iter.c new file +240 -0
240 lines not on this page · raw message
reftable/iter.h new file +63 -0
63 lines not on this page · raw message
reftable/merged.c new file +325 -0
325 lines not on this page · raw message
reftable/merged.h new file +38 -0
@@ -0,0 +1,38 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef MERGED_H +#define MERGED_H + +#include "pq.h" +#include "reftable.h" + +struct reftable_merged_table { +struct reftable_reader **stack; +int stack_len; +uint32_t hash_id; +bool suppress_deletions; + +uint64_t min; +uint64_t max; +}; + +struct merged_iter { +struct reftable_iterator *stack; +uint32_t hash_id; +int stack_len; +byte typ; +bool suppress_deletions; +struct merged_iter_pqueue pq; +}; + +void merged_table_clear (struct reftable_merged_table *mt); +int merged_table_seek_record (struct reftable_merged_table *mt, +struct reftable_iterator *it,struct record rec); + +#endif
reftable/pq.c new file +114 -0
114 lines not on this page · raw message
reftable/pq.h new file +34 -0
@@ -0,0 +1,34 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef PQ_H +#define PQ_H + +#include "record.h" + +struct pq_entry { +int index; +struct record rec; +}; + +int pq_less (struct pq_entry a,struct pq_entry b); + +struct merged_iter_pqueue { +struct pq_entry *heap; +int len; +int cap; +}; + +struct pq_entry merged_iter_pqueue_top (struct merged_iter_pqueue pq); +bool merged_iter_pqueue_is_empty (struct merged_iter_pqueue pq); +void merged_iter_pqueue_check (struct merged_iter_pqueue pq); +struct pq_entry merged_iter_pqueue_remove (struct merged_iter_pqueue *pq); +void merged_iter_pqueue_add (struct merged_iter_pqueue *pq,struct pq_entry e); +void merged_iter_pqueue_clear (struct merged_iter_pqueue *pq); + +#endif
reftable/reader.c new file +753 -0
753 lines not on this page · raw message
reftable/reader.h new file +65 -0
65 lines not on this page · raw message
reftable/record.c new file +1141 -0
361 lines not on this page · raw message
reftable/record.h new file +121 -0
121 lines not on this page · raw message
reftable/refname.c new file +215 -0
215 lines not on this page · raw message
reftable/reftable.c new file +91 -0
91 lines not on this page · raw message
reftable/reftable.h new file +564 -0
564 lines not on this page · raw message
reftable/slice.c new file +225 -0
225 lines not on this page · raw message
reftable/slice.h new file +76 -0
76 lines not on this page · raw message
reftable/stack.c new file +1245 -0
1,245 lines not on this page · raw message
reftable/stack.h new file +48 -0
48 lines not on this page · raw message
reftable/system.h new file +54 -0
54 lines not on this page · raw message
reftable/tree.c new file +67 -0
67 lines not on this page · raw message
reftable/writer.c new file +665 -0
665 lines not on this page · raw message
reftable/writer.h new file +60 -0
60 lines not on this page · raw message
reftable/zlib-compat.c new file +92 -0
92 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
gitgitgadget
Part of the diff is not shown here. View the raw message.
PATCH v13 11/13 Reftable support for git-core
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
For background, see the previous commit introducing the library.
This introduces the refs/reftable-backend.c containing reftable powered ref storage backend.
It can be activated by passing --ref-storage=reftable to "git init".
TODO:
* Resolve spots marked with XXX
Example use: see t/t0031-reftable.sh
--- .../technical/repository-version.txt | 7 + Makefile | 27 +- builtin/clone.c | 3 +- builtin/init-db.c | 56 +- cache.h | 6 +- refs.c | 27 +- refs.h | 3 + refs/refs-internal.h | 1 + refs/reftable-backend.c | 1218 +++++++++++++++++ repository.c | 2 + repository.h | 3 + setup.c | 12 +- t/t0031-reftable.sh | 120 ++ 13 files changed, 1455 insertions(+), 30 deletions(-) create mode 100644 refs/reftable-backend.c create mode 100755 t/t0031-reftable.sh
Documentation/technical/repository-version.txt +7 -0
@@ -100,3 +100,10 @@ If set, by default "git config" reads from both "config" and multiple working directory mode, "config" file is shared while "config.worktree" is per-working directory (i.e., it's in GIT_COMMON_DIR/worktrees/<id>/config.worktree) + +==== `refStorage` + +Specifies the file format for the ref database. Values are `files` +(for the traditional packed + loose ref format) and `reftable` for the +binary reftable format. See https://github.com/google/reftable for +more information.
Makefile +25 -2
@@ -810,6 +810,7 @@ TEST_SHELL_PATH = $(SHELL_PATH) LIB_FILE = libgit.a XDIFF_LIB = xdiff/lib.a VCSSVN_LIB = vcs-svn/lib.a +REFTABLE_LIB = reftable/libreftable.a GENERATED_H += config-list.h GENERATED_H += command-list.h
@@ -962,6 +963,7 @@ LIB_OBJS += ref-filter.o LIB_OBJS += reflog-walk.o LIB_OBJS += refs.o LIB_OBJS += refs/files-backend.o +LIB_OBJS += refs/reftable-backend.o LIB_OBJS += refs/iterator.o LIB_OBJS += refs/packed-backend.o LIB_OBJS += refs/ref-cache.o
@@ -1165,7 +1167,7 @@ THIRD_PARTY_SOURCES += compat/regex/% THIRD_PARTY_SOURCES += sha1collisiondetection/% THIRD_PARTY_SOURCES += sha1dc/% -GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) +GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) EXTLIBS = GIT_USER_AGENT = git/$(GIT_VERSION)
@@ -2360,11 +2362,29 @@ VCSSVN_OBJS += vcs-svn/sliding_window.o VCSSVN_OBJS += vcs-svn/svndiff.o VCSSVN_OBJS += vcs-svn/svndump.o +REFTABLE_OBJS += reftable/basics.o +REFTABLE_OBJS += reftable/block.o +REFTABLE_OBJS += reftable/file.o +REFTABLE_OBJS += reftable/iter.o +REFTABLE_OBJS += reftable/merged.o +REFTABLE_OBJS += reftable/pq.o +REFTABLE_OBJS += reftable/reader.o +REFTABLE_OBJS += reftable/record.o +REFTABLE_OBJS += reftable/refname.o +REFTABLE_OBJS += reftable/reftable.o +REFTABLE_OBJS += reftable/slice.o +REFTABLE_OBJS += reftable/stack.o +REFTABLE_OBJS += reftable/tree.o +REFTABLE_OBJS += reftable/writer.o +REFTABLE_OBJS += reftable/zlib-compat.o + + TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) OBJECTS := $(LIB_OBJS) $(BUILTIN_OBJS) $(PROGRAM_OBJS) $(TEST_OBJS) \ $(XDIFF_OBJS) \ $(VCSSVN_OBJS) \ $(FUZZ_OBJS) \ + $(REFTABLE_OBJS) \ common-main.o \ git.o ifndef NO_CURL
@@ -2505,6 +2525,9 @@ $(XDIFF_LIB): $(XDIFF_OBJS) $(VCSSVN_LIB): $(VCSSVN_OBJS) $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ +$(REFTABLE_LIB): $(REFTABLE_OBJS) + $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ + export DEFAULT_EDITOR DEFAULT_PAGER Documentation/GIT-EXCLUDED-PROGRAMS: FORCE
@@ -3120,7 +3143,7 @@ cocciclean: clean: profile-clean coverage-clean cocciclean $(RM) *.res $(RM) $(OBJECTS) - $(RM) $(LIB_FILE) $(XDIFF_LIB) $(VCSSVN_LIB) + $(RM) $(LIB_FILE) $(XDIFF_LIB) $(VCSSVN_LIB) $(REFTABLE_LIB) $(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) git$X $(RM) $(TEST_PROGRAMS) $(RM) $(FUZZ_PROGRAMS)
builtin/clone.c +2 -1
@@ -1108,7 +1108,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix) } } -init_db (git_dir, real_git_dir, option_template,GIT_HASH_UNKNOWN ,INIT_DB_QUIET ); +init_db (git_dir, real_git_dir, option_template,GIT_HASH_UNKNOWN , +DEFAULT_REF_STORAGE ,INIT_DB_QUIET );if (real_git_dir) git_dir = real_git_dir;
builtin/init-db.c +42 -14
132 lines not on this page · raw message
cache.h +4 -2
@@ -628,8 +628,9 @@ int path_inside_repo(const char *prefix, const char *path);int init_db (const char *git_dir,const char *real_git_dir,const char *template_dir,int hash_algo, -unsigned int flags); -void initialize_repository_version (int hash_algo); +const char *ref_storage_format,unsigned int flags); +void initialize_repository_version (int hash_algo, +const char *ref_storage_format);void sanitize_stdfds (void );int daemonize (void );
@@ -1043,6 +1044,7 @@ struct repository_format {int is_bare;int hash_algo;char *work_tree; +char *ref_storage;struct string_list unknown_extensions; };
refs.c +19 -8
@@ -17,10 +17,16 @@#include "argv-array.h" #include "repository.h" +const char *default_ref_storage (void ) +{ +const char *test =getenv ("GIT_TEST_REFTABLE" ); +return test ?"reftable" :"files" ; +} +/* * List of all available backends */ -static struct ref_storage_be *refs_backends = &refs_be_files; +static struct ref_storage_be *refs_backends = &refs_be_reftable;static struct ref_storage_be *find_ref_storage_backend (const char *name) {
@@ -1711,13 +1717,13 @@ static struct ref_store *lookup_ref_store_map(struct hashmap *map,* Create, record, and return a ref_store instance for the specified * gitdir. */ -static struct ref_store *ref_store_init (const char *gitdir, +static struct ref_store *ref_store_init (const char *gitdir,const char *be_name,unsigned int flags) { -const char *be_name ="files" ; -struct ref_storage_be *be =find_ref_storage_backend (be_name); +struct ref_storage_be *be;struct ref_store *refs; + be =find_ref_storage_backend (be_name);if (!be)BUG ("reference backend %s is unknown" , be_name);
@@ -1733,7 +1739,11 @@ struct ref_store *get_main_ref_store(struct repository *r)if (!r->gitdir)BUG ("attempting to get main_ref_store outside of repository" ); - r->refs_private =ref_store_init (r->gitdir,REF_STORE_ALL_CAPS ); + r->refs_private =ref_store_init (r->gitdir, + r->ref_storage_format ? + r->ref_storage_format : +DEFAULT_REF_STORAGE , +REF_STORE_ALL_CAPS );return r->refs_private; }
@@ -1788,7 +1798,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule) goto done;/* assume that add_submodule_odb() has been called */ - refs =ref_store_init (submodule_sb.buf, + refs =ref_store_init (submodule_sb.buf,DEFAULT_REF_STORAGE ,/* XXX */ REF_STORE_READ |REF_STORE_ODB );register_ref_store_map (&submodule_ref_stores,"submodule" , refs, submodule);
@@ -1802,6 +1812,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule)struct ref_store *get_worktree_ref_store (const struct worktree *wt) { +const char *format =DEFAULT_REF_STORAGE ;/* XXX */ struct ref_store *refs;const char *id;
@@ -1815,9 +1826,9 @@ struct ref_store *get_worktree_ref_store(const struct worktree *wt)if (wt->id) refs =ref_store_init (git_common_path ("worktrees/%s" , wt->id), -REF_STORE_ALL_CAPS ); + format,REF_STORE_ALL_CAPS );else - refs =ref_store_init (get_git_common_dir (), + refs =ref_store_init (get_git_common_dir (), format,REF_STORE_ALL_CAPS );if (refs)
refs.h +3 -0
@@ -9,6 +9,9 @@ struct string_list;struct string_list_item ;struct worktree ; +/* Returns the ref storage backend to use by default. */ +const char *default_ref_storage (void ); +/* * Resolve a reference, recursively following symbolic refererences. *
refs/refs-internal.h +1 -0
@@ -673,6 +673,7 @@ struct ref_storage_be { };extern struct ref_storage_be refs_be_files; +extern struct ref_storage_be refs_be_reftable;extern struct ref_storage_be refs_be_packed;/*
refs/reftable-backend.c new file +1218 -0
1,218 lines not on this page · raw message
repository.c +2 -0
@@ -178,6 +178,8 @@ int repo_init(struct repository *repo,if (worktree)repo_set_worktree (repo, worktree); + repo->ref_storage_format =xstrdup_or_null (format.ref_storage); +clear_repository_format (&format);return 0 ;
repository.h +3 -0
@@ -74,6 +74,9 @@ struct repository {*/ struct ref_store *refs_private; +/* The format to use for the ref database. */ +char *ref_storage_format; +/* * Contains path to often used file names. */
setup.c +9 -3
@@ -468,9 +468,11 @@ static int check_repo_format(const char *var, const char *value, void *vdata)if (!value)return config_error_nonbool (var); data->partial_clone =xstrdup (value); - }else if (!strcmp (ext,"worktreeconfig" )) + }else if (!strcmp (ext,"worktreeconfig" )) { data->worktree_config =git_config_bool (var, value); -else + }else if (!strcmp (ext,"refstorage" )) { + data->ref_storage =xstrdup (value); + }else string_list_append (&data->unknown_extensions, ext); }
@@ -559,6 +561,7 @@ void clear_repository_format(struct repository_format *format)string_list_clear (&format->unknown_extensions,0 );free (format->work_tree);free (format->partial_clone); +free (format->ref_storage);init_repository_format (format); }
@@ -1204,8 +1207,11 @@ const char *setup_git_directory_gently(int *nongit_ok) gitdir =DEFAULT_GIT_DIR_ENVIRONMENT ;setup_git_env (gitdir); } -if (startup_info->have_repository) +if (startup_info->have_repository) {repo_set_hash_algo (the_repository, repo_fmt.hash_algo); + the_repository->ref_storage_format = +xstrdup_or_null (repo_fmt.ref_storage); + } }strbuf_release (&dir);
t/t0031-reftable.sh new file +120 -0
120 lines not on this page · raw message
Signature · 1 line
gitgitgadget
Re: [PATCH v13 11/13] Reftable support for git-core
builtin/clone.c int cmd_clone(int argc, const char **argv, const char *prefix) · line 1112
Quoted 11 lines · Han-Wen Nienhuys via GitGitGadget
builtin/clone.c +2
-1@@ -1108,7 +1108,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix) } } -init_db (git_dir, real_git_dir, option_template,GIT_HASH_UNKNOWN ,INIT_DB_QUIET ); +init_db (git_dir, real_git_dir, option_template,GIT_HASH_UNKNOWN , +DEFAULT_REF_STORAGE ,INIT_DB_QUIET );
Where does this symbol come from?
In addition, this will be overwritten in a later step to default_ref_storage(); perhaps in this step we should use that instead from the beginning?
PATCH v13 12/13 vcxproj: adjust for the reftable changes
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Johannes Schindelin <[email protected]>
This allows Git to be compiled via Visual Studio again after integrating the hn/reftable branch.
--- config.mak.uname | 2 +- contrib/buildsystems/Generators/Vcxproj.pm | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-)
config.mak.uname +1 -1
@@ -710,7 +710,7 @@ vcxproj: # Make .vcxproj files and add them unset QUIET_GEN QUIET_BUILT_IN; \ perl contrib/buildsystems/generate -g Vcxproj - git add -f git.sln {*,*/lib,t/helper/*}/*.vcxproj + git add -f git.sln {*,*/lib,*/libreftable,t/helper/*}/*.vcxproj # Generate the LinkOrCopyBuiltins.targets and LinkOrCopyRemoteHttp.targets file (echo '<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">' && \
contrib/buildsystems/Generators/Vcxproj.pm +10 -1
@@ -77,7 +77,7 @@ sub createProject { my $libs_release = "\n "; my $libs_debug = "\n "; if (!$static_library) { - $libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); + $libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib|reftable\/libreftable\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); $libs_debug = $libs_release; $libs_debug =~ s/zlib\.lib/zlibd\.lib/g; $libs_debug =~ s/libcurl\.lib/libcurl-d\.lib/g;
@@ -231,6 +231,7 @@ sub createProject { EOM if (!$static_library || $target =~ 'vcs-svn' || $target =~ 'xdiff') { my $uuid_libgit = $$build_structure{"LIBS_libgit_GUID"}; + my $uuid_libreftable = $$build_structure{"LIBS_reftable/libreftable_GUID"}; my $uuid_xdiff_lib = $$build_structure{"LIBS_xdiff/lib_GUID"}; print F << "EOM";
@@ -240,6 +241,14 @@ sub createProject { <ReferenceOutputAssembly>false</ReferenceOutputAssembly> </ProjectReference> EOM + if (!($name =~ /xdiff|libreftable/)) { + print F << "EOM"; + <ProjectReference Include="$cdup\\reftable\\libreftable\\libreftable.vcxproj"> + <Project>$uuid_libreftable</Project> + <ReferenceOutputAssembly>false</ReferenceOutputAssembly> + </ProjectReference> +EOM + } if (!($name =~ 'xdiff')) { print F << "EOM"; <ProjectReference Include="$cdup\\xdiff\\lib\\xdiff_lib.vcxproj">
Signature · 1 line
gitgitgadget
PATCH v13 13/13 Add some reftable testing infrastructure
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
* Add GIT_TEST_REFTABLE environment var to control default ref storage
* Add test_prerequisite REFTABLE. Skip t/t3210-pack-refs.sh for REFTABLE.
Major test failures:
* t9903-bash-prompt - The bash mode reads .git/HEAD directly
* t1400-update-ref.sh - Reads from .git/{refs,logs} directly
* t1404-update-ref-errors.sh - Manipulates .git/refs/ directly
* t1405 - inspecs .git/ directly.
* t1450-fsck.sh - manipulates .git/ directly to create invalid state
* Rebase, cherry-pick: pseudo refs aren't written through the refs backend.
Other tests by decreasing brokenness:
t1407-worktree-ref-store.sh - 5 of 5 t1413-reflog-detach.sh - 7 of 7 t1415-worktree-refs.sh - 11 of 11 t3908-stash-in-worktree.sh - 2 of 2 t4207-log-decoration-colors.sh - 2 of 2 t5515-fetch-merge-logic.sh - 17 of 17 t5900-repo-selection.sh - 8 of 8 t6016-rev-list-graph-simplify-history.sh - 12 of 12 t5573-pull-verify-signatures.sh - 15 of 16 t5612-clone-refspec.sh - 12 of 13 t5514-fetch-multiple.sh - 11 of 12 t6030-bisect-porcelain.sh - 64 of 71 t5533-push-cas.sh - 15 of 17 t5539-fetch-http-shallow.sh - 7 of 8 t7413-submodule-is-active.sh - 7 of 8 t2400-worktree-add.sh - 59 of 69 t0100-previous.sh - 5 of 6 t7419-submodule-set-branch.sh - 5 of 6 t1404-update-ref-errors.sh - 44 of 53 t6003-rev-list-topo-order.sh - 29 of 35 t1409-avoid-packing-refs.sh - 9 of 11 t5541-http-push-smart.sh - 31 of 38 t5407-post-rewrite-hook.sh - 13 of 16 t9903-bash-prompt.sh - 52 of 66 t1414-reflog-walk.sh - 9 of 12 t1507-rev-parse-upstream.sh - 21 of 28 t2404-worktree-config.sh - 9 of 12 t1505-rev-parse-last.sh - 5 of 7 t7510-signed-commit.sh - 16 of 23 t2018-checkout-branch.sh - 15 of 22
(..etc)
--- builtin/clone.c | 2 +- builtin/init-db.c | 2 +- refs.c | 6 +++--- t/t1409-avoid-packing-refs.sh | 6 ++++++ t/t3210-pack-refs.sh | 6 ++++++ t/test-lib.sh | 5 +++++ 6 files changed, 22 insertions(+), 5 deletions(-)
builtin/clone.c +1 -1
@@ -1109,7 +1109,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix) }init_db (git_dir, real_git_dir, option_template,GIT_HASH_UNKNOWN , -DEFAULT_REF_STORAGE ,INIT_DB_QUIET ); +default_ref_storage (),INIT_DB_QUIET );if (real_git_dir) git_dir = real_git_dir;
builtin/init-db.c +1 -1
@@ -545,7 +545,7 @@ static const char *const init_db_usage[] = {int cmd_init_db (int argc,const char **argv,const char *prefix) {const char *git_dir; -const char *ref_storage_format =DEFAULT_REF_STORAGE ; +const char *ref_storage_format =default_ref_storage ();const char *real_git_dir =NULL ;const char *work_tree;const char *template_dir =NULL ;
refs.c +3 -3
@@ -1742,7 +1742,7 @@ struct ref_store *get_main_ref_store(struct repository *r) r->refs_private =ref_store_init (r->gitdir, r->ref_storage_format ? r->ref_storage_format : -DEFAULT_REF_STORAGE , +default_ref_storage (),REF_STORE_ALL_CAPS );return r->refs_private; }
@@ -1798,7 +1798,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule) goto done;/* assume that add_submodule_odb() has been called */ - refs =ref_store_init (submodule_sb.buf,DEFAULT_REF_STORAGE ,/* XXX */ + refs =ref_store_init (submodule_sb.buf,default_ref_storage (),REF_STORE_READ |REF_STORE_ODB );register_ref_store_map (&submodule_ref_stores,"submodule" , refs, submodule);
@@ -1812,7 +1812,7 @@ struct ref_store *get_submodule_ref_store(const char *submodule)struct ref_store *get_worktree_ref_store (const struct worktree *wt) { -const char *format =DEFAULT_REF_STORAGE ;/* XXX */ +const char *format =default_ref_storage ();struct ref_store *refs;const char *id;
t/t1409-avoid-packing-refs.sh +6 -0
@@ -4,6 +4,12 @@ test_description='avoid rewriting packed-refs unnecessarily' . ./test-lib.sh +if test_have_prereq REFTABLE +then + skip_all='skipping pack-refs tests; incompatible with reftable' + test_done +fi + # Add an identifying mark to the packed-refs file header line. This # shouldn't upset readers, and it should be omitted if the file is # ever rewritten.
t/t3210-pack-refs.sh +6 -0
@@ -11,6 +11,12 @@ semantic is still the same. ' . ./test-lib.sh +if test_have_prereq REFTABLE +then + skip_all='skipping pack-refs tests; incompatible with reftable' + test_done +fi + test_expect_success 'enable reflogs' ' git config core.logallrefupdates true '
t/test-lib.sh +5 -0
@@ -1516,6 +1516,11 @@ FreeBSD) ;; esac +if test -n "$GIT_TEST_REFTABLE" +then + test_set_prereq REFTABLE +fi + ( COLUMNS=1 && test $COLUMNS = 1 ) && test_set_prereq COLUMNS_CAN_BE_1 test -z "$NO_PERL" && test_set_prereq PERL test -z "$NO_PTHREADS" && test_set_prereq PTHREADS
Signature · 1 line
gitgitgadget
Re: [PATCH v13 13/13] Add some reftable testing infrastructure
Quoted 18 lines · Han-Wen Nienhuys via GitGitGadget
builtin/init-db.c +1
-1@@ -545,7 +545,7 @@ static const char *const init_db_usage[] = {int cmd_init_db (int argc,const char **argv,const char *prefix) {const char *git_dir; -const char *ref_storage_format =DEFAULT_REF_STORAGE ; +const char *ref_storage_format =default_ref_storage ();const char *real_git_dir =NULL ;const char *work_tree;const char *template_dir =NULL ;refs.c +0
-0
refs.c struct ref_store *get_main_ref_store(struct repository *r) · line 1748
Quoted 8 lines
refs.c +1
-1@@ -1742,7 +1742,7 @@ struct ref_store *get_main_ref_store(struct repository *r) r->refs_private =ref_store_init (r->gitdir, r->ref_storage_format ? r->ref_storage_format : -DEFAULT_REF_STORAGE , +default_ref_storage (),REF_STORE_ALL_CAPS );return r->refs_private; }
Would it make sense to let NULL stand for "we use the default, whatever it is" so that anybody outside the implementation of the refs API does not have to care what the default is, and make ref_store_init() function the only thing that knows what it is?
Re: [PATCH v13 13/13] Add some reftable testing infrastructure
Quoted 26 lines · Junio C Hamano
Quoted 23 lines · Han-Wen Nienhuys via GitGitGadget
builtin/init-db.c +1
-1@@ -545,7 +545,7 @@ static const char *const init_db_usage[] = {int cmd_init_db (int argc,const char **argv,const char *prefix) {const char *git_dir; -const char *ref_storage_format =DEFAULT_REF_STORAGE ; +const char *ref_storage_format =default_ref_storage ();const char *real_git_dir =NULL ;const char *work_tree;const char *template_dir =NULL ;refs.c +1
-1@@ -1742,7 +1742,7 @@ struct ref_store *get_main_ref_store(struct repository *r) r->refs_private = ref_store_init(r->gitdir, r->ref_storage_format ? r->ref_storage_format : - DEFAULT_REF_STORAGE, + default_ref_storage(),
Quoted 8 lines
refs.c struct ref_store *get_main_ref_store(struct repository *r) · line 1748
Quoted 3 lines
refs.c +0
-0REF_STORE_ALL_CAPS);return r->refs_private; }Would it make sense to let NULL stand for "we use the default, whatever it is" so that anybody outside the implementation of the refs API does not have to care what the default is, and make
ref_store_init()function the only thing that knows what it is?
Is this your strong recommendation in the form of a question? :)
If it is just a question: maybe. I don't know enough of Git to say for sure.
How do submodules decide on other settings, such as the hash ID?
Signature · 7 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Re: [PATCH v13 13/13] Add some reftable testing infrastructure
Quoted 19 lines · Han-Wen Nienhuys
Quoted 17 lines · Junio C Hamano
refs.c struct ref_store *get_main_ref_store(struct repository *r) · line 1748
Quoted 12 lines
refs.c +1
-1@@ -1742,7 +1742,7 @@ struct ref_store *get_main_ref_store(struct repository *r) r->refs_private =ref_store_init (r->gitdir, r->ref_storage_format ? r->ref_storage_format : -DEFAULT_REF_STORAGE , +default_ref_storage (),REF_STORE_ALL_CAPS );return r->refs_private; }Would it make sense to let NULL stand for "we use the default, whatever it is" so that anybody outside the implementation of the refs API does not have to care what the default is, and make
ref_store_init()function the only thing that knows what it is?
Quoted 3 lines
Is this your strong recommendation in the form of a question? :)
If it is just a question: maybe. I don't know enough of Git to say for sure.
I try not to ask a rhetorical question without marking it explicitly as such. The question "instead of having to sprinkle ?: all over the place on the callers' side, would it make it better or worse to give a 'not decided' default to the ref_storage_format field?" came to mind while reading the series.
Quoted 1 line
How do submodules decide on other settings, such as the hash ID?
I am not sure the issue of "what default to use" is limited to the submodule case (e.g. "git init" and "git clone" starts from having no "current repository" and they need to make the decision among available choices). As I've seen this series and SHA-256 series conflict in the "init" area, it probably is time for me to introduce you to Brian (cc'ed) to pick his brain ;-).
Thanks.
Quoted 13 lines · Han-Wen Nienhuys via GitGitGadget
Han-Wen Nienhuys (11):
refs.h: clarify reflog iteration order
t: use update-ref and show-ref to reading/writing refs
refs: document howref_iterator_advance_fnshould handle symrefs
reftable: clarify how empty tables should be written
reftable: define version 2 of the spec to accomodate SHA256 Write pseudorefs through ref backends.
Iterate over the "refs/" namespace in for_each_[raw]ref Add .gitattributes for the reftable/ directory
Add reftable library
Reftable support for git-core
Add some reftable testing infrastructure
Quoted 5 lines
Johannes Schindelin (1):
vcxproj: adjust for the reftable changesJonathan Nieder (1):
reftable: file format documentation
I often wish we had something better than shortlog when writing a cover letter for a multi-author series like this.
Anyway.
As I already said earlier today [*1*], I think some parts of the series can be reviewed before the "dense code" part, so let's draw the line between patch 8 and 9 (or patch 7 and 8) and try to see if we can get the parts before the line reviewed adequately and merge them to at lesat 'next' during this cycle, which will close in 3 weeks.
1/13 refs.h: clarify reflog iteration order
2/13 t: use update-ref and show-ref to reading/writing refs 3/13 refs: document how ref_iterator_advance_fn should handle symrefs 4/13 reftable: file format documentation
5/13 reftable: clarify how empty tables should be written 6/13 reftable: define version 2 of the spec to accomodate SHA256 7/13 Write pseudorefs through ref backends.
8/13 Iterate over the "refs/" namespace in for_each_[raw]ref 9/13 Add .gitattributes for the reftable/ directory
10/13 Add reftable library
11/13 Reftable support for git-core
12/13 vcxproj: adjust for the reftable changes
13/13 Add some reftable testing infrastructure
Thanks.
(ps) I took the Cc: list from a message by JNeider earlier today.
[Reference]
Quoted 6 lines · Junio C Hamano
As I already said earlier today [*1*], I think some parts of the series can be reviewed before the "dense code" part, so let's draw the line between patch 8 and 9 (or patch 7 and 8) and try to see if we can get the parts before the line reviewed adequately and merge them to at lesat 'next' during this cycle, which will close in 3 weeks.
I concocted 7 (write pseudorefs) yesterday in an hour or so. It looks like it passes tests, but I think it could do with more scrutiny. I propose to merge 1-6, and leave 7-8 until I figure out how pseudorefs and worktree refs really work.
Quoted 13 lines
1/13
refs.h: clarify reflog iteration order
2/13 t: use update-ref and show-ref to reading/writing refs 3/13 refs: document howref_iterator_advance_fnshould handle symrefs 4/13 reftable: file format documentation
5/13 reftable: clarify how empty tables should be written 6/13 reftable: define version 2 of the spec to accomodate SHA256 7/13 Write pseudorefs through ref backends.
8/13 Iterate over the "refs/" namespace in for_each_[raw]ref 9/13 Add .gitattributes for the reftable/ directory
10/13 Add reftable library
11/13 Reftable support for git-core
12/13 vcxproj: adjust for the reftable changes
13/13 Add some reftable testing infrastructure
Signature · 11 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Quoted 8 lines · Han-Wen Nienhuys
Quoted 6 lines · Junio C Hamano
As I already said earlier today [*1*], I think some parts of the series can be reviewed before the "dense code" part, so let's draw the line between patch 8 and 9 (or patch 7 and 8) and try to see if we can get the parts before the line reviewed adequately and merge them to at lesat 'next' during this cycle, which will close in 3 weeks.
Quoted 4 lines
I concocted 7 (write pseudorefs) yesterday in an hour or so. It looks like it passes tests, but I think it could do with more scrutiny. I propose to merge 1-6, and leave 7-8 until I figure out how pseudorefs and worktree refs really work.
OK. Today's pushout has the boundary at that spot between 6 and 7.
v14 cover Reftable support git-core
In reply to Han-Wen Nienhuys via GitGitGadgetThis adds the reftable library, and hooks it up as a ref backend. Based on hn/refs-cleanup.
Includes testing support, to test: make -C t/ GIT_TEST_REFTABLE=1
Summary 18904 tests pass 2750 tests fail
Some issues:
* many tests inspect .git/{logs,heads}/ directly.
* worktrees broken.
* Rebase/cherry-pick still largely broken
v16
* handle pseudo refs.
* various bugfixes and fixes for mem leaks.
v17
* many style tweaks in reftable/
* fix rebase
* GIT_DEBUG_REFS support.
Han-Wen Nienhuys (8):
Write pseudorefs through ref backends.
Move REF_LOG_ONLY to refs-internal.h
Iterate over the "refs/" namespace in for_each_[raw]ref Add .gitattributes for the reftable/ directory Add reftable library
Reftable support for git-core
Add GIT_DEBUG_REFS debugging mechanism
Add reftable testing infrastructure
Johannes Schindelin (1):
vcxproj: adjust for the reftable changes
.../technical/repository-version.txt | 7 + Makefile | 28 +- builtin/clone.c | 3 +- builtin/init-db.c | 56 +- cache.h | 6 +- config.mak.uname | 2 +- contrib/buildsystems/Generators/Vcxproj.pm | 11 +- refs.c | 150 +- refs.h | 14 + refs/debug.c | 309 ++++ refs/files-backend.c | 121 +- refs/packed-backend.c | 21 +- refs/refs-internal.h | 26 + refs/reftable-backend.c | 1313 +++++++++++++++++ reftable/.gitattributes | 1 + reftable/LICENSE | 31 + reftable/README.md | 11 + reftable/VERSION | 5 + reftable/basics.c | 215 +++ reftable/basics.h | 53 + reftable/block.c | 435 ++++++ reftable/block.h | 129 ++ reftable/constants.h | 21 + reftable/file.c | 97 ++ reftable/iter.c | 241 +++ reftable/iter.h | 63 + reftable/merged.c | 327 ++++ reftable/merged.h | 39 + reftable/pq.c | 114 ++ reftable/pq.h | 34 + reftable/reader.c | 754 ++++++++++ reftable/reader.h | 65 + reftable/record.c | 1154 +++++++++++++++ reftable/record.h | 128 ++ reftable/refname.c | 215 +++ reftable/refname.h | 38 + reftable/reftable.c | 92 ++ reftable/reftable.h | 564 +++++++ reftable/slice.c | 225 +++ reftable/slice.h | 77 + reftable/stack.c | 1234 ++++++++++++++++ reftable/stack.h | 48 + reftable/system.h | 54 + reftable/tree.c | 64 + reftable/tree.h | 34 + reftable/update.sh | 24 + reftable/writer.c | 659 +++++++++ reftable/writer.h | 60 + reftable/zlib-compat.c | 92 ++ repository.c | 2 + repository.h | 3 + setup.c | 12 +- t/t0031-reftable.sh | 136 ++ t/t0033-debug-refs.sh | 18 + t/t1409-avoid-packing-refs.sh | 6 + t/t1450-fsck.sh | 6 + t/t3210-pack-refs.sh | 6 + t/t9903-bash-prompt.sh | 6 + t/test-lib.sh | 5 + 59 files changed, 9523 insertions(+), 141 deletions(-) create mode 100644 refs/debug.c create mode 100644 refs/reftable-backend.c create mode 100644 reftable/.gitattributes create mode 100644 reftable/LICENSE create mode 100644 reftable/README.md create mode 100644 reftable/VERSION create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/constants.h create mode 100644 reftable/file.c create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/refname.c create mode 100644 reftable/refname.h create mode 100644 reftable/reftable.c create mode 100644 reftable/reftable.h create mode 100644 reftable/slice.c create mode 100644 reftable/slice.h create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/system.h create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100755 reftable/update.sh create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h create mode 100644 reftable/zlib-compat.c create mode 100755 t/t0031-reftable.sh create mode 100755 t/t0033-debug-refs.sh
base-commit: 369e06977b0442dc153312a6e7fb2ecd8ff540c5
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-539%2Fhanwen%2Freftable-v14
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-539/hanwen/reftable-v14
Pull-Request: https://github.com/gitgitgadget/git/pull/539
Range-diff vs v13:
1: 8394c156eb4 < -: ----------- refs.h: clarify reflog iteration order
2: dbf45fe8753 < -: ----------- t: use update-ref and show-ref to reading/writing refs
3: be083a85fb5 < -: ----------- refs: document how ref_iterator_advance_fn should handle symrefs
4: 96fd9814a67 < -: ----------- reftable: file format documentation
5: 7aa3f92fca0 < -: ----------- reftable: clarify how empty tables should be written
6: 1e3c8f2d3e8 < -: ----------- reftable: define version 2 of the spec to accomodate SHA256
7: 2c2f94ddc0e ! 1: 46d04f6740e Write pseudorefs through ref backends.
@@ refs.c: long get_files_ref_lock_timeout_ms(void)
int refs_delete_ref(struct ref_store *refs, const char *msg,
const char *refname,
@@ refs.c: int refs_delete_ref(struct ref_store *refs, const char *msg,
- struct strbuf err = STRBUF_INIT;
if (ref_type(refname) == REF_TYPE_PSEUDOREF) {
-- assert(refs == get_main_ref_store(the_repository));
+ assert(refs == get_main_ref_store(the_repository));
- return delete_pseudoref(refname, old_oid);
+ return ref_store_delete_pseudoref(refs, refname, old_oid);
}
@@ refs/files-backend.c: static int lock_raw_ref(struct files_ref_store *refs,
struct files_ref_iterator {
struct ref_iterator base;
-@@ refs/files-backend.c: static int files_init_db(struct ref_store *ref_store, struct strbuf *err)
- return 0;
- }
+@@ refs/files-backend.c: struct ref_storage_be refs_be_files = {
+ files_rename_ref,
+ files_copy_ref,
--struct ref_storage_be refs_be_files = {
-- NULL,
-- "files",
-- files_ref_store_create,
-- files_init_db,
-- files_transaction_prepare,
-- files_transaction_finish,
-- files_transaction_abort,
-- files_initial_transaction_commit,
--
-- files_pack_refs,
-- files_create_symref,
-- files_delete_refs,
-- files_rename_ref,
-- files_copy_ref,
--
-- files_ref_iterator_begin,
-- files_read_raw_ref,
--
-- files_reflog_iterator_begin,
-- files_for_each_reflog_ent,
-- files_for_each_reflog_ent_reverse,
-- files_reflog_exists,
-- files_create_reflog,
-- files_delete_reflog,
-- files_reflog_expire
--};
-+struct ref_storage_be refs_be_files = { NULL,
-+ "files",
-+ files_ref_store_create,
-+ files_init_db,
-+ files_transaction_prepare,
-+ files_transaction_finish,
-+ files_transaction_abort,
-+ files_initial_transaction_commit,
-+
-+ files_pack_refs,
-+ files_create_symref,
-+ files_delete_refs,
-+ files_rename_ref,
-+ files_copy_ref,
++ files_write_pseudoref,
++ files_delete_pseudoref,
+
-+ files_write_pseudoref,
-+ files_delete_pseudoref,
-+
-+ files_ref_iterator_begin,
-+ files_read_raw_ref,
-+
-+ files_reflog_iterator_begin,
-+ files_for_each_reflog_ent,
-+ files_for_each_reflog_ent_reverse,
-+ files_reflog_exists,
-+ files_create_reflog,
-+ files_delete_reflog,
-+ files_reflog_expire };
+ files_ref_iterator_begin,
+ files_read_raw_ref,
+
+@@ refs/files-backend.c: struct ref_storage_be refs_be_files = {
+ files_reflog_exists,
+ files_create_reflog,
+ files_delete_reflog,
+- files_reflog_expire
++ files_reflog_expire,
+ };
## refs/packed-backend.c ##
-@@ refs/packed-backend.c: static int packed_reflog_expire(struct ref_store *ref_store,
+@@ refs/packed-backend.c: static int packed_copy_ref(struct ref_store *ref_store,
+ BUG("packed reference store does not support copying references");
}
- struct ref_storage_be refs_be_packed = {
-- NULL,
-- "packed",
-- packed_ref_store_create,
-- packed_init_db,
-- packed_transaction_prepare,
-- packed_transaction_finish,
-- packed_transaction_abort,
-- packed_initial_transaction_commit,
--
-- packed_pack_refs,
-- packed_create_symref,
-- packed_delete_refs,
-- packed_rename_ref,
-- packed_copy_ref,
--
-- packed_ref_iterator_begin,
-- packed_read_raw_ref,
--
-- packed_reflog_iterator_begin,
-- packed_for_each_reflog_ent,
-- packed_for_each_reflog_ent_reverse,
-- packed_reflog_exists,
-- packed_create_reflog,
-- packed_delete_reflog,
-- packed_reflog_expire
-+ NULL, "packed", packed_ref_store_create, packed_init_db,
-+ packed_transaction_prepare, packed_transaction_finish,
-+ packed_transaction_abort, packed_initial_transaction_commit,
-+
-+ packed_pack_refs, packed_create_symref, packed_delete_refs,
-+ packed_rename_ref, packed_copy_ref,
++static int packed_write_pseudoref(struct ref_store *ref_store,
++ const char *pseudoref,
++ const struct object_id *oid,
++ const struct object_id *old_oid,
++ struct strbuf *err)
++{
++ BUG("packed reference store does not support writing pseudo-references");
++}
+
-+ /* XXX */
-+ NULL, NULL,
++static int packed_delete_pseudoref(struct ref_store *ref_store,
++ const char *pseudoref,
++ const struct object_id *old_oid)
++{
++ BUG("packed reference store does not support deleting pseudo-references");
++}
+
-+ packed_ref_iterator_begin, packed_read_raw_ref,
+ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_store)
+ {
+ return empty_ref_iterator_begin();
+@@ refs/packed-backend.c: struct ref_storage_be refs_be_packed = {
+ packed_rename_ref,
+ packed_copy_ref,
+
++ packed_write_pseudoref,
++ packed_delete_pseudoref,
+
-+ packed_reflog_iterator_begin, packed_for_each_reflog_ent,
-+ packed_for_each_reflog_ent_reverse, packed_reflog_exists,
-+ packed_create_reflog, packed_delete_reflog, packed_reflog_expire
+ packed_ref_iterator_begin,
+ packed_read_raw_ref,
+
+@@ refs/packed-backend.c: struct ref_storage_be refs_be_packed = {
+ packed_reflog_exists,
+ packed_create_reflog,
+ packed_delete_reflog,
+- packed_reflog_expire
++ packed_reflog_expire,
};
## refs/refs-internal.h ##
-: ----------- > 2: c650f7e4345 Move REF_LOG_ONLY to refs-internal.h
8: 3becaaee66a = 3: 0c953fce52a Iterate over the "refs/" namespace in for_each_[raw]ref
9: a6f77965f84 = 4: 206b7d329f8 Add .gitattributes for the reftable/ directory
10: 8103703c358 ! 5: 9a8e504a1d0 Add reftable library
@@ reftable/README.md (new)
## reftable/VERSION (new) ##
@@
-+commit e74c14b66b6c15f6526c485f2e45d3f2735d359d
++commit bad78b6de70700933a2f93ebadaa37a5e852d9ea
+Author: Han-Wen Nienhuys <[email protected]>
-+Date: Mon May 11 21:02:55 2020 +0200
-+
-+ C: handle out-of-date reftable stacks
-+
-+ * Make reftable_stack_reload() check out-of-dateness. This makes it very cheap,
-+ allowing it to be called often. This is useful because Git calls itself often,
-+ which effectively requires a reload.
-+
-+ * In reftable_stack_add(), check the return value of stack_uptodate(), leading
-+ to erroneously succeeding the transaction.
-+
-+ * A test that exercises the above.
++Date: Mon May 18 20:12:57 2020 +0200
++
++ C: get rid of inner blocks for local scoping
## reftable/basics.c (new) ##
@@
@@ reftable/block.c (new)
+ return bw->buf[bw->header_off];
+}
+
-+/* adds the record to the block. Returns -1 if it does not fit, 0 on
++/* adds the reftable_record to the block. Returns -1 if it does not fit, 0 on
+ success */
-+int block_writer_add(struct block_writer *w, struct record rec)
++int block_writer_add(struct block_writer *w, struct reftable_record *rec)
+{
+ struct slice empty = { 0 };
+ struct slice last = w->entries % w->restart_interval == 0 ? empty :
@@ reftable/block.c (new)
+ struct slice key = { 0 };
+ int n = 0;
+
-+ record_key(rec, &key);
-+ n = encode_key(&restart, out, last, key, record_val_type(rec));
++ reftable_record_key(rec, &key);
++ n = reftable_encode_key(&restart, out, last, key,
++ reftable_record_val_type(rec));
+ if (n < 0) {
+ goto err;
+ }
+ slice_consume(&out, n);
+
-+ n = record_encode(rec, out, w->hash_size);
++ n = reftable_record_encode(rec, out, w->hash_size);
+ if (n < 0) {
+ goto err;
+ }
@@ reftable/block.c (new)
+ goto err;
+ }
+
-+ slice_clear(&key);
++ slice_release(&key);
+ return 0;
+
+err:
-+ slice_clear(&key);
++ slice_release(&key);
+ return -1;
+}
+
@@ reftable/block.c (new)
+ }
+
+ if (Z_OK != zresult) {
-+ slice_clear(&compressed);
++ slice_release(&compressed);
+ return REFTABLE_ZLIB_ERROR;
+ }
+
+ memcpy(w->buf + block_header_skip, compressed.buf,
+ dest_len);
+ w->next = dest_len + block_header_skip;
-+ slice_clear(&compressed);
++ slice_release(&compressed);
+ break;
+ }
+ }
@@ reftable/block.c (new)
+ byte typ = block->data[header_off];
+ uint32_t sz = get_be24(block->data + header_off + 1);
+
-+ if (!is_block_type(typ)) {
++ if (!reftable_is_block_type(typ)) {
+ return REFTABLE_FORMAT_ERROR;
+ }
+
@@ reftable/block.c (new)
+ uncompressed.buf + block_header_skip,
+ &dst_len, block->data + block_header_skip,
+ &src_len)) {
-+ slice_clear(&uncompressed);
++ slice_release(&uncompressed);
+ return REFTABLE_ZLIB_ERROR;
+ }
+
@@ reftable/block.c (new)
+ struct slice rkey = { 0 };
+ struct slice last_key = { 0 };
+ byte unused_extra;
-+ int n = decode_key(&rkey, &unused_extra, last_key, in);
++ int n = reftable_decode_key(&rkey, &unused_extra, last_key, in);
+ if (n < 0) {
+ a->error = 1;
+ return -1;
+ }
+
+ {
-+ int result = slice_compare(a->key, rkey);
-+ slice_clear(&rkey);
++ int result = slice_cmp(a->key, rkey);
++ slice_release(&rkey);
+ return result;
+ }
+}
@@ reftable/block.c (new)
+ slice_copy(&dest->last_key, src->last_key);
+}
+
-+int block_iter_next(struct block_iter *it, struct record rec)
++int block_iter_next(struct block_iter *it, struct reftable_record *rec)
+{
+ struct slice in = {
+ .buf = it->br->block.data + it->next_off,
@@ reftable/block.c (new)
+ return 1;
+ }
+
-+ n = decode_key(&key, &extra, it->last_key, in);
++ n = reftable_decode_key(&key, &extra, it->last_key, in);
+ if (n < 0) {
+ return -1;
+ }
+
+ slice_consume(&in, n);
-+ n = record_decode(rec, key, extra, in, it->br->hash_size);
++ n = reftable_record_decode(rec, key, extra, in, it->br->hash_size);
+ if (n < 0) {
+ return -1;
+ }
@@ reftable/block.c (new)
+
+ slice_copy(&it->last_key, key);
+ it->next_off += start.len - in.len;
-+ slice_clear(&key);
++ slice_release(&key);
+ return 0;
+}
+
@@ reftable/block.c (new)
+ };
+
+ byte extra = 0;
-+ int n = decode_key(key, &extra, empty, in);
++ int n = reftable_decode_key(key, &extra, empty, in);
+ if (n < 0) {
+ return n;
+ }
@@ reftable/block.c (new)
+
+void block_iter_close(struct block_iter *it)
+{
-+ slice_clear(&it->last_key);
++ slice_release(&it->last_key);
+}
+
+int block_reader_seek(struct block_reader *br, struct block_iter *it,
@@ reftable/block.c (new)
+ .key = want,
+ .r = br,
+ };
-+ struct record rec = new_record(block_reader_type(br));
++ struct reftable_record rec = reftable_new_record(block_reader_type(br));
+ struct slice key = { 0 };
+ int err = 0;
+ struct block_iter next = { 0 };
@@ reftable/block.c (new)
+ */
+ while (true) {
+ block_iter_copy_from(&next, it);
-+ err = block_iter_next(&next, rec);
++ err = block_iter_next(&next, &rec);
+ if (err < 0) {
+ goto exit;
+ }
+
-+ record_key(rec, &key);
-+ if (err > 0 || slice_compare(key, want) >= 0) {
++ reftable_record_key(&rec, &key);
++ if (err > 0 || slice_cmp(key, want) >= 0) {
+ err = 0;
+ goto exit;
+ }
@@ reftable/block.c (new)
+ }
+
+exit:
-+ slice_clear(&key);
-+ slice_clear(&next.last_key);
-+ record_destroy(&rec);
++ slice_release(&key);
++ slice_release(&next.last_key);
++ reftable_record_destroy(&rec);
+
+ return err;
+}
@@ reftable/block.c (new)
+void block_writer_clear(struct block_writer *bw)
+{
+ FREE_AND_NULL(bw->restarts);
-+ slice_clear(&bw->last_key);
++ slice_release(&bw->last_key);
+ /* the block is not owned. */
+}
@@ reftable/block.h (new)
+byte block_writer_type(struct block_writer *bw);
+
+/* appends the record, or -1 if it doesn't fit. */
-+int block_writer_add(struct block_writer *w, struct record rec);
++int block_writer_add(struct block_writer *w, struct reftable_record *rec);
+
+/* appends the key restarts, and compress the block if necessary. */
+int block_writer_finish(struct block_writer *w);
@@ reftable/block.h (new)
+void block_iter_copy_from(struct block_iter *dest, struct block_iter *src);
+
+/* return < 0 for error, 0 for OK, > 0 for EOF. */
-+int block_iter_next(struct block_iter *it, struct record rec);
++int block_iter_next(struct block_iter *it, struct reftable_record *rec);
+
+/* Seek to `want` with in the block pointed to by `it` */
+int block_iter_seek(struct block_iter *it, struct slice want);
@@ reftable/file.c (new)
+ struct stat st = { 0 };
+ int err = 0;
+ int fd = open(name, O_RDONLY);
++ struct file_block_source *p = NULL;
+ if (fd < 0) {
+ if (errno == ENOENT) {
+ return REFTABLE_NOT_EXIST_ERROR;
@@ reftable/file.c (new)
+ return -1;
+ }
+
-+ {
-+ struct file_block_source *p =
-+ reftable_calloc(sizeof(struct file_block_source));
-+ p->size = st.st_size;
-+ p->fd = fd;
++ p = reftable_calloc(sizeof(struct file_block_source));
++ p->size = st.st_size;
++ p->fd = fd;
+
-+ assert(bs->ops == NULL);
-+ bs->ops = &file_vtable;
-+ bs->arg = p;
-+ }
++ assert(bs->ops == NULL);
++ bs->ops = &file_vtable;
++ bs->arg = p;
+ return 0;
+}
+
@@ reftable/iter.c (new)
+#include "reader.h"
+#include "reftable.h"
+
-+bool iterator_is_null(struct reftable_iterator it)
++bool iterator_is_null(struct reftable_iterator *it)
+{
-+ return it.ops == NULL;
++ return it->ops == NULL;
+}
+
-+static int empty_iterator_next(void *arg, struct record rec)
++static int empty_iterator_next(void *arg, struct reftable_record *rec)
+{
+ return 1;
+}
@@ reftable/iter.c (new)
+ it->ops = &empty_vtable;
+}
+
-+int iterator_next(struct reftable_iterator it, struct record rec)
++int iterator_next(struct reftable_iterator *it, struct reftable_record *rec)
+{
-+ return it.ops->next(it.iter_arg, rec);
++ return it->ops->next(it->iter_arg, rec);
+}
+
+void reftable_iterator_destroy(struct reftable_iterator *it)
@@ reftable/iter.c (new)
+ FREE_AND_NULL(it->iter_arg);
+}
+
-+int reftable_iterator_next_ref(struct reftable_iterator it,
++int reftable_iterator_next_ref(struct reftable_iterator *it,
+ struct reftable_ref_record *ref)
+{
-+ struct record rec = { 0 };
-+ record_from_ref(&rec, ref);
-+ return iterator_next(it, rec);
++ struct reftable_record rec = { 0 };
++ reftable_record_from_ref(&rec, ref);
++ return iterator_next(it, &rec);
+}
+
-+int reftable_iterator_next_log(struct reftable_iterator it,
++int reftable_iterator_next_log(struct reftable_iterator *it,
+ struct reftable_log_record *log)
+{
-+ struct record rec = { 0 };
-+ record_from_log(&rec, log);
-+ return iterator_next(it, rec);
++ struct reftable_record rec = { 0 };
++ reftable_record_from_log(&rec, log);
++ return iterator_next(it, &rec);
+}
+
+static void filtering_ref_iterator_close(void *iter_arg)
+{
+ struct filtering_ref_iterator *fri =
+ (struct filtering_ref_iterator *)iter_arg;
-+ slice_clear(&fri->oid);
++ slice_release(&fri->oid);
+ reftable_iterator_destroy(&fri->it);
+}
+
-+static int filtering_ref_iterator_next(void *iter_arg, struct record rec)
++static int filtering_ref_iterator_next(void *iter_arg,
++ struct reftable_record *rec)
+{
+ struct filtering_ref_iterator *fri =
+ (struct filtering_ref_iterator *)iter_arg;
+ struct reftable_ref_record *ref =
-+ (struct reftable_ref_record *)rec.data;
++ (struct reftable_ref_record *)rec->data;
+ int err = 0;
+ while (true) {
-+ err = reftable_iterator_next_ref(fri->it, ref);
++ err = reftable_iterator_next_ref(&fri->it, ref);
+ if (err != 0) {
+ break;
+ }
@@ reftable/iter.c (new)
+ if (fri->double_check) {
+ struct reftable_iterator it = { 0 };
+
-+ err = reftable_table_seek_ref(fri->tab, &it,
++ err = reftable_table_seek_ref(&fri->tab, &it,
+ ref->ref_name);
+ if (err == 0) {
-+ err = reftable_iterator_next_ref(it, ref);
++ err = reftable_iterator_next_ref(&it, ref);
+ }
+
+ reftable_iterator_destroy(&it);
@@ reftable/iter.c (new)
+ struct indexed_table_ref_iter *it = (struct indexed_table_ref_iter *)p;
+ block_iter_close(&it->cur);
+ reftable_block_done(&it->block_reader.block);
-+ slice_clear(&it->oid);
++ slice_release(&it->oid);
+}
+
+static int indexed_table_ref_iter_next_block(struct indexed_table_ref_iter *it)
@@ reftable/iter.c (new)
+ return 0;
+}
+
-+static int indexed_table_ref_iter_next(void *p, struct record rec)
++static int indexed_table_ref_iter_next(void *p, struct reftable_record *rec)
+{
+ struct indexed_table_ref_iter *it = (struct indexed_table_ref_iter *)p;
+ struct reftable_ref_record *ref =
-+ (struct reftable_ref_record *)rec.data;
++ (struct reftable_ref_record *)rec->data;
+
+ while (true) {
+ int err = block_iter_next(&it->cur, rec);
@@ reftable/iter.h (new)
+#include "slice.h"
+
+struct reftable_iterator_vtable {
-+ int (*next)(void *iter_arg, struct record rec);
++ int (*next)(void *iter_arg, struct reftable_record *rec);
+ void (*close)(void *iter_arg);
+};
+
+void iterator_set_empty(struct reftable_iterator *it);
-+int iterator_next(struct reftable_iterator it, struct record rec);
++int iterator_next(struct reftable_iterator *it, struct reftable_record *rec);
+
+/* Returns true for a zeroed out iterator, such as the one returned from
+ iterator_destroy. */
-+bool iterator_is_null(struct reftable_iterator it);
++bool iterator_is_null(struct reftable_iterator *it);
+
+/* iterator that produces only ref records that point to `oid` */
+struct filtering_ref_iterator {
@@ reftable/merged.c (new)
+{
+ int i = 0;
+ for (i = 0; i < mi->stack_len; i++) {
-+ struct record rec = new_record(mi->typ);
-+ int err = iterator_next(mi->stack[i], rec);
++ struct reftable_record rec = reftable_new_record(mi->typ);
++ int err = iterator_next(&mi->stack[i], &rec);
+ if (err < 0) {
+ return err;
+ }
+
+ if (err > 0) {
+ reftable_iterator_destroy(&mi->stack[i]);
-+ record_destroy(&rec);
++ reftable_record_destroy(&rec);
+ } else {
+ struct pq_entry e = {
+ .rec = rec,
@@ reftable/merged.c (new)
+ reftable_free(mi->stack);
+}
+
-+static int merged_iter_advance_subiter(struct merged_iter *mi, size_t idx)
++static int merged_iter_advance_nonnull_subiter(struct merged_iter *mi,
++ size_t idx)
+{
-+ if (iterator_is_null(mi->stack[idx])) {
-+ return 0;
++ struct reftable_record rec = reftable_new_record(mi->typ);
++ struct pq_entry e = {
++ .rec = rec,
++ .index = idx,
++ };
++ int err = iterator_next(&mi->stack[idx], &rec);
++ if (err < 0) {
++ return err;
+ }
+
-+ {
-+ struct record rec = new_record(mi->typ);
-+ struct pq_entry e = {
-+ .rec = rec,
-+ .index = idx,
-+ };
-+ int err = iterator_next(mi->stack[idx], rec);
-+ if (err < 0) {
-+ return err;
-+ }
++ if (err > 0) {
++ reftable_iterator_destroy(&mi->stack[idx]);
++ reftable_record_destroy(&rec);
++ return 0;
++ }
+
-+ if (err > 0) {
-+ reftable_iterator_destroy(&mi->stack[idx]);
-+ record_destroy(&rec);
-+ return 0;
-+ }
++ merged_iter_pqueue_add(&mi->pq, e);
++ return 0;
++}
+
-+ merged_iter_pqueue_add(&mi->pq, e);
++static int merged_iter_advance_subiter(struct merged_iter *mi, size_t idx)
++{
++ if (iterator_is_null(&mi->stack[idx])) {
++ return 0;
+ }
-+ return 0;
++ return merged_iter_advance_nonnull_subiter(mi, idx);
+}
+
-+static int merged_iter_next_entry(struct merged_iter *mi, struct record rec)
++static int merged_iter_next_entry(struct merged_iter *mi,
++ struct reftable_record *rec)
+{
+ struct slice entry_key = { 0 };
+ struct pq_entry entry = { 0 };
@@ reftable/merged.c (new)
+ such a deployment, the loop below must be changed to collect all
+ entries for the same key, and return new the newest one.
+ */
-+ record_key(entry.rec, &entry_key);
++ reftable_record_key(&entry.rec, &entry_key);
+ while (!merged_iter_pqueue_is_empty(mi->pq)) {
+ struct pq_entry top = merged_iter_pqueue_top(mi->pq);
+ struct slice k = { 0 };
+ int err = 0, cmp = 0;
+
-+ record_key(top.rec, &k);
++ reftable_record_key(&top.rec, &k);
+
-+ cmp = slice_compare(k, entry_key);
-+ slice_clear(&k);
++ cmp = slice_cmp(k, entry_key);
++ slice_release(&k);
+
+ if (cmp > 0) {
+ break;
@@ reftable/merged.c (new)
+ if (err < 0) {
+ return err;
+ }
-+ record_destroy(&top.rec);
++ reftable_record_destroy(&top.rec);
+ }
+
-+ record_copy_from(rec, entry.rec, hash_size(mi->hash_id));
-+ record_destroy(&entry.rec);
-+ slice_clear(&entry_key);
++ reftable_record_copy_from(rec, &entry.rec, hash_size(mi->hash_id));
++ reftable_record_destroy(&entry.rec);
++ slice_release(&entry_key);
+ return 0;
+}
+
-+static int merged_iter_next(struct merged_iter *mi, struct record rec)
++static int merged_iter_next(struct merged_iter *mi, struct reftable_record *rec)
+{
+ while (true) {
+ int err = merged_iter_next_entry(mi, rec);
+ if (err == 0 && mi->suppress_deletions &&
-+ record_is_deletion(rec)) {
++ reftable_record_is_deletion(rec)) {
+ continue;
+ }
+
@@ reftable/merged.c (new)
+ }
+}
+
-+static int merged_iter_next_void(void *p, struct record rec)
++static int merged_iter_next_void(void *p, struct reftable_record *rec)
+{
+ struct merged_iter *mi = (struct merged_iter *)p;
+ if (merged_iter_pqueue_is_empty(mi->pq)) {
@@ reftable/merged.c (new)
+ struct reftable_reader **stack, int n,
+ uint32_t hash_id)
+{
++ struct reftable_merged_table *m = NULL;
+ uint64_t last_max = 0;
+ uint64_t first_min = 0;
+ int i = 0;
@@ reftable/merged.c (new)
+ last_max = reftable_reader_max_update_index(r);
+ }
+
-+ {
-+ struct reftable_merged_table m = {
-+ .stack = stack,
-+ .stack_len = n,
-+ .min = first_min,
-+ .max = last_max,
-+ .hash_id = hash_id,
-+ };
-+
-+ *dest = reftable_calloc(sizeof(struct reftable_merged_table));
-+ **dest = m;
-+ }
++ m = (struct reftable_merged_table *)reftable_calloc(
++ sizeof(struct reftable_merged_table));
++ m->stack = stack;
++ m->stack_len = n;
++ m->min = first_min;
++ m->max = last_max;
++ m->hash_id = hash_id;
++ *dest = m;
+ return 0;
+}
+
@@ reftable/merged.c (new)
+}
+
+int merged_table_seek_record(struct reftable_merged_table *mt,
-+ struct reftable_iterator *it, struct record rec)
++ struct reftable_iterator *it,
++ struct reftable_record *rec)
+{
+ struct reftable_iterator *iters = reftable_calloc(
+ sizeof(struct reftable_iterator) * mt->stack_len);
+ struct merged_iter merged = {
+ .stack = iters,
-+ .typ = record_type(rec),
++ .typ = reftable_record_type(rec),
+ .hash_id = mt->hash_id,
+ .suppress_deletions = mt->suppress_deletions,
+ };
@@ reftable/merged.c (new)
+ struct reftable_ref_record ref = {
+ .ref_name = (char *)name,
+ };
-+ struct record rec = { 0 };
-+ record_from_ref(&rec, &ref);
-+ return merged_table_seek_record(mt, it, rec);
++ struct reftable_record rec = { 0 };
++ reftable_record_from_ref(&rec, &ref);
++ return merged_table_seek_record(mt, it, &rec);
+}
+
+int reftable_merged_table_seek_log_at(struct reftable_merged_table *mt,
@@ reftable/merged.c (new)
+ .ref_name = (char *)name,
+ .update_index = update_index,
+ };
-+ struct record rec = { 0 };
-+ record_from_log(&rec, &log);
-+ return merged_table_seek_record(mt, it, rec);
++ struct reftable_record rec = { 0 };
++ reftable_record_from_log(&rec, &log);
++ return merged_table_seek_record(mt, it, &rec);
+}
+
+int reftable_merged_table_seek_log(struct reftable_merged_table *mt,
@@ reftable/merged.h (new)
+
+void merged_table_clear(struct reftable_merged_table *mt);
+int merged_table_seek_record(struct reftable_merged_table *mt,
-+ struct reftable_iterator *it, struct record rec);
++ struct reftable_iterator *it,
++ struct reftable_record *rec);
+
+#endif
@@ reftable/pq.c (new)
+ struct slice ak = { 0 };
+ struct slice bk = { 0 };
+ int cmp = 0;
-+ record_key(a.rec, &ak);
-+ record_key(b.rec, &bk);
++ reftable_record_key(&a.rec, &ak);
++ reftable_record_key(&b.rec, &bk);
+
-+ cmp = slice_compare(ak, bk);
++ cmp = slice_cmp(ak, bk);
+
-+ slice_clear(&ak);
-+ slice_clear(&bk);
++ slice_release(&ak);
++ slice_release(&bk);
+
+ if (cmp == 0) {
+ return a.index > b.index;
@@ reftable/pq.c (new)
+{
+ int i = 0;
+ for (i = 0; i < pq->len; i++) {
-+ record_destroy(&pq->heap[i].rec);
++ reftable_record_destroy(&pq->heap[i].rec);
+ }
+ FREE_AND_NULL(pq->heap);
+ pq->len = pq->cap = 0;
@@ reftable/pq.h (new)
+
+struct pq_entry {
+ int index;
-+ struct record rec;
++ struct reftable_record rec;
+};
+
+int pq_less(struct pq_entry a, struct pq_entry b);
@@ reftable/reader.c (new)
+#include "reftable.h"
+#include "tree.h"
+
-+uint64_t block_source_size(struct reftable_block_source source)
++uint64_t block_source_size(struct reftable_block_source *source)
+{
-+ return source.ops->size(source.arg);
++ return source->ops->size(source->arg);
+}
+
-+int block_source_read_block(struct reftable_block_source source,
++int block_source_read_block(struct reftable_block_source *source,
+ struct reftable_block *dest, uint64_t off,
+ uint32_t size)
+{
-+ int result = source.ops->read_block(source.arg, dest, off, size);
-+ dest->source = source;
++ int result = source->ops->read_block(source->arg, dest, off, size);
++ dest->source = *source;
+ return result;
+}
+
@@ reftable/reader.c (new)
+ sz = r->size - off;
+ }
+
-+ return block_source_read_block(r->source, dest, off, sz);
++ return block_source_read_block(&r->source, dest, off, sz);
+}
+
+uint32_t reftable_reader_hash_id(struct reftable_reader *r)
@@ reftable/reader.c (new)
+static int parse_footer(struct reftable_reader *r, byte *footer, byte *header)
+{
+ byte *f = footer;
++ byte first_block_typ;
+ int err = 0;
++ uint32_t computed_crc;
++ uint32_t file_crc;
++
+ if (memcmp(f, "REFT", 4)) {
+ err = REFTABLE_FORMAT_ERROR;
+ goto exit;
@@ reftable/reader.c (new)
+ r->log_offsets.index_offset = get_be64(f);
+ f += 8;
+
-+ {
-+ uint32_t computed_crc = crc32(0, footer, f - footer);
-+ uint32_t file_crc = get_be32(f);
-+ f += 4;
-+ if (computed_crc != file_crc) {
-+ err = REFTABLE_FORMAT_ERROR;
-+ goto exit;
-+ }
++ computed_crc = crc32(0, footer, f - footer);
++ file_crc = get_be32(f);
++ f += 4;
++ if (computed_crc != file_crc) {
++ err = REFTABLE_FORMAT_ERROR;
++ goto exit;
+ }
+
-+ {
-+ byte first_block_typ = header[header_size(r->version)];
-+ r->ref_offsets.present = (first_block_typ == BLOCK_TYPE_REF);
-+ r->ref_offsets.offset = 0;
-+ r->log_offsets.present = (first_block_typ == BLOCK_TYPE_LOG ||
-+ r->log_offsets.offset > 0);
-+ r->obj_offsets.present = r->obj_offsets.offset > 0;
-+ }
++ first_block_typ = header[header_size(r->version)];
++ r->ref_offsets.present = (first_block_typ == BLOCK_TYPE_REF);
++ r->ref_offsets.offset = 0;
++ r->log_offsets.present = (first_block_typ == BLOCK_TYPE_LOG ||
++ r->log_offsets.offset > 0);
++ r->obj_offsets.present = r->obj_offsets.offset > 0;
+ err = 0;
+exit:
+ return err;
+}
+
-+int init_reader(struct reftable_reader *r, struct reftable_block_source source,
++int init_reader(struct reftable_reader *r, struct reftable_block_source *source,
+ const char *name)
+{
+ struct reftable_block footer = { 0 };
@@ reftable/reader.c (new)
+ }
+
+ r->size = block_source_size(source) - footer_size(r->version);
-+ r->source = source;
++ r->source = *source;
+ r->name = xstrdup(name);
+ r->hash_id = 0;
+
@@ reftable/reader.c (new)
+ block_iter_copy_from(&dest->bi, &src->bi);
+}
+
-+static int table_iter_next_in_block(struct table_iter *ti, struct record rec)
++static int table_iter_next_in_block(struct table_iter *ti,
++ struct reftable_record *rec)
+{
+ int res = block_iter_next(&ti->bi, rec);
-+ if (res == 0 && record_type(rec) == BLOCK_TYPE_REF) {
-+ ((struct reftable_ref_record *)rec.data)->update_index +=
++ if (res == 0 && reftable_record_type(rec) == BLOCK_TYPE_REF) {
++ ((struct reftable_ref_record *)rec->data)->update_index +=
+ ti->r->min_update_index;
+ }
+
@@ reftable/reader.c (new)
+ }
+
+ *typ = data[0];
-+ if (is_block_type(*typ)) {
++ if (reftable_is_block_type(*typ)) {
+ result = get_be24(data + 1);
+ }
+ return result;
@@ reftable/reader.c (new)
+ return 0;
+}
+
-+static int table_iter_next(struct table_iter *ti, struct record rec)
++static int table_iter_next(struct table_iter *ti, struct reftable_record *rec)
+{
-+ if (record_type(rec) != ti->typ) {
++ if (reftable_record_type(rec) != ti->typ) {
+ return REFTABLE_API_ERROR;
+ }
+
@@ reftable/reader.c (new)
+ }
+}
+
-+static int table_iter_next_void(void *ti, struct record rec)
++static int table_iter_next_void(void *ti, struct reftable_record *rec)
+{
+ return table_iter_next((struct table_iter *)ti, rec);
+}
@@ reftable/reader.c (new)
+}
+
+static int reader_seek_linear(struct reftable_reader *r, struct table_iter *ti,
-+ struct record want)
++ struct reftable_record *want)
+{
-+ struct record rec = new_record(record_type(want));
++ struct reftable_record rec =
++ reftable_new_record(reftable_record_type(want));
+ struct slice want_key = { 0 };
+ struct slice got_key = { 0 };
+ struct table_iter next = { 0 };
+ int err = -1;
-+ record_key(want, &want_key);
++ reftable_record_key(want, &want_key);
+
+ while (true) {
+ err = table_iter_next_block(&next, ti);
@@ reftable/reader.c (new)
+ goto exit;
+ }
+ {
-+ int cmp = slice_compare(got_key, want_key);
++ int cmp = slice_cmp(got_key, want_key);
+ if (cmp > 0) {
+ table_iter_block_done(&next);
+ break;
@@ reftable/reader.c (new)
+
+exit:
+ block_iter_close(&next.bi);
-+ record_destroy(&rec);
-+ slice_clear(&want_key);
-+ slice_clear(&got_key);
++ reftable_record_destroy(&rec);
++ slice_release(&want_key);
++ slice_release(&got_key);
+ return err;
+}
+
+static int reader_seek_indexed(struct reftable_reader *r,
-+ struct reftable_iterator *it, struct record rec)
++ struct reftable_iterator *it,
++ struct reftable_record *rec)
+{
-+ struct index_record want_index = { 0 };
-+ struct record want_index_rec = { 0 };
-+ struct index_record index_result = { 0 };
-+ struct record index_result_rec = { 0 };
++ struct reftable_index_record want_index = { 0 };
++ struct reftable_record want_index_rec = { 0 };
++ struct reftable_index_record index_result = { 0 };
++ struct reftable_record index_result_rec = { 0 };
+ struct table_iter index_iter = { 0 };
+ struct table_iter next = { 0 };
+ int err = 0;
+
-+ record_key(rec, &want_index.last_key);
-+ record_from_index(&want_index_rec, &want_index);
-+ record_from_index(&index_result_rec, &index_result);
++ reftable_record_key(rec, &want_index.last_key);
++ reftable_record_from_index(&want_index_rec, &want_index);
++ reftable_record_from_index(&index_result_rec, &index_result);
+
-+ err = reader_start(r, &index_iter, record_type(rec), true);
++ err = reader_start(r, &index_iter, reftable_record_type(rec), true);
+ if (err < 0) {
+ goto exit;
+ }
+
-+ err = reader_seek_linear(r, &index_iter, want_index_rec);
++ err = reader_seek_linear(r, &index_iter, &want_index_rec);
+ while (true) {
-+ err = table_iter_next(&index_iter, index_result_rec);
++ err = table_iter_next(&index_iter, &index_result_rec);
+ table_iter_block_done(&index_iter);
+ if (err != 0) {
+ goto exit;
@@ reftable/reader.c (new)
+ goto exit;
+ }
+
-+ if (next.typ == record_type(rec)) {
++ if (next.typ == reftable_record_type(rec)) {
+ err = 0;
+ break;
+ }
@@ reftable/reader.c (new)
+exit:
+ block_iter_close(&next.bi);
+ table_iter_close(&index_iter);
-+ record_clear(want_index_rec);
-+ record_clear(index_result_rec);
++ reftable_record_clear(&want_index_rec);
++ reftable_record_clear(&index_result_rec);
+ return err;
+}
+
+static int reader_seek_internal(struct reftable_reader *r,
-+ struct reftable_iterator *it, struct record rec)
++ struct reftable_iterator *it,
++ struct reftable_record *rec)
+{
+ struct reftable_reader_offsets *offs =
-+ reader_offsets_for(r, record_type(rec));
++ reader_offsets_for(r, reftable_record_type(rec));
+ uint64_t idx = offs->index_offset;
+ struct table_iter ti = { 0 };
+ int err = 0;
@@ reftable/reader.c (new)
+ return reader_seek_indexed(r, it, rec);
+ }
+
-+ err = reader_start(r, &ti, record_type(rec), false);
++ err = reader_start(r, &ti, reftable_record_type(rec), false);
+ if (err < 0) {
+ return err;
+ }
@@ reftable/reader.c (new)
+}
+
+int reader_seek(struct reftable_reader *r, struct reftable_iterator *it,
-+ struct record rec)
++ struct reftable_record *rec)
+{
-+ byte typ = record_type(rec);
++ byte typ = reftable_record_type(rec);
+
+ struct reftable_reader_offsets *offs = reader_offsets_for(r, typ);
+ if (!offs->present) {
@@ reftable/reader.c (new)
+ struct reftable_ref_record ref = {
+ .ref_name = (char *)name,
+ };
-+ struct record rec = { 0 };
-+ record_from_ref(&rec, &ref);
-+ return reader_seek(r, it, rec);
++ struct reftable_record rec = { 0 };
++ reftable_record_from_ref(&rec, &ref);
++ return reader_seek(r, it, &rec);
+}
+
+int reftable_reader_seek_log_at(struct reftable_reader *r,
@@ reftable/reader.c (new)
+ .ref_name = (char *)name,
+ .update_index = update_index,
+ };
-+ struct record rec = { 0 };
-+ record_from_log(&rec, &log);
-+ return reader_seek(r, it, rec);
++ struct reftable_record rec = { 0 };
++ reftable_record_from_log(&rec, &log);
++ return reader_seek(r, it, &rec);
+}
+
+int reftable_reader_seek_log(struct reftable_reader *r,
@@ reftable/reader.c (new)
+}
+
+int reftable_new_reader(struct reftable_reader **p,
-+ struct reftable_block_source src, char const *name)
++ struct reftable_block_source *src, char const *name)
+{
+ struct reftable_reader *rd =
+ reftable_calloc(sizeof(struct reftable_reader));
@@ reftable/reader.c (new)
+ if (err == 0) {
+ *p = rd;
+ } else {
-+ block_source_close(&src);
++ block_source_close(src);
+ reftable_free(rd);
+ }
+ return err;
@@ reftable/reader.c (new)
+ struct reftable_iterator *it,
+ byte *oid)
+{
-+ struct obj_record want = {
++ struct reftable_obj_record want = {
+ .hash_prefix = oid,
+ .hash_prefix_len = r->object_id_len,
+ };
-+ struct record want_rec = { 0 };
++ struct reftable_record want_rec = { 0 };
+ struct reftable_iterator oit = { 0 };
-+ struct obj_record got = { 0 };
-+ struct record got_rec = { 0 };
++ struct reftable_obj_record got = { 0 };
++ struct reftable_record got_rec = { 0 };
+ int err = 0;
++ struct indexed_table_ref_iter *itr = NULL;
+
+ /* Look through the reverse index. */
-+ record_from_obj(&want_rec, &want);
-+ err = reader_seek(r, &oit, want_rec);
++ reftable_record_from_obj(&want_rec, &want);
++ err = reader_seek(r, &oit, &want_rec);
+ if (err != 0) {
+ goto exit;
+ }
+
-+ /* read out the obj_record */
-+ record_from_obj(&got_rec, &got);
-+ err = iterator_next(oit, got_rec);
++ /* read out the reftable_obj_record */
++ reftable_record_from_obj(&got_rec, &got);
++ err = iterator_next(&oit, &got_rec);
+ if (err < 0) {
+ goto exit;
+ }
@@ reftable/reader.c (new)
+ goto exit;
+ }
+
-+ {
-+ struct indexed_table_ref_iter *itr = NULL;
-+ err = new_indexed_table_ref_iter(&itr, r, oid,
-+ hash_size(r->hash_id),
-+ got.offsets, got.offset_len);
-+ if (err < 0) {
-+ goto exit;
-+ }
-+ got.offsets = NULL;
-+ iterator_from_indexed_table_ref_iter(it, itr);
++ err = new_indexed_table_ref_iter(&itr, r, oid, hash_size(r->hash_id),
++ got.offsets, got.offset_len);
++ if (err < 0) {
++ goto exit;
+ }
++ got.offsets = NULL;
++ iterator_from_indexed_table_ref_iter(it, itr);
+
+exit:
+ reftable_iterator_destroy(&oit);
-+ record_clear(got_rec);
++ reftable_record_clear(&got_rec);
+ return err;
+}
+
@@ reftable/reader.h (new)
+#include "record.h"
+#include "reftable.h"
+
-+uint64_t block_source_size(struct reftable_block_source source);
++uint64_t block_source_size(struct reftable_block_source *source);
+
-+int block_source_read_block(struct reftable_block_source source,
++int block_source_read_block(struct reftable_block_source *source,
+ struct reftable_block *dest, uint64_t off,
+ uint32_t size);
+void block_source_close(struct reftable_block_source *source);
@@ reftable/reader.h (new)
+ struct reftable_reader_offsets log_offsets;
+};
+
-+int init_reader(struct reftable_reader *r, struct reftable_block_source source,
++int init_reader(struct reftable_reader *r, struct reftable_block_source *source,
+ const char *name);
+int reader_seek(struct reftable_reader *r, struct reftable_iterator *it,
-+ struct record rec);
++ struct reftable_record *rec);
+void reader_close(struct reftable_reader *r);
+const char *reader_name(struct reftable_reader *r);
+
@@ reftable/record.c (new)
+{
+ byte buf[10] = { 0 };
+ int i = 9;
++ int n = 0;
+ buf[i] = (byte)(val & 0x7f);
+ i--;
+ while (true) {
@@ reftable/record.c (new)
+ i--;
+ }
+
-+ {
-+ int n = sizeof(buf) - i - 1;
-+ if (dest.len < n) {
-+ return -1;
-+ }
-+ memcpy(dest.buf, &buf[i + 1], n);
-+ return n;
++ n = sizeof(buf) - i - 1;
++ if (dest.len < n) {
++ return -1;
+ }
++ memcpy(dest.buf, &buf[i + 1], n);
++ return n;
+}
+
-+int is_block_type(byte typ)
++int reftable_is_block_type(byte typ)
+{
+ switch (typ) {
+ case BLOCK_TYPE_REF:
@@ reftable/record.c (new)
+ return start.len - s.len;
+}
+
-+int encode_key(bool *restart, struct slice dest, struct slice prev_key,
-+ struct slice key, byte extra)
++int reftable_encode_key(bool *restart, struct slice dest, struct slice prev_key,
++ struct slice key, byte extra)
+{
+ struct slice start = dest;
+ int prefix_len = common_prefix_size(prev_key, key);
@@ reftable/record.c (new)
+ return start.len - dest.len;
+}
+
-+int decode_key(struct slice *key, byte *extra, struct slice last_key,
-+ struct slice in)
++int reftable_decode_key(struct slice *key, byte *extra, struct slice last_key,
++ struct slice in)
+{
+ int start_len = in.len;
+ uint64_t prefix_len = 0;
@@ reftable/record.c (new)
+ (const struct reftable_ref_record *)p);
+}
+
-+struct record_vtable reftable_ref_record_vtable = {
++struct reftable_record_vtable reftable_ref_record_vtable = {
+ .key = &reftable_ref_record_key,
+ .type = BLOCK_TYPE_REF,
+ .copy_from = &reftable_ref_record_copy_from,
@@ reftable/record.c (new)
+ .is_deletion = &reftable_ref_record_is_deletion_void,
+};
+
-+static void obj_record_key(const void *r, struct slice *dest)
++static void reftable_obj_record_key(const void *r, struct slice *dest)
+{
-+ const struct obj_record *rec = (const struct obj_record *)r;
++ const struct reftable_obj_record *rec =
++ (const struct reftable_obj_record *)r;
+ slice_resize(dest, rec->hash_prefix_len);
+ memcpy(dest->buf, rec->hash_prefix, rec->hash_prefix_len);
+}
+
-+static void obj_record_clear(void *rec)
++static void reftable_obj_record_clear(void *rec)
+{
-+ struct obj_record *obj = (struct obj_record *)rec;
++ struct reftable_obj_record *obj = (struct reftable_obj_record *)rec;
+ FREE_AND_NULL(obj->hash_prefix);
+ FREE_AND_NULL(obj->offsets);
-+ memset(obj, 0, sizeof(struct obj_record));
++ memset(obj, 0, sizeof(struct reftable_obj_record));
+}
+
-+static void obj_record_copy_from(void *rec, const void *src_rec, int hash_size)
++static void reftable_obj_record_copy_from(void *rec, const void *src_rec,
++ int hash_size)
+{
-+ struct obj_record *obj = (struct obj_record *)rec;
-+ const struct obj_record *src = (const struct obj_record *)src_rec;
++ struct reftable_obj_record *obj = (struct reftable_obj_record *)rec;
++ const struct reftable_obj_record *src =
++ (const struct reftable_obj_record *)src_rec;
+
-+ obj_record_clear(obj);
++ reftable_obj_record_clear(obj);
+ *obj = *src;
+ obj->hash_prefix = reftable_malloc(obj->hash_prefix_len);
+ memcpy(obj->hash_prefix, src->hash_prefix, obj->hash_prefix_len);
@@ reftable/record.c (new)
+ }
+}
+
-+static byte obj_record_val_type(const void *rec)
++static byte reftable_obj_record_val_type(const void *rec)
+{
-+ struct obj_record *r = (struct obj_record *)rec;
++ struct reftable_obj_record *r = (struct reftable_obj_record *)rec;
+ if (r->offset_len > 0 && r->offset_len < 8) {
+ return r->offset_len;
+ }
+ return 0;
+}
+
-+static int obj_record_encode(const void *rec, struct slice s, int hash_size)
++static int reftable_obj_record_encode(const void *rec, struct slice s,
++ int hash_size)
+{
-+ struct obj_record *r = (struct obj_record *)rec;
++ struct reftable_obj_record *r = (struct reftable_obj_record *)rec;
+ struct slice start = s;
++ int i = 0;
+ int n = 0;
++ uint64_t last = 0;
+ if (r->offset_len == 0 || r->offset_len >= 8) {
+ n = put_var_int(s, r->offset_len);
+ if (n < 0) {
@@ reftable/record.c (new)
+ }
+ slice_consume(&s, n);
+
-+ {
-+ uint64_t last = r->offsets[0];
-+ int i = 0;
-+ for (i = 1; i < r->offset_len; i++) {
-+ int n = put_var_int(s, r->offsets[i] - last);
-+ if (n < 0) {
-+ return -1;
-+ }
-+ slice_consume(&s, n);
-+ last = r->offsets[i];
++ last = r->offsets[0];
++ for (i = 1; i < r->offset_len; i++) {
++ int n = put_var_int(s, r->offsets[i] - last);
++ if (n < 0) {
++ return -1;
+ }
++ slice_consume(&s, n);
++ last = r->offsets[i];
+ }
+ return start.len - s.len;
+}
+
-+static int obj_record_decode(void *rec, struct slice key, byte val_type,
-+ struct slice in, int hash_size)
++static int reftable_obj_record_decode(void *rec, struct slice key,
++ byte val_type, struct slice in,
++ int hash_size)
+{
+ struct slice start = in;
-+ struct obj_record *r = (struct obj_record *)rec;
++ struct reftable_obj_record *r = (struct reftable_obj_record *)rec;
+ uint64_t count = val_type;
+ int n = 0;
++ uint64_t last;
++ int j;
+ r->hash_prefix = reftable_malloc(key.len);
+ memcpy(r->hash_prefix, key.buf, key.len);
+ r->hash_prefix_len = key.len;
@@ reftable/record.c (new)
+ }
+ slice_consume(&in, n);
+
-+ {
-+ uint64_t last = r->offsets[0];
-+ int j = 1;
-+ while (j < count) {
-+ uint64_t delta = 0;
-+ int n = get_var_int(&delta, in);
-+ if (n < 0) {
-+ return n;
-+ }
-+ slice_consume(&in, n);
-+
-+ last = r->offsets[j] = (delta + last);
-+ j++;
++ last = r->offsets[0];
++ j = 1;
++ while (j < count) {
++ uint64_t delta = 0;
++ int n = get_var_int(&delta, in);
++ if (n < 0) {
++ return n;
+ }
++ slice_consume(&in, n);
++
++ last = r->offsets[j] = (delta + last);
++ j++;
+ }
+ return start.len - in.len;
+}
@@ reftable/record.c (new)
+ return false;
+}
+
-+struct record_vtable obj_record_vtable = {
-+ .key = &obj_record_key,
++struct reftable_record_vtable reftable_obj_record_vtable = {
++ .key = &reftable_obj_record_key,
+ .type = BLOCK_TYPE_OBJ,
-+ .copy_from = &obj_record_copy_from,
-+ .val_type = &obj_record_val_type,
-+ .encode = &obj_record_encode,
-+ .decode = &obj_record_decode,
-+ .clear = &obj_record_clear,
++ .copy_from = &reftable_obj_record_copy_from,
++ .val_type = &reftable_obj_record_val_type,
++ .encode = &reftable_obj_record_encode,
++ .decode = &reftable_obj_record_decode,
++ .clear = &reftable_obj_record_clear,
+ .is_deletion = not_a_deletion,
+};
+
@@ reftable/record.c (new)
+ memcpy(r->message, dest.buf, dest.len);
+ r->message[dest.len] = 0;
+
-+ slice_clear(&dest);
++ slice_release(&dest);
+ return start.len - in.len;
+
+error:
-+ slice_clear(&dest);
++ slice_release(&dest);
+ return REFTABLE_FORMAT_ERROR;
+}
+
@@ reftable/record.c (new)
+ (const struct reftable_log_record *)p);
+}
+
-+struct record_vtable reftable_log_record_vtable = {
++struct reftable_record_vtable reftable_log_record_vtable = {
+ .key = &reftable_log_record_key,
+ .type = BLOCK_TYPE_LOG,
+ .copy_from = &reftable_log_record_copy_from,
@@ reftable/record.c (new)
+ .is_deletion = &reftable_log_record_is_deletion_void,
+};
+
-+struct record new_record(byte typ)
++struct reftable_record reftable_new_record(byte typ)
+{
-+ struct record rec = { NULL };
++ struct reftable_record rec = { NULL };
+ switch (typ) {
+ case BLOCK_TYPE_REF: {
+ struct reftable_ref_record *r =
+ reftable_calloc(sizeof(struct reftable_ref_record));
-+ record_from_ref(&rec, r);
++ reftable_record_from_ref(&rec, r);
+ return rec;
+ }
+
+ case BLOCK_TYPE_OBJ: {
-+ struct obj_record *r =
-+ reftable_calloc(sizeof(struct obj_record));
-+ record_from_obj(&rec, r);
++ struct reftable_obj_record *r =
++ reftable_calloc(sizeof(struct reftable_obj_record));
++ reftable_record_from_obj(&rec, r);
+ return rec;
+ }
+ case BLOCK_TYPE_LOG: {
+ struct reftable_log_record *r =
+ reftable_calloc(sizeof(struct reftable_log_record));
-+ record_from_log(&rec, r);
++ reftable_record_from_log(&rec, r);
+ return rec;
+ }
+ case BLOCK_TYPE_INDEX: {
-+ struct index_record *r =
-+ reftable_calloc(sizeof(struct index_record));
-+ record_from_index(&rec, r);
++ struct reftable_index_record *r =
++ reftable_calloc(sizeof(struct reftable_index_record));
++ reftable_record_from_index(&rec, r);
+ return rec;
+ }
+ }
@@ reftable/record.c (new)
+ return rec;
+}
+
-+void *record_yield(struct record *rec)
++void *reftable_record_yield(struct reftable_record *rec)
+{
+ void *p = rec->data;
+ rec->data = NULL;
+ return p;
+}
+
-+void record_destroy(struct record *rec)
++void reftable_record_destroy(struct reftable_record *rec)
+{
-+ record_clear(*rec);
-+ reftable_free(record_yield(rec));
++ reftable_record_clear(rec);
++ reftable_free(reftable_record_yield(rec));
+}
+
-+static void index_record_key(const void *r, struct slice *dest)
++static void reftable_index_record_key(const void *r, struct slice *dest)
+{
-+ struct index_record *rec = (struct index_record *)r;
++ struct reftable_index_record *rec = (struct reftable_index_record *)r;
+ slice_copy(dest, rec->last_key);
+}
+
-+static void index_record_copy_from(void *rec, const void *src_rec,
-+ int hash_size)
++static void reftable_index_record_copy_from(void *rec, const void *src_rec,
++ int hash_size)
+{
-+ struct index_record *dst = (struct index_record *)rec;
-+ struct index_record *src = (struct index_record *)src_rec;
++ struct reftable_index_record *dst = (struct reftable_index_record *)rec;
++ struct reftable_index_record *src =
++ (struct reftable_index_record *)src_rec;
+
+ slice_copy(&dst->last_key, src->last_key);
+ dst->offset = src->offset;
+}
+
-+static void index_record_clear(void *rec)
++static void reftable_index_record_clear(void *rec)
+{
-+ struct index_record *idx = (struct index_record *)rec;
-+ slice_clear(&idx->last_key);
++ struct reftable_index_record *idx = (struct reftable_index_record *)rec;
++ slice_release(&idx->last_key);
+}
+
-+static byte index_record_val_type(const void *rec)
++static byte reftable_index_record_val_type(const void *rec)
+{
+ return 0;
+}
+
-+static int index_record_encode(const void *rec, struct slice out, int hash_size)
++static int reftable_index_record_encode(const void *rec, struct slice out,
++ int hash_size)
+{
-+ const struct index_record *r = (const struct index_record *)rec;
++ const struct reftable_index_record *r =
++ (const struct reftable_index_record *)rec;
+ struct slice start = out;
+
+ int n = put_var_int(out, r->offset);
@@ reftable/record.c (new)
+ return start.len - out.len;
+}
+
-+static int index_record_decode(void *rec, struct slice key, byte val_type,
-+ struct slice in, int hash_size)
++static int reftable_index_record_decode(void *rec, struct slice key,
++ byte val_type, struct slice in,
++ int hash_size)
+{
+ struct slice start = in;
-+ struct index_record *r = (struct index_record *)rec;
++ struct reftable_index_record *r = (struct reftable_index_record *)rec;
+ int n = 0;
+
+ slice_copy(&r->last_key, key);
@@ reftable/record.c (new)
+ return start.len - in.len;
+}
+
-+struct record_vtable index_record_vtable = {
-+ .key = &index_record_key,
++struct reftable_record_vtable reftable_index_record_vtable = {
++ .key = &reftable_index_record_key,
+ .type = BLOCK_TYPE_INDEX,
-+ .copy_from = &index_record_copy_from,
-+ .val_type = &index_record_val_type,
-+ .encode = &index_record_encode,
-+ .decode = &index_record_decode,
-+ .clear = &index_record_clear,
++ .copy_from = &reftable_index_record_copy_from,
++ .val_type = &reftable_index_record_val_type,
++ .encode = &reftable_index_record_encode,
++ .decode = &reftable_index_record_decode,
++ .clear = &reftable_index_record_clear,
+ .is_deletion = ¬_a_deletion,
+};
+
-+void record_key(struct record rec, struct slice *dest)
++void reftable_record_key(struct reftable_record *rec, struct slice *dest)
+{
-+ rec.ops->key(rec.data, dest);
++ rec->ops->key(rec->data, dest);
+}
+
-+byte record_type(struct record rec)
++byte reftable_record_type(struct reftable_record *rec)
+{
-+ return rec.ops->type;
++ return rec->ops->type;
+}
+
-+int record_encode(struct record rec, struct slice dest, int hash_size)
++int reftable_record_encode(struct reftable_record *rec, struct slice dest,
++ int hash_size)
+{
-+ return rec.ops->encode(rec.data, dest, hash_size);
++ return rec->ops->encode(rec->data, dest, hash_size);
+}
+
-+void record_copy_from(struct record rec, struct record src, int hash_size)
++void reftable_record_copy_from(struct reftable_record *rec,
++ struct reftable_record *src, int hash_size)
+{
-+ assert(src.ops->type == rec.ops->type);
++ assert(src->ops->type == rec->ops->type);
+
-+ rec.ops->copy_from(rec.data, src.data, hash_size);
++ rec->ops->copy_from(rec->data, src->data, hash_size);
+}
+
-+byte record_val_type(struct record rec)
++byte reftable_record_val_type(struct reftable_record *rec)
+{
-+ return rec.ops->val_type(rec.data);
++ return rec->ops->val_type(rec->data);
+}
+
-+int record_decode(struct record rec, struct slice key, byte extra,
-+ struct slice src, int hash_size)
++int reftable_record_decode(struct reftable_record *rec, struct slice key,
++ byte extra, struct slice src, int hash_size)
+{
-+ return rec.ops->decode(rec.data, key, extra, src, hash_size);
++ return rec->ops->decode(rec->data, key, extra, src, hash_size);
+}
+
-+void record_clear(struct record rec)
++void reftable_record_clear(struct reftable_record *rec)
+{
-+ rec.ops->clear(rec.data);
++ rec->ops->clear(rec->data);
+}
+
-+bool record_is_deletion(struct record rec)
++bool reftable_record_is_deletion(struct reftable_record *rec)
+{
-+ return rec.ops->is_deletion(rec.data);
++ return rec->ops->is_deletion(rec->data);
+}
+
-+void record_from_ref(struct record *rec, struct reftable_ref_record *ref_rec)
++void reftable_record_from_ref(struct reftable_record *rec,
++ struct reftable_ref_record *ref_rec)
+{
+ assert(rec->ops == NULL);
+ rec->data = ref_rec;
+ rec->ops = &reftable_ref_record_vtable;
+}
+
-+void record_from_obj(struct record *rec, struct obj_record *obj_rec)
++void reftable_record_from_obj(struct reftable_record *rec,
++ struct reftable_obj_record *obj_rec)
+{
+ assert(rec->ops == NULL);
+ rec->data = obj_rec;
-+ rec->ops = &obj_record_vtable;
++ rec->ops = &reftable_obj_record_vtable;
+}
+
-+void record_from_index(struct record *rec, struct index_record *index_rec)
++void reftable_record_from_index(struct reftable_record *rec,
++ struct reftable_index_record *index_rec)
+{
+ assert(rec->ops == NULL);
+ rec->data = index_rec;
-+ rec->ops = &index_record_vtable;
++ rec->ops = &reftable_index_record_vtable;
+}
+
-+void record_from_log(struct record *rec, struct reftable_log_record *log_rec)
++void reftable_record_from_log(struct reftable_record *rec,
++ struct reftable_log_record *log_rec)
+{
+ assert(rec->ops == NULL);
+ rec->data = log_rec;
+ rec->ops = &reftable_log_record_vtable;
+}
+
-+struct reftable_ref_record *record_as_ref(struct record rec)
++struct reftable_ref_record *reftable_record_as_ref(struct reftable_record *rec)
+{
-+ assert(record_type(rec) == BLOCK_TYPE_REF);
-+ return (struct reftable_ref_record *)rec.data;
++ assert(reftable_record_type(rec) == BLOCK_TYPE_REF);
++ return (struct reftable_ref_record *)rec->data;
+}
+
-+struct reftable_log_record *record_as_log(struct record rec)
++struct reftable_log_record *reftable_record_as_log(struct reftable_record *rec)
+{
-+ assert(record_type(rec) == BLOCK_TYPE_LOG);
-+ return (struct reftable_log_record *)rec.data;
++ assert(reftable_record_type(rec) == BLOCK_TYPE_LOG);
++ return (struct reftable_log_record *)rec->data;
+}
+
+static bool hash_equal(byte *a, byte *b, int hash_size)
@@ reftable/record.h (new)
+int put_var_int(struct slice dest, uint64_t val);
+
+/* Methods for records. */
-+struct record_vtable {
++struct reftable_record_vtable {
+ /* encode the key of to a byte slice. */
+ void (*key)(const void *rec, struct slice *dest);
+
@@ reftable/record.h (new)
+};
+
+/* record is a generic wrapper for different types of records. */
-+struct record {
++struct reftable_record {
+ void *data;
-+ struct record_vtable *ops;
++ struct reftable_record_vtable *ops;
+};
+
+/* returns true for recognized block types. Block start with the block type. */
-+int is_block_type(byte typ);
++int reftable_is_block_type(byte typ);
+
+/* creates a malloced record of the given type. Dispose with record_destroy */
-+struct record new_record(byte typ);
++struct reftable_record reftable_new_record(byte typ);
+
-+extern struct record_vtable reftable_ref_record_vtable;
++extern struct reftable_record_vtable reftable_ref_record_vtable;
+
+/* Encode `key` into `dest`. Sets `restart` to indicate a restart. Returns
+ number of bytes written. */
-+int encode_key(bool *restart, struct slice dest, struct slice prev_key,
-+ struct slice key, byte extra);
++int reftable_encode_key(bool *restart, struct slice dest, struct slice prev_key,
++ struct slice key, byte extra);
+
+/* Decode into `key` and `extra` from `in` */
-+int decode_key(struct slice *key, byte *extra, struct slice last_key,
-+ struct slice in);
++int reftable_decode_key(struct slice *key, byte *extra, struct slice last_key,
++ struct slice in);
+
-+/* index_record are used internally to speed up lookups. */
-+struct index_record {
++/* reftable_index_record are used internally to speed up lookups. */
++struct reftable_index_record {
+ uint64_t offset; /* Offset of block */
+ struct slice last_key; /* Last key of the block. */
+};
+
-+/* obj_record stores an object ID => ref mapping. */
-+struct obj_record {
++/* reftable_obj_record stores an object ID => ref mapping. */
++struct reftable_obj_record {
+ byte *hash_prefix; /* leading bytes of the object ID */
+ int hash_prefix_len; /* number of leading bytes. Constant
+ * across a single table. */
@@ reftable/record.h (new)
+
+/* see struct record_vtable */
+
-+void record_key(struct record rec, struct slice *dest);
-+byte record_type(struct record rec);
-+void record_copy_from(struct record rec, struct record src, int hash_size);
-+byte record_val_type(struct record rec);
-+int record_encode(struct record rec, struct slice dest, int hash_size);
-+int record_decode(struct record rec, struct slice key, byte extra,
-+ struct slice src, int hash_size);
-+bool record_is_deletion(struct record rec);
++void reftable_record_key(struct reftable_record *rec, struct slice *dest);
++byte reftable_record_type(struct reftable_record *rec);
++void reftable_record_copy_from(struct reftable_record *rec,
++ struct reftable_record *src, int hash_size);
++byte reftable_record_val_type(struct reftable_record *rec);
++int reftable_record_encode(struct reftable_record *rec, struct slice dest,
++ int hash_size);
++int reftable_record_decode(struct reftable_record *rec, struct slice key,
++ byte extra, struct slice src, int hash_size);
++bool reftable_record_is_deletion(struct reftable_record *rec);
+
+/* zeroes out the embedded record */
-+void record_clear(struct record rec);
++void reftable_record_clear(struct reftable_record *rec);
+
-+/* clear out the record, yielding the record data that was encapsulated. */
-+void *record_yield(struct record *rec);
++/* clear out the record, yielding the reftable_record data that was
++ * encapsulated. */
++void *reftable_record_yield(struct reftable_record *rec);
+
+/* clear and deallocate embedded record, and zero `rec`. */
-+void record_destroy(struct record *rec);
++void reftable_record_destroy(struct reftable_record *rec);
+
+/* initialize generic records from concrete records. The generic record should
+ * be zeroed out. */
-+void record_from_obj(struct record *rec, struct obj_record *objrec);
-+void record_from_index(struct record *rec, struct index_record *idxrec);
-+void record_from_ref(struct record *rec, struct reftable_ref_record *refrec);
-+void record_from_log(struct record *rec, struct reftable_log_record *logrec);
-+struct reftable_ref_record *record_as_ref(struct record ref);
-+struct reftable_log_record *record_as_log(struct record ref);
++void reftable_record_from_obj(struct reftable_record *rec,
++ struct reftable_obj_record *objrec);
++void reftable_record_from_index(struct reftable_record *rec,
++ struct reftable_index_record *idxrec);
++void reftable_record_from_ref(struct reftable_record *rec,
++ struct reftable_ref_record *refrec);
++void reftable_record_from_log(struct reftable_record *rec,
++ struct reftable_log_record *logrec);
++struct reftable_ref_record *reftable_record_as_ref(struct reftable_record *ref);
++struct reftable_log_record *reftable_record_as_log(struct reftable_record *ref);
+
+/* for qsort. */
+int reftable_ref_record_compare_name(const void *a, const void *b);
@@ reftable/refname.c (new)
+ }
+ }
+
-+ err = reftable_table_read_ref(mod->tab, name, &ref);
++ err = reftable_table_read_ref(&mod->tab, name, &ref);
+ reftable_ref_record_clear(&ref);
+ return err;
+}
@@ reftable/refname.c (new)
+ goto exit;
+ }
+ }
-+ err = reftable_table_seek_ref(mod->tab, &it, prefix);
++ err = reftable_table_seek_ref(&mod->tab, &it, prefix);
+ if (err) {
+ goto exit;
+ }
+
+ while (true) {
-+ err = reftable_iterator_next_ref(it, &ref);
++ err = reftable_iterator_next_ref(&it, &ref);
+ if (err) {
+ goto exit;
+ }
@@ reftable/refname.c (new)
+ goto exit;
+ }
+ slice_set_string(&slashed, mod->add[i]);
-+ slice_append_string(&slashed, "/");
++ slice_addstr(&slashed, "/");
+
+ err = modification_has_ref_with_prefix(
+ mod, slice_as_string(&slashed));
@@ reftable/refname.c (new)
+ }
+ err = 0;
+exit:
-+ slice_clear(&slashed);
++ slice_release(&slashed);
+ return err;
+}
@@ reftable/reftable.c (new)
+#include "merged.h"
+
+struct reftable_table_vtable {
-+ int (*seek)(void *tab, struct reftable_iterator *it, struct record);
++ int (*seek)(void *tab, struct reftable_iterator *it,
++ struct reftable_record *);
+};
+
+static int reftable_reader_seek_void(void *tab, struct reftable_iterator *it,
-+ struct record rec)
++ struct reftable_record *rec)
+{
+ return reader_seek((struct reftable_reader *)tab, it, rec);
+}
@@ reftable/reftable.c (new)
+
+static int reftable_merged_table_seek_void(void *tab,
+ struct reftable_iterator *it,
-+ struct record rec)
++ struct reftable_record *rec)
+{
+ return merged_table_seek_record((struct reftable_merged_table *)tab, it,
+ rec);
@@ reftable/reftable.c (new)
+ .seek = reftable_merged_table_seek_void,
+};
+
-+int reftable_table_seek_ref(struct reftable_table tab,
++int reftable_table_seek_ref(struct reftable_table *tab,
+ struct reftable_iterator *it, const char *name)
+{
+ struct reftable_ref_record ref = {
+ .ref_name = (char *)name,
+ };
-+ struct record rec = { 0 };
-+ record_from_ref(&rec, &ref);
-+ return tab.ops->seek(tab.table_arg, it, rec);
++ struct reftable_record rec = { 0 };
++ reftable_record_from_ref(&rec, &ref);
++ return tab->ops->seek(tab->table_arg, it, &rec);
+}
+
+void reftable_table_from_reader(struct reftable_table *tab,
@@ reftable/reftable.c (new)
+ tab->table_arg = merged;
+}
+
-+int reftable_table_read_ref(struct reftable_table tab, const char *name,
++int reftable_table_read_ref(struct reftable_table *tab, const char *name,
+ struct reftable_ref_record *ref)
+{
+ struct reftable_iterator it = { 0 };
@@ reftable/reftable.c (new)
+ goto exit;
+ }
+
-+ err = reftable_iterator_next_ref(it, ref);
++ err = reftable_iterator_next_ref(&it, ref);
+ if (err) {
+ goto exit;
+ }
@@ reftable/reftable.h (new)
+/* reads the next reftable_ref_record. Returns < 0 for error, 0 for OK and > 0:
+ end of iteration.
+*/
-+int reftable_iterator_next_ref(struct reftable_iterator it,
++int reftable_iterator_next_ref(struct reftable_iterator *it,
+ struct reftable_ref_record *ref);
+
+/* reads the next reftable_log_record. Returns < 0 for error, 0 for OK and > 0:
+ end of iteration.
+*/
-+int reftable_iterator_next_log(struct reftable_iterator it,
++int reftable_iterator_next_log(struct reftable_iterator *it,
+ struct reftable_log_record *log);
+
+/* releases resources associated with an iterator. */
@@ reftable/reftable.h (new)
+ * closed on calling reftable_reader_destroy().
+ */
+int reftable_new_reader(struct reftable_reader **pp,
-+ struct reftable_block_source src, const char *name);
++ struct reftable_block_source *src, const char *name);
+
+/* reftable_reader_seek_ref returns an iterator where 'name' would be inserted
+ in the table. To seek to the start of the table, use name = "".
@@ reftable/reftable.h (new)
+ example:
+
+ struct reftable_reader *r = NULL;
-+ int err = reftable_new_reader(&r, src, "filename");
++ int err = reftable_new_reader(&r, &src, "filename");
+ if (err < 0) { ... }
+ struct reftable_iterator it = {0};
+ err = reftable_reader_seek_ref(r, &it, "refs/heads/master");
+ if (err < 0) { ... }
+ struct reftable_ref_record ref = {0};
+ while (1) {
-+ err = reftable_iterator_next_ref(it, &ref);
++ err = reftable_iterator_next_ref(&it, &ref);
+ if (err > 0) {
+ break;
+ }
@@ reftable/reftable.h (new)
+ void *table_arg;
+};
+
-+int reftable_table_seek_ref(struct reftable_table tab,
++int reftable_table_seek_ref(struct reftable_table *tab,
+ struct reftable_iterator *it, const char *name);
+
+void reftable_table_from_reader(struct reftable_table *tab,
@@ reftable/reftable.h (new)
+
+/* convenience function to read a single ref. Returns < 0 for error, 0
+ for success, and 1 if ref not found. */
-+int reftable_table_read_ref(struct reftable_table tab, const char *name,
++int reftable_table_read_ref(struct reftable_table *tab, const char *name,
+ struct reftable_ref_record *ref);
+
+/****************************************************************
@@ reftable/slice.c (new)
+ s->len = l;
+}
+
-+void slice_append_string(struct slice *d, const char *s)
++void slice_addstr(struct slice *d, const char *s)
+{
+ int l1 = d->len;
+ int l2 = strlen(s);
@@ reftable/slice.c (new)
+ memcpy(d->buf + l1, s, l2);
+}
+
-+void slice_append(struct slice *s, struct slice a)
++void slice_addbuf(struct slice *s, struct slice a)
+{
+ int end = s->len;
+ slice_resize(s, s->len + a.len);
@@ reftable/slice.c (new)
+ s->len -= n;
+}
+
-+byte *slice_yield(struct slice *s)
++byte *slice_detach(struct slice *s)
+{
+ byte *p = s->buf;
+ s->buf = NULL;
@@ reftable/slice.c (new)
+ return p;
+}
+
-+void slice_clear(struct slice *s)
++void slice_release(struct slice *s)
+{
-+ reftable_free(slice_yield(s));
++ reftable_free(slice_detach(s));
+}
+
+void slice_copy(struct slice *dest, struct slice src)
@@ reftable/slice.c (new)
+ slice_resize(&s, in.len + 1);
+ s.buf[in.len] = 0;
+ memcpy(s.buf, in.buf, in.len);
-+ return (char *)slice_yield(&s);
++ return (char *)slice_detach(&s);
+}
+
+bool slice_equal(struct slice a, struct slice b)
@@ reftable/slice.c (new)
+ return memcmp(a.buf, b.buf, a.len) == 0;
+}
+
-+int slice_compare(struct slice a, struct slice b)
++int slice_cmp(struct slice a, struct slice b)
+{
+ int min = a.len < b.len ? a.len : b.len;
+ int res = memcmp(a.buf, b.buf, min);
@@ reftable/slice.c (new)
+ }
+}
+
-+int slice_write(struct slice *b, byte *data, size_t sz)
++int slice_add(struct slice *b, byte *data, size_t sz)
+{
+ if (b->len + sz > b->cap) {
+ int newcap = 2 * b->cap + 1;
@@ reftable/slice.c (new)
+ return sz;
+}
+
-+int slice_write_void(void *b, byte *data, size_t sz)
++int slice_add_void(void *b, byte *data, size_t sz)
+{
-+ return slice_write((struct slice *)b, data, sz);
++ return slice_add((struct slice *)b, data, sz);
+}
+
+static uint64_t slice_size(void *b)
@@ reftable/slice.h (new)
+};
+
+void slice_set_string(struct slice *dest, const char *src);
-+void slice_append_string(struct slice *dest, const char *src);
-+/* Set length to 0, but retain buffer */
-+void slice_clear(struct slice *slice);
++void slice_addstr(struct slice *dest, const char *src);
++
++/* Deallocate and clear slice */
++void slice_release(struct slice *slice);
+
+/* Return a malloced string for `src` */
+char *slice_to_string(struct slice src);
@@ reftable/slice.h (new)
+bool slice_equal(struct slice a, struct slice b);
+
+/* Return `buf`, clearing out `s` */
-+byte *slice_yield(struct slice *s);
++byte *slice_detach(struct slice *s);
+
+/* Copy bytes */
+void slice_copy(struct slice *dest, struct slice src);
@@ reftable/slice.h (new)
+void slice_resize(struct slice *s, int l);
+
+/* Signed comparison */
-+int slice_compare(struct slice a, struct slice b);
++int slice_cmp(struct slice a, struct slice b);
+
+/* Append `data` to the `dest` slice. */
-+int slice_write(struct slice *dest, byte *data, size_t sz);
++int slice_add(struct slice *dest, byte *data, size_t sz);
+
+/* Append `add` to `dest. */
-+void slice_append(struct slice *dest, struct slice add);
++void slice_addbuf(struct slice *dest, struct slice add);
+
-+/* Like slice_write, but suitable for passing to reftable_new_writer
++/* Like slice_add, but suitable for passing to reftable_new_writer
+ */
-+int slice_write_void(void *b, byte *data, size_t sz);
++int slice_add_void(void *b, byte *data, size_t sz);
+
+/* Find the longest shared prefix size of `a` and `b` */
+int common_prefix_size(struct slice a, struct slice b);
@@ reftable/stack.c (new)
+ *dest = NULL;
+
+ slice_set_string(&list_file_name, dir);
-+ slice_append_string(&list_file_name, "/tables.list");
++ slice_addstr(&list_file_name, "/tables.list");
+
+ p->list_file = slice_to_string(list_file_name);
-+ slice_clear(&list_file_name);
++ slice_release(&list_file_name);
+ p->reftable_dir = xstrdup(dir);
+ p->config = config;
+
@@ reftable/stack.c (new)
+ reftable_malloc(sizeof(struct reftable_reader *) * names_len);
+ int new_tables_len = 0;
+ struct reftable_merged_table *new_merged = NULL;
-+
++ int i;
+ struct slice table_path = { 0 };
+
+ while (*names) {
@@ reftable/stack.c (new)
+ if (rd == NULL) {
+ struct reftable_block_source src = { 0 };
+ slice_set_string(&table_path, st->reftable_dir);
-+ slice_append_string(&table_path, "/");
-+ slice_append_string(&table_path, name);
++ slice_addstr(&table_path, "/");
++ slice_addstr(&table_path, name);
+
+ err = reftable_block_source_from_file(
+ &src, slice_as_string(&table_path));
@@ reftable/stack.c (new)
+ goto exit;
+ }
+
-+ err = reftable_new_reader(&rd, src, name);
++ err = reftable_new_reader(&rd, &src, name);
+ if (err < 0) {
+ goto exit;
+ }
@@ reftable/stack.c (new)
+ new_merged->suppress_deletions = true;
+ st->merged = new_merged;
+
-+ {
-+ int i = 0;
-+ for (i = 0; i < cur_len; i++) {
-+ if (cur[i] != NULL) {
-+ reader_close(cur[i]);
-+ reftable_reader_free(cur[i]);
-+ }
++ for (i = 0; i < cur_len; i++) {
++ if (cur[i] != NULL) {
++ reader_close(cur[i]);
++ reftable_reader_free(cur[i]);
+ }
+ }
+
+exit:
-+ slice_clear(&table_path);
-+ {
-+ int i = 0;
-+ for (i = 0; i < new_tables_len; i++) {
-+ reader_close(new_tables[i]);
-+ reftable_reader_free(new_tables[i]);
-+ }
++ slice_release(&table_path);
++ for (i = 0; i < new_tables_len; i++) {
++ reader_close(new_tables[i]);
++ reftable_reader_free(new_tables[i]);
+ }
+ reftable_free(new_tables);
+ reftable_free(cur);
@@ reftable/stack.c (new)
+ add->stack = st;
+
+ slice_set_string(&add->lock_file_name, st->list_file);
-+ slice_append_string(&add->lock_file_name, ".lock");
++ slice_addstr(&add->lock_file_name, ".lock");
+
+ add->lock_file_fd = open(slice_as_string(&add->lock_file_name),
+ O_EXCL | O_CREAT | O_WRONLY, 0644);
@@ reftable/stack.c (new)
+ struct slice nm = { 0 };
+ for (i = 0; i < add->new_tables_len; i++) {
+ slice_set_string(&nm, add->stack->list_file);
-+ slice_append_string(&nm, "/");
-+ slice_append_string(&nm, add->new_tables[i]);
++ slice_addstr(&nm, "/");
++ slice_addstr(&nm, add->new_tables[i]);
+ unlink(slice_as_string(&nm));
+ reftable_free(add->new_tables[i]);
+ add->new_tables[i] = NULL;
@@ reftable/stack.c (new)
+ }
+ if (add->lock_file_name.len > 0) {
+ unlink(slice_as_string(&add->lock_file_name));
-+ slice_clear(&add->lock_file_name);
++ slice_release(&add->lock_file_name);
+ }
+
+ free_names(add->names);
+ add->names = NULL;
-+ slice_clear(&nm);
++ slice_release(&nm);
+}
+
+void reftable_addition_destroy(struct reftable_addition *add)
@@ reftable/stack.c (new)
+ }
+
+ for (i = 0; i < add->stack->merged->stack_len; i++) {
-+ slice_append_string(&table_list,
-+ add->stack->merged->stack[i]->name);
-+ slice_append_string(&table_list, "\n");
++ slice_addstr(&table_list, add->stack->merged->stack[i]->name);
++ slice_addstr(&table_list, "\n");
+ }
+ for (i = 0; i < add->new_tables_len; i++) {
-+ slice_append_string(&table_list, add->new_tables[i]);
-+ slice_append_string(&table_list, "\n");
++ slice_addstr(&table_list, add->new_tables[i]);
++ slice_addstr(&table_list, "\n");
+ }
+
+ err = write(add->lock_file_fd, table_list.buf, table_list.len);
-+ slice_clear(&table_list);
++ slice_release(&table_list);
+ if (err < 0) {
+ err = REFTABLE_IO_ERROR;
+ goto exit;
@@ reftable/stack.c (new)
+ format_name(&next_name, add->next_update_index, add->next_update_index);
+
+ slice_set_string(&temp_tab_file_name, add->stack->reftable_dir);
-+ slice_append_string(&temp_tab_file_name, "/");
-+ slice_append(&temp_tab_file_name, next_name);
-+ slice_append_string(&temp_tab_file_name, ".temp.XXXXXX");
++ slice_addstr(&temp_tab_file_name, "/");
++ slice_addbuf(&temp_tab_file_name, next_name);
++ slice_addstr(&temp_tab_file_name, ".temp.XXXXXX");
+
+ tab_fd = mkstemp((char *)slice_as_string(&temp_tab_file_name));
+ if (tab_fd < 0) {
@@ reftable/stack.c (new)
+ }
+
+ format_name(&next_name, wr->min_update_index, wr->max_update_index);
-+ slice_append_string(&next_name, ".ref");
++ slice_addstr(&next_name, ".ref");
+
+ slice_set_string(&tab_file_name, add->stack->reftable_dir);
-+ slice_append_string(&tab_file_name, "/");
-+ slice_append(&tab_file_name, next_name);
++ slice_addstr(&tab_file_name, "/");
++ slice_addbuf(&tab_file_name, next_name);
+
+ /* TODO: should check destination out of paranoia */
+ err = rename(slice_as_string(&temp_tab_file_name),
@@ reftable/stack.c (new)
+ unlink(slice_as_string(&temp_tab_file_name));
+ }
+
-+ slice_clear(&temp_tab_file_name);
-+ slice_clear(&tab_file_name);
-+ slice_clear(&next_name);
++ slice_release(&temp_tab_file_name);
++ slice_release(&tab_file_name);
++ slice_release(&next_name);
+ reftable_writer_free(wr);
+ return err;
+}
@@ reftable/stack.c (new)
+ reftable_reader_max_update_index(st->merged->stack[first]));
+
+ slice_set_string(temp_tab, st->reftable_dir);
-+ slice_append_string(temp_tab, "/");
-+ slice_append(temp_tab, next_name);
-+ slice_append_string(temp_tab, ".temp.XXXXXX");
++ slice_addstr(temp_tab, "/");
++ slice_addbuf(temp_tab, next_name);
++ slice_addstr(temp_tab, ".temp.XXXXXX");
+
+ tab_fd = mkstemp((char *)slice_as_string(temp_tab));
+ wr = reftable_new_writer(reftable_fd_write, &tab_fd, &st->config);
@@ reftable/stack.c (new)
+ }
+ if (err != 0 && temp_tab->len > 0) {
+ unlink(slice_as_string(temp_tab));
-+ slice_clear(temp_tab);
++ slice_release(temp_tab);
+ }
-+ slice_clear(&next_name);
++ slice_release(&next_name);
+ return err;
+}
+
@@ reftable/stack.c (new)
+ }
+
+ while (true) {
-+ err = reftable_iterator_next_ref(it, &ref);
++ err = reftable_iterator_next_ref(&it, &ref);
+ if (err > 0) {
+ err = 0;
+ break;
@@ reftable/stack.c (new)
+ }
+
+ while (true) {
-+ err = reftable_iterator_next_log(it, &log);
++ err = reftable_iterator_next_log(&it, &log);
+ if (err > 0) {
+ err = 0;
+ break;
@@ reftable/stack.c (new)
+ bool have_lock = false;
+ int lock_file_fd = 0;
+ int compact_count = last - first + 1;
++ char **listp = NULL;
+ char **delete_on_success =
+ reftable_calloc(sizeof(char *) * (compact_count + 1));
+ char **subtable_locks =
@@ reftable/stack.c (new)
+ st->stats.attempts++;
+
+ slice_set_string(&lock_file_name, st->list_file);
-+ slice_append_string(&lock_file_name, ".lock");
++ slice_addstr(&lock_file_name, ".lock");
+
+ lock_file_fd = open(slice_as_string(&lock_file_name),
+ O_EXCL | O_CREAT | O_WRONLY, 0644);
@@ reftable/stack.c (new)
+ struct slice subtab_file_name = { 0 };
+ struct slice subtab_lock = { 0 };
+ slice_set_string(&subtab_file_name, st->reftable_dir);
-+ slice_append_string(&subtab_file_name, "/");
-+ slice_append_string(&subtab_file_name,
-+ reader_name(st->merged->stack[i]));
++ slice_addstr(&subtab_file_name, "/");
++ slice_addstr(&subtab_file_name,
++ reader_name(st->merged->stack[i]));
+
+ slice_copy(&subtab_lock, subtab_file_name);
-+ slice_append_string(&subtab_lock, ".lock");
++ slice_addstr(&subtab_lock, ".lock");
+
+ {
+ int sublock_file_fd =
@@ reftable/stack.c (new)
+
+ format_name(&new_table_name, st->merged->stack[first]->min_update_index,
+ st->merged->stack[last]->max_update_index);
-+ slice_append_string(&new_table_name, ".ref");
++ slice_addstr(&new_table_name, ".ref");
+
+ slice_set_string(&new_table_path, st->reftable_dir);
-+ slice_append_string(&new_table_path, "/");
++ slice_addstr(&new_table_path, "/");
+
-+ slice_append(&new_table_path, new_table_name);
++ slice_addbuf(&new_table_path, new_table_name);
+
+ if (!is_empty_table) {
+ err = rename(slice_as_string(&temp_tab_file_name),
@@ reftable/stack.c (new)
+ }
+
+ for (i = 0; i < first; i++) {
-+ slice_append_string(&ref_list_contents,
-+ st->merged->stack[i]->name);
-+ slice_append_string(&ref_list_contents, "\n");
++ slice_addstr(&ref_list_contents, st->merged->stack[i]->name);
++ slice_addstr(&ref_list_contents, "\n");
+ }
+ if (!is_empty_table) {
-+ slice_append(&ref_list_contents, new_table_name);
-+ slice_append_string(&ref_list_contents, "\n");
++ slice_addbuf(&ref_list_contents, new_table_name);
++ slice_addstr(&ref_list_contents, "\n");
+ }
+ for (i = last + 1; i < st->merged->stack_len; i++) {
-+ slice_append_string(&ref_list_contents,
-+ st->merged->stack[i]->name);
-+ slice_append_string(&ref_list_contents, "\n");
++ slice_addstr(&ref_list_contents, st->merged->stack[i]->name);
++ slice_addstr(&ref_list_contents, "\n");
+ }
+
+ err = write(lock_file_fd, ref_list_contents.buf, ref_list_contents.len);
@@ reftable/stack.c (new)
+ */
+ err = reftable_stack_reload_maybe_reuse(st, first < last);
+
-+ {
-+ char **p = delete_on_success;
-+ while (*p) {
-+ if (strcmp(*p, slice_as_string(&new_table_path))) {
-+ unlink(*p);
-+ }
-+ p++;
++ listp = delete_on_success;
++ while (*listp) {
++ if (strcmp(*listp, slice_as_string(&new_table_path))) {
++ unlink(*listp);
+ }
++ listp++;
+ }
+
+exit:
+ free_names(delete_on_success);
-+ {
-+ char **p = subtable_locks;
-+ while (*p) {
-+ unlink(*p);
-+ p++;
-+ }
++
++ listp = subtable_locks;
++ while (*listp) {
++ unlink(*listp);
++ listp++;
+ }
+ free_names(subtable_locks);
+ if (lock_file_fd > 0) {
@@ reftable/stack.c (new)
+ if (have_lock) {
+ unlink(slice_as_string(&lock_file_name));
+ }
-+ slice_clear(&new_table_name);
-+ slice_clear(&new_table_path);
-+ slice_clear(&ref_list_contents);
-+ slice_clear(&temp_tab_file_name);
-+ slice_clear(&lock_file_name);
++ slice_release(&new_table_name);
++ slice_release(&new_table_path);
++ slice_release(&ref_list_contents);
++ slice_release(&temp_tab_file_name);
++ slice_release(&lock_file_name);
+ return err;
+}
+
@@ reftable/stack.c (new)
+{
+ struct reftable_table tab = { NULL };
+ reftable_table_from_merged_table(&tab, reftable_stack_merged_table(st));
-+ return reftable_table_read_ref(tab, refname, ref);
++ return reftable_table_read_ref(&tab, refname, ref);
+}
+
+int reftable_stack_read_log(struct reftable_stack *st, const char *refname,
@@ reftable/stack.c (new)
+ goto exit;
+ }
+
-+ err = reftable_iterator_next_log(it, log);
++ err = reftable_iterator_next_log(&it, log);
+ if (err) {
+ goto exit;
+ }
@@ reftable/stack.c (new)
+ goto exit;
+ }
+
-+ err = reftable_new_reader(&rd, src, new_tab_name);
++ err = reftable_new_reader(&rd, &src, new_tab_name);
+ if (err < 0) {
+ goto exit;
+ }
@@ reftable/stack.c (new)
+
+ while (true) {
+ struct reftable_ref_record ref = { 0 };
-+ err = reftable_iterator_next_ref(it, &ref);
++ err = reftable_iterator_next_ref(&it, &ref);
+ if (err > 0) {
+ break;
+ }
@@ reftable/tree.c (new)
+ int (*compare)(const void *, const void *),
+ int insert)
+{
++ int res;
+ if (*rootp == NULL) {
+ if (!insert) {
+ return NULL;
@@ reftable/tree.c (new)
+ }
+ }
+
-+ {
-+ int res = compare(key, (*rootp)->key);
-+ if (res < 0) {
-+ return tree_search(key, &(*rootp)->left, compare,
-+ insert);
-+ } else if (res > 0) {
-+ return tree_search(key, &(*rootp)->right, compare,
-+ insert);
-+ }
++ res = compare(key, (*rootp)->key);
++ if (res < 0) {
++ return tree_search(key, &(*rootp)->left, compare, insert);
++ } else if (res > 0) {
++ return tree_search(key, &(*rootp)->right, compare, insert);
+ }
+ return *rootp;
+}
@@ reftable/writer.c (new)
+
+static int obj_index_tree_node_compare(const void *a, const void *b)
+{
-+ return slice_compare(((const struct obj_index_tree_node *)a)->hash,
-+ ((const struct obj_index_tree_node *)b)->hash);
++ return slice_cmp(((const struct obj_index_tree_node *)a)->hash,
++ ((const struct obj_index_tree_node *)b)->hash);
+}
+
+static void writer_index_hash(struct reftable_writer *w, struct slice hash)
@@ reftable/writer.c (new)
+ key->offsets[key->offset_len++] = off;
+}
+
-+static int writer_add_record(struct reftable_writer *w, struct record rec)
++static int writer_add_record(struct reftable_writer *w,
++ struct reftable_record *rec)
+{
+ int result = -1;
+ struct slice key = { 0 };
+ int err = 0;
-+ record_key(rec, &key);
-+ if (slice_compare(w->last_key, key) >= 0) {
++ reftable_record_key(rec, &key);
++ if (slice_cmp(w->last_key, key) >= 0) {
+ goto exit;
+ }
+
+ slice_copy(&w->last_key, key);
+ if (w->block_writer == NULL) {
-+ writer_reinit_block_writer(w, record_type(rec));
++ writer_reinit_block_writer(w, reftable_record_type(rec));
+ }
+
-+ assert(block_writer_type(w->block_writer) == record_type(rec));
++ assert(block_writer_type(w->block_writer) == reftable_record_type(rec));
+
+ if (block_writer_add(w->block_writer, rec) == 0) {
+ result = 0;
@@ reftable/writer.c (new)
+ goto exit;
+ }
+
-+ writer_reinit_block_writer(w, record_type(rec));
++ writer_reinit_block_writer(w, reftable_record_type(rec));
+ err = block_writer_add(w->block_writer, rec);
+ if (err < 0) {
+ result = err;
@@ reftable/writer.c (new)
+
+ result = 0;
+exit:
-+ slice_clear(&key);
++ slice_release(&key);
+ return result;
+}
+
+int reftable_writer_add_ref(struct reftable_writer *w,
+ struct reftable_ref_record *ref)
+{
-+ struct record rec = { 0 };
++ struct reftable_record rec = { 0 };
+ struct reftable_ref_record copy = *ref;
+ int err = 0;
+
@@ reftable/writer.c (new)
+ return REFTABLE_API_ERROR;
+ }
+
-+ record_from_ref(&rec, ©);
++ reftable_record_from_ref(&rec, ©);
+ copy.update_index -= w->min_update_index;
-+ err = writer_add_record(w, rec);
++ err = writer_add_record(w, &rec);
+ if (err < 0) {
+ return err;
+ }
@@ reftable/writer.c (new)
+int reftable_writer_add_log(struct reftable_writer *w,
+ struct reftable_log_record *log)
+{
++ struct reftable_record rec = { 0 };
++ int err;
+ if (log->ref_name == NULL) {
+ return REFTABLE_API_ERROR;
+ }
@@ reftable/writer.c (new)
+ w->next -= w->pending_padding;
+ w->pending_padding = 0;
+
-+ {
-+ struct record rec = { 0 };
-+ int err;
-+ record_from_log(&rec, log);
-+ err = writer_add_record(w, rec);
-+ return err;
-+ }
++ reftable_record_from_log(&rec, log);
++ err = writer_add_record(w, &rec);
++ return err;
+}
+
+int reftable_writer_add_logs(struct reftable_writer *w,
@@ reftable/writer.c (new)
+ int before_blocks = w->stats.idx_stats.blocks;
+ int err = writer_flush_block(w);
+ int i = 0;
++ struct reftable_block_stats *bstats = NULL;
+ if (err < 0) {
+ return err;
+ }
+
+ while (w->index_len > threshold) {
-+ struct index_record *idx = NULL;
++ struct reftable_index_record *idx = NULL;
+ int idx_len = 0;
+
+ max_level++;
@@ reftable/writer.c (new)
+ w->index_len = 0;
+ w->index_cap = 0;
+ for (i = 0; i < idx_len; i++) {
-+ struct record rec = { 0 };
-+ record_from_index(&rec, idx + i);
-+ if (block_writer_add(w->block_writer, rec) == 0) {
++ struct reftable_record rec = { 0 };
++ reftable_record_from_index(&rec, idx + i);
++ if (block_writer_add(w->block_writer, &rec) == 0) {
+ continue;
+ }
+
@@ reftable/writer.c (new)
+
+ writer_reinit_block_writer(w, BLOCK_TYPE_INDEX);
+
-+ err = block_writer_add(w->block_writer, rec);
++ err = block_writer_add(w->block_writer, &rec);
+ if (err != 0) {
+ /* write into fresh block should always succeed
+ */
@@ reftable/writer.c (new)
+ }
+ }
+ for (i = 0; i < idx_len; i++) {
-+ slice_clear(&idx[i].last_key);
++ slice_release(&idx[i].last_key);
+ }
+ reftable_free(idx);
+ }
@@ reftable/writer.c (new)
+ return err;
+ }
+
-+ {
-+ struct reftable_block_stats *bstats =
-+ writer_reftable_block_stats(w, typ);
-+ bstats->index_blocks =
-+ w->stats.idx_stats.blocks - before_blocks;
-+ bstats->index_offset = index_start;
-+ bstats->max_index_level = max_level;
-+ }
++ bstats = writer_reftable_block_stats(w, typ);
++ bstats->index_blocks = w->stats.idx_stats.blocks - before_blocks;
++ bstats->index_offset = index_start;
++ bstats->max_index_level = max_level;
+
+ /* Reinit lastKey, as the next section can start with any key. */
+ w->last_key.len = 0;
@@ reftable/writer.c (new)
+{
+ struct write_record_arg *arg = (struct write_record_arg *)void_arg;
+ struct obj_index_tree_node *entry = (struct obj_index_tree_node *)key;
-+ struct obj_record obj_rec = {
++ struct reftable_obj_record obj_rec = {
+ .hash_prefix = entry->hash.buf,
+ .hash_prefix_len = arg->w->stats.object_id_len,
+ .offsets = entry->offsets,
+ .offset_len = entry->offset_len,
+ };
-+ struct record rec = { 0 };
++ struct reftable_record rec = { 0 };
+ if (arg->err < 0) {
+ goto exit;
+ }
+
-+ record_from_obj(&rec, &obj_rec);
-+ arg->err = block_writer_add(arg->w->block_writer, rec);
++ reftable_record_from_obj(&rec, &obj_rec);
++ arg->err = block_writer_add(arg->w->block_writer, &rec);
+ if (arg->err == 0) {
+ goto exit;
+ }
@@ reftable/writer.c (new)
+ }
+
+ writer_reinit_block_writer(arg->w, BLOCK_TYPE_OBJ);
-+ arg->err = block_writer_add(arg->w->block_writer, rec);
++ arg->err = block_writer_add(arg->w->block_writer, &rec);
+ if (arg->err == 0) {
+ goto exit;
+ }
+ obj_rec.offset_len = 0;
-+ arg->err = block_writer_add(arg->w->block_writer, rec);
++ arg->err = block_writer_add(arg->w->block_writer, &rec);
+
+ /* Should be able to write into a fresh block. */
+ assert(arg->err == 0);
@@ reftable/writer.c (new)
+ struct obj_index_tree_node *entry = (struct obj_index_tree_node *)key;
+
+ FREE_AND_NULL(entry->offsets);
-+ slice_clear(&entry->hash);
++ slice_release(&entry->hash);
+ reftable_free(entry);
+}
+
@@ reftable/writer.c (new)
+ /* free up memory. */
+ block_writer_clear(&w->block_writer_data);
+ writer_clear_index(w);
-+ slice_clear(&w->last_key);
++ slice_release(&w->last_key);
+ return err;
+}
+
@@ reftable/writer.c (new)
+{
+ int i = 0;
+ for (i = 0; i < w->index_len; i++) {
-+ slice_clear(&w->index[i].last_key);
++ slice_release(&w->index[i].last_key);
+ }
+
+ FREE_AND_NULL(w->index);
@@ reftable/writer.c (new)
+ int raw_bytes = block_writer_finish(w->block_writer);
+ int padding = 0;
+ int err = 0;
++ struct reftable_index_record ir = { 0 };
+ if (raw_bytes < 0) {
+ return raw_bytes;
+ }
@@ reftable/writer.c (new)
+ if (w->index_cap == w->index_len) {
+ w->index_cap = 2 * w->index_cap + 1;
+ w->index = reftable_realloc(
-+ w->index, sizeof(struct index_record) * w->index_cap);
++ w->index,
++ sizeof(struct reftable_index_record) * w->index_cap);
+ }
+
-+ {
-+ struct index_record ir = {
-+ .offset = w->next,
-+ };
-+ slice_copy(&ir.last_key, w->block_writer->last_key);
-+ w->index[w->index_len] = ir;
-+ }
++ ir.offset = w->next;
++ slice_copy(&ir.last_key, w->block_writer->last_key);
++ w->index[w->index_len] = ir;
+
+ w->index_len++;
+ w->next += padding + raw_bytes;
@@ reftable/writer.h (new)
+ struct block_writer block_writer_data;
+
+ /* pending index records for the current section */
-+ struct index_record *index;
++ struct reftable_index_record *index;
+ int index_len;
+ int index_cap;
+
11: ace95b6cd88 ! 6: 865c2c4567a Reftable support for git-core
@@ refs/reftable-backend.c (new)
+ struct reftable_stack *stack;
+};
+
++static int reftable_read_raw_ref(struct ref_store *ref_store,
++ const char *refname, struct object_id *oid,
++ struct strbuf *referent, unsigned int *type);
++
+static void clear_reftable_log_record(struct reftable_log_record *log)
+{
+ log->old_hash = NULL;
@@ refs/reftable-backend.c (new)
+ struct git_reftable_iterator *ri =
+ (struct git_reftable_iterator *)ref_iterator;
+ while (ri->err == 0) {
-+ ri->err = reftable_iterator_next_ref(ri->iter, &ri->ref);
++ ri->err = reftable_iterator_next_ref(&ri->iter, &ri->ref);
+ if (ri->err) {
+ break;
+ }
@@ refs/reftable-backend.c (new)
+ return &ri->base;
+}
+
++static int fixup_symrefs(struct ref_store *ref_store,
++ struct ref_transaction *transaction)
++{
++ struct strbuf referent = STRBUF_INIT;
++ int i = 0;
++ int err = 0;
++
++ for (i = 0; i < transaction->nr; i++) {
++ struct ref_update *update = transaction->updates[i];
++ struct object_id old_oid;
++
++ err = reftable_read_raw_ref(ref_store, update->refname,
++ &old_oid, &referent,
++ /* mutate input, like
++ files-backend.c */
++ &update->type);
++ if (err < 0 && errno == ENOENT &&
++ is_null_oid(&update->old_oid)) {
++ err = 0;
++ }
++ if (err < 0)
++ goto done;
++
++ if (!(update->type & REF_ISSYMREF))
++ continue;
++
++ if (update->flags & REF_NO_DEREF) {
++ /* what should happen here? See files-backend.c
++ * lock_ref_for_update. */
++ } else {
++ /*
++ If we are updating a symref (eg. HEAD), we should also
++ update the branch that the symref points to.
++
++ This is generic functionality, and would be better
++ done in refs.c, but the current implementation is
++ intertwined with the locking in files-backend.c.
++ */
++ int new_flags = update->flags;
++ struct ref_update *new_update = NULL;
++
++ /* if this is an update for HEAD, should also record a
++ log entry for HEAD? See files-backend.c,
++ split_head_update()
++ */
++ new_update = ref_transaction_add_update(
++ transaction, referent.buf, new_flags,
++ &update->new_oid, &update->old_oid,
++ update->msg);
++ new_update->parent_update = update;
++
++ /* files-backend sets REF_LOG_ONLY here. */
++ update->flags |= REF_NO_DEREF | REF_LOG_ONLY;
++ update->flags &= ~REF_HAVE_OLD;
++ }
++ }
++
++done:
++ strbuf_release(&referent);
++ return err;
++}
++
+static int reftable_transaction_prepare(struct ref_store *ref_store,
+ struct ref_transaction *transaction,
+ struct strbuf *errbuf)
+{
-+ /* XXX rewrite using the reftable transaction API. */
+ struct git_reftable_ref_store *refs =
+ (struct git_reftable_ref_store *)ref_store;
++ struct reftable_addition *add = NULL;
+ int err = refs->err;
+ if (err < 0) {
+ goto done;
+ }
++
+ err = reftable_stack_reload(refs->stack);
+ if (err) {
+ goto done;
+ }
+
++ err = reftable_stack_new_addition(&add, refs->stack);
++ if (err) {
++ goto done;
++ }
++
++ err = fixup_symrefs(ref_store, transaction);
++ if (err) {
++ goto done;
++ }
++
++ transaction->backend_data = add;
++ transaction->state = REF_TRANSACTION_PREPARED;
++
+done:
++ if (err < 0) {
++ transaction->state = REF_TRANSACTION_CLOSED;
++ strbuf_addf(errbuf, "reftable: transaction prepare: %s",
++ reftable_error_str(err));
++ }
++
+ return err;
+}
+
@@ refs/reftable-backend.c (new)
+ struct ref_transaction *transaction,
+ struct strbuf *err)
+{
-+ struct git_reftable_ref_store *refs =
-+ (struct git_reftable_ref_store *)ref_store;
-+ (void)refs;
++ struct reftable_addition *add =
++ (struct reftable_addition *)transaction->backend_data;
++ reftable_addition_destroy(add);
++ transaction->backend_data = NULL;
+ return 0;
+}
+
@@ refs/reftable-backend.c (new)
+
+static int ref_update_cmp(const void *a, const void *b)
+{
-+ return strcmp(((struct ref_update *)a)->refname,
-+ ((struct ref_update *)b)->refname);
++ return strcmp((*(struct ref_update **)a)->refname,
++ (*(struct ref_update **)b)->refname);
+}
+
+static int write_transaction_table(struct reftable_writer *writer, void *arg)
@@ refs/reftable-backend.c (new)
+ QSORT(sorted, transaction->nr, ref_update_cmp);
+ reftable_writer_set_limits(writer, ts, ts);
+
-+ for (i = 0; i < transaction->nr; i++) {
-+ struct ref_update *u = sorted[i];
-+ if (u->flags & REF_HAVE_OLD) {
-+ err = reftable_check_old_oid(transaction->ref_store,
-+ u->refname, &u->old_oid);
-+ if (err < 0) {
-+ goto exit;
-+ }
-+ }
-+ }
+
+ for (i = 0; i < transaction->nr; i++) {
+ struct ref_update *u = sorted[i];
@@ refs/reftable-backend.c (new)
+ log->update_index = ts;
+ log->message = u->msg;
+
++ if (u->flags & REF_LOG_ONLY) {
++ continue;
++ }
++
+ if (u->flags & REF_HAVE_NEW) {
-+ struct object_id out_oid;
-+ int out_flags = 0;
-+ /* Memory owned by refs_resolve_ref_unsafe, no need to
-+ * free(). */
-+ const char *resolved = refs_resolve_ref_unsafe(
-+ transaction->ref_store, u->refname, 0, &out_oid,
-+ &out_flags);
+ struct reftable_ref_record ref = { NULL };
+ struct object_id peeled;
-+ int peel_error = peel_object(&u->new_oid, &peeled);
+
-+ ref.ref_name =
-+ (char *)(resolved ? resolved : u->refname);
-+ log->ref_name = ref.ref_name;
++ int peel_error = peel_object(&u->new_oid, &peeled);
++ ref.ref_name = (char *)u->refname;
+
+ if (!is_null_oid(&u->new_oid)) {
+ ref.value = u->new_oid.hash;
@@ refs/reftable-backend.c (new)
+
+ err = reftable_writer_add_ref(writer, &ref);
+ if (err < 0) {
-+ goto exit;
++ goto done;
+ }
+ }
+ }
@@ refs/reftable-backend.c (new)
+ err = reftable_writer_add_log(writer, &logs[i]);
+ clear_reftable_log_record(&logs[i]);
+ if (err < 0) {
-+ goto exit;
++ goto done;
+ }
+ }
+
-+exit:
++done:
+ free(logs);
+ free(sorted);
+ return err;
+}
+
-+static int reftable_transaction_commit(struct ref_store *ref_store,
++static int reftable_transaction_finish(struct ref_store *ref_store,
+ struct ref_transaction *transaction,
+ struct strbuf *errmsg)
+{
-+ struct git_reftable_ref_store *refs =
-+ (struct git_reftable_ref_store *)ref_store;
++ struct reftable_addition *add =
++ (struct reftable_addition *)transaction->backend_data;
+ int err = 0;
-+ if (refs->err < 0) {
-+ return refs->err;
++ int i;
++
++ for (i = 0; i < transaction->nr; i++) {
++ struct ref_update *u = transaction->updates[i];
++ if (u->flags & REF_HAVE_OLD) {
++ err = reftable_check_old_oid(transaction->ref_store,
++ u->refname, &u->old_oid);
++ if (err < 0) {
++ goto done;
++ }
++ }
+ }
+
-+ err = reftable_stack_add(refs->stack, &write_transaction_table,
-+ transaction);
++ err = reftable_addition_add(add, &write_transaction_table, transaction);
+ if (err < 0) {
++ goto done;
++ }
++
++ err = reftable_addition_commit(add);
++
++done:
++ reftable_addition_destroy(add);
++ transaction->state = REF_TRANSACTION_CLOSED;
++ transaction->backend_data = NULL;
++ if (err) {
+ strbuf_addf(errmsg, "reftable: transaction failure: %s",
+ reftable_error_str(err));
+ return -1;
+ }
-+
-+ return 0;
++ return err;
+}
+
-+static int reftable_transaction_finish(struct ref_store *ref_store,
-+ struct ref_transaction *transaction,
-+ struct strbuf *err)
++static int
++reftable_transaction_initial_commit(struct ref_store *ref_store,
++ struct ref_transaction *transaction,
++ struct strbuf *errmsg)
+{
-+ return reftable_transaction_commit(ref_store, transaction, err);
++ return reftable_transaction_finish(ref_store, transaction, errmsg);
+}
+
+struct write_pseudoref_arg {
@@ refs/reftable-backend.c (new)
+ int err = reftable_stack_read_ref(arg->stack, arg->oldname, &ref);
+
+ if (err) {
-+ goto exit;
++ goto done;
+ }
+
+ /* XXX do ref renames overwrite the target? */
+ if (reftable_stack_read_ref(arg->stack, arg->newname, &ref) == 0) {
-+ goto exit;
++ goto done;
+ }
+
+ free(ref.ref_name);
@@ refs/reftable-backend.c (new)
+
+ err = reftable_writer_add_refs(writer, todo, 2);
+ if (err < 0) {
-+ goto exit;
++ goto done;
+ }
+ }
+
@@ refs/reftable-backend.c (new)
+ clear_reftable_log_record(&todo[1]);
+
+ if (err < 0) {
-+ goto exit;
++ goto done;
+ }
+
+ } else {
+ /* XXX symrefs? */
+ }
+
-+exit:
++done:
+ reftable_ref_record_clear(&ref);
+ return err;
+}
@@ refs/reftable-backend.c (new)
+ (struct reftable_reflog_ref_iterator *)ref_iterator;
+
+ while (1) {
-+ int err = reftable_iterator_next_log(ri->iter, &ri->log);
++ int err = reftable_iterator_next_log(&ri->iter, &ri->log);
+ if (err > 0) {
+ return ITER_DONE;
+ }
@@ refs/reftable-backend.c (new)
+ mt = reftable_stack_merged_table(refs->stack);
+ err = reftable_merged_table_seek_log(mt, &it, refname);
+ while (err == 0) {
-+ err = reftable_iterator_next_log(it, &log);
++ err = reftable_iterator_next_log(&it, &log);
+ if (err != 0) {
+ break;
+ }
@@ refs/reftable-backend.c (new)
+
+ while (err == 0) {
+ struct reftable_log_record log = { NULL };
-+ err = reftable_iterator_next_log(it, &log);
++ err = reftable_iterator_next_log(&it, &log);
+ if (err != 0) {
+ break;
+ }
@@ refs/reftable-backend.c (new)
+ if (refs->err < 0) {
+ return refs->err;
+ }
++ err = reftable_stack_reload(refs->stack);
++ if (err) {
++ goto done;
++ }
+
+ mt = reftable_stack_merged_table(refs->stack);
+ err = reftable_merged_table_seek_log(mt, &it, refname);
+ if (err < 0) {
-+ return err;
++ goto done;
+ }
+
+ while (1) {
+ struct object_id ooid;
+ struct object_id noid;
+
-+ int err = reftable_iterator_next_log(it, &log);
++ int err = reftable_iterator_next_log(&it, &log);
+ if (err < 0) {
-+ return err;
++ goto done;
+ }
+
+ if (err > 0 || strcmp(log.ref_name, refname)) {
@@ refs/reftable-backend.c (new)
+ add_log_tombstone(&arg, refname, log.update_index);
+ }
+ }
++ err = reftable_stack_add(refs->stack, &write_reflog_expiry_table, &arg);
++
++done:
+ reftable_log_record_clear(&log);
+ reftable_iterator_destroy(&it);
-+ err = reftable_stack_add(refs->stack, &write_reflog_expiry_table, &arg);
+ clear_log_tombstones(&arg);
+ return err;
+}
@@ refs/reftable-backend.c (new)
+ if (err > 0) {
+ errno = ENOENT;
+ err = -1;
-+ goto exit;
++ goto done;
+ }
+ if (err < 0) {
+ errno = reftable_error_to_errno(err);
+ err = -1;
-+ goto exit;
++ goto done;
+ }
+ if (ref.target != NULL) {
-+ /* XXX recurse? */
+ strbuf_reset(referent);
+ strbuf_addstr(referent, ref.target);
+ *type |= REF_ISSYMREF;
@@ refs/reftable-backend.c (new)
+ errno = EINVAL;
+ err = -1;
+ }
-+exit:
++done:
+ reftable_ref_record_clear(&ref);
+ return err;
+}
@@ refs/reftable-backend.c (new)
+ reftable_transaction_prepare,
+ reftable_transaction_finish,
+ reftable_transaction_abort,
-+ reftable_transaction_commit,
++ reftable_transaction_initial_commit,
+
+ reftable_pack_refs,
+ reftable_create_symref,
@@ refs/reftable-backend.c (new)
+ reftable_reflog_exists,
+ reftable_create_reflog,
+ reftable_delete_reflog,
-+ reftable_reflog_expire
++ reftable_reflog_expire,
+};
## repository.c ##
@@ t/t0031-reftable.sh (new)
+ test_cmp expect actual
+'
+
-+test_expect_success 'basic operation of reftable storage: commit, reflog, repack' '
++test_expect_success 'basic operation of reftable storage: commit, show-ref' '
+ initialize &&
+ test_commit file &&
+ test_write_lines refs/heads/master refs/tags/file >expect &&
+ git show-ref &&
+ git show-ref | cut -f2 -d" " > actual &&
-+ test_cmp actual expect &&
++ test_cmp actual expect
++'
++
++test_expect_success 'reflog, repack' '
++ initialize &&
+ for count in $(test_seq 1 10)
+ do
+ test_commit "number $count" file.t $count number-$count ||
@@ t/t0031-reftable.sh (new)
+ ls -1 .git/reftable >table-files &&
+ test_line_count = 2 table-files &&
+ git reflog refs/heads/master >output &&
-+ test_line_count = 11 output &&
-+ grep "commit (initial): file" output &&
++ test_line_count = 10 output &&
++ grep "commit (initial): number 1" output &&
+ grep "commit: number 10" output &&
+ git gc &&
+ git reflog refs/heads/master >output &&
@@ t/t0031-reftable.sh (new)
+ test -f file2
+'
+
++# cherry-pick uses a pseudo ref.
++test_expect_success 'rebase' '
++ initialize &&
++ test_commit message1 file1 &&
++ test_commit message2 file2 &&
++ git branch source &&
++ git checkout HEAD^ &&
++ test_commit message3 file3 &&
++ git rebase source &&
++ test -f file2
++'
++
+test_done
+
-: ----------- > 7: 6b248d5fdb4 Add GIT_DEBUG_REFS debugging mechanism
12: b96f0712d44 = 8: 54102355ce7 vcxproj: adjust for the reftable changes
13: 0e732d30b51 ! 9: 7764ebf0956 Add some reftable testing infrastructure @@ Metadata
Author: Han-Wen Nienhuys <[email protected]>
## Commit message ##
- Add some reftable testing infrastructure
+ Add reftable testing infrastructure
* Add GIT_TEST_REFTABLE environment var to control default ref storage
- * Add test_prerequisite REFTABLE. Skip t/t3210-pack-refs.sh for REFTABLE.
+ * Add test_prerequisite REFTABLE.
+
+ * Skip some tests that are incompatible:
+
+ * t3210-pack-refs.sh - does not apply
+ * t9903-bash-prompt - The bash mode reads .git/HEAD directly
+ * t1450-fsck.sh - manipulates .git/ directly to create invalid state
Major test failures:
- * t9903-bash-prompt - The bash mode reads .git/HEAD directly
* t1400-update-ref.sh - Reads from .git/{refs,logs} directly
* t1404-update-ref-errors.sh - Manipulates .git/refs/ directly
* t1405 - inspecs .git/ directly.
- * t1450-fsck.sh - manipulates .git/ directly to create invalid state
- * Rebase, cherry-pick: pseudo refs aren't written through the refs backend.
-
- Other tests by decreasing brokenness:
-
- t1407-worktree-ref-store.sh - 5 of 5
- t1413-reflog-detach.sh - 7 of 7
- t1415-worktree-refs.sh - 11 of 11
- t3908-stash-in-worktree.sh - 2 of 2
- t4207-log-decoration-colors.sh - 2 of 2
- t5515-fetch-merge-logic.sh - 17 of 17
- t5900-repo-selection.sh - 8 of 8
- t6016-rev-list-graph-simplify-history.sh - 12 of 12
- t5573-pull-verify-signatures.sh - 15 of 16
- t5612-clone-refspec.sh - 12 of 13
- t5514-fetch-multiple.sh - 11 of 12
- t6030-bisect-porcelain.sh - 64 of 71
- t5533-push-cas.sh - 15 of 17
- t5539-fetch-http-shallow.sh - 7 of 8
- t7413-submodule-is-active.sh - 7 of 8
- t2400-worktree-add.sh - 59 of 69
- t0100-previous.sh - 5 of 6
- t7419-submodule-set-branch.sh - 5 of 6
- t1404-update-ref-errors.sh - 44 of 53
- t6003-rev-list-topo-order.sh - 29 of 35
- t1409-avoid-packing-refs.sh - 9 of 11
- t5541-http-push-smart.sh - 31 of 38
- t5407-post-rewrite-hook.sh - 13 of 16
- t9903-bash-prompt.sh - 52 of 66
- t1414-reflog-walk.sh - 9 of 12
- t1507-rev-parse-upstream.sh - 21 of 28
- t2404-worktree-config.sh - 9 of 12
- t1505-rev-parse-last.sh - 5 of 7
- t7510-signed-commit.sh - 16 of 23
- t2018-checkout-branch.sh - 15 of 22
- (..etc)
-
+ t6030-bisect-porcelain.sh - 62 of 72
+ t2400-worktree-add.sh - 58 of 69
+ t3200-branch.sh - 58 of 145
+ t7406-submodule-update.sh - 54 of 54
+ t5601-clone.sh - 51 of 105
+ t9903-bash-prompt.sh - 50 of 66
+ t1404-update-ref-errors.sh - 44 of 53
+ t5510-fetch.sh - 40 of 171
+ t7400-submodule-basic.sh - 38 of 111
+ t3514-cherry-pick-revert-gpg.sh - 36 of 36
+ ..
@@ refs.c: struct ref_store *get_main_ref_store(struct repository *r)
- DEFAULT_REF_STORAGE,
+ default_ref_storage(),
REF_STORE_ALL_CAPS);
- return r->refs_private;
- }
+ if (getenv("GIT_DEBUG_REFS")) {
+ r->refs_private = debug_wrap(r->refs_private);
@@ refs.c: struct ref_store *get_submodule_ref_store(const char *submodule)
goto done;
@@ t/t1409-avoid-packing-refs.sh: test_description='avoid rewriting packed-refs unn
# shouldn't upset readers, and it should be omitted if the file is # ever rewritten.
+ ## t/t1450-fsck.sh ##
+@@ t/t1450-fsck.sh: test_description='git fsck random collection of tests +
+ . ./test-lib.sh
+
++if test_have_prereq REFTABLE
++then
++ skip_all='skipping tests; incompatible with reftable'
++ test_done
++fi
++
+ test_expect_success setup '
+ test_oid_init &&
+ git config gc.auto 0 &&
+
## t/t3210-pack-refs.sh ##
@@ t/t3210-pack-refs.sh: semantic is still the same.
'
@@ t/t3210-pack-refs.sh: semantic is still the same.
git config core.logallrefupdates true
'
+ ## t/t9903-bash-prompt.sh ##
+@@ t/t9903-bash-prompt.sh: test_description='test git-specific bash prompt functions' +
+ . ./lib-bash.sh
+
++if test_have_prereq REFTABLE
++then
++ skip_all='skipping tests; incompatible with reftable'
++ test_done
++fi
++
+ . "$GIT_BUILD_DIR/contrib/completion/git-prompt.sh"
+
+ actual="$TRASH_DIRECTORY/actual"
+
## t/test-lib.sh ##
@@ t/test-lib.sh: FreeBSD)
;;
Signature · 1 line
gitgitgadget