Adds reftable library code from https://github.com/hanwen/reftable.
cover Adds reftable library code from https://github.com/hanwen/reftable.
The reftable format is described in Documentation/technical/reftable.txt.
This is a fully reentrant implementation of reading and writing the reftable file format, and should be suitable for embedding in libgit2 too. It does not hook the code up to git to function as a ref storage backend yet.
Han-Wen Nienhuys (19):
hash.h: provide constants for the hash IDs
reftable: RFC: add LICENSE
reftable: add error related functionality
reftable: utility functions
reftable: add blocksource, an abstraction for random access reads
reftable: (de)serialization for the polymorphic record type. Provide zlib's uncompress2 from compat/zlib-compat.c
reftable: reading/writing blocks
reftable: a generic binary tree implementation
reftable: write reftable files
reftable: generic interface to tables
reftable: read reftable files
reftable: reftable file level tests
reftable: add a heap-based priority queue for reftable records
reftable: add merged table view
reftable: implement refname validation
reftable: implement stack, a mutable database of reftable files.
reftable: add dump utility
Add "test-tool dump-reftable" command.
Makefile | 53 +- ci/lib.sh | 1 + compat/.gitattributes | 1 + compat/zlib-uncompress2.c | 92 ++ config.mak.uname | 1 + configure.ac | 13 + contrib/buildsystems/CMakeLists.txt | 14 +- contrib/buildsystems/Generators/Vcxproj.pm | 11 +- hash.h | 6 + object-file.c | 7 +- reftable/LICENSE | 31 + reftable/basics.c | 128 ++ reftable/basics.h | 60 + reftable/basics_test.c | 98 ++ reftable/block.c | 448 +++++++ reftable/block.h | 127 ++ reftable/block_test.c | 120 ++ reftable/blocksource.c | 148 +++ reftable/blocksource.h | 22 + reftable/constants.h | 21 + reftable/dump.c | 107 ++ reftable/error.c | 41 + reftable/generic.c | 169 +++ reftable/generic.h | 32 + reftable/iter.c | 194 +++ reftable/iter.h | 69 + reftable/merged.c | 362 +++++ reftable/merged.h | 35 + reftable/merged_test.c | 292 ++++ reftable/pq.c | 115 ++ reftable/pq.h | 32 + reftable/pq_test.c | 72 + reftable/publicbasics.c | 58 + reftable/reader.c | 801 +++++++++++ reftable/reader.h | 66 + reftable/readwrite_test.c | 652 +++++++++ reftable/record.c | 1212 +++++++++++++++++ reftable/record.h | 139 ++ reftable/record_test.c | 412 ++++++ reftable/refname.c | 209 +++ reftable/refname.h | 29 + reftable/refname_test.c | 102 ++ reftable/reftable-blocksource.h | 49 + reftable/reftable-error.h | 62 + reftable/reftable-generic.h | 47 + reftable/reftable-iterator.h | 39 + reftable/reftable-malloc.h | 18 + reftable/reftable-merged.h | 72 + reftable/reftable-reader.h | 101 ++ reftable/reftable-record.h | 114 ++ reftable/reftable-stack.h | 128 ++ reftable/reftable-tests.h | 23 + reftable/reftable-writer.h | 148 +++ reftable/reftable.c | 115 ++ reftable/stack.c | 1396 ++++++++++++++++++++ reftable/stack.h | 41 + reftable/stack_test.c | 949 +++++++++++++ reftable/system.h | 24 + reftable/test_framework.c | 23 + reftable/test_framework.h | 53 + reftable/tree.c | 63 + reftable/tree.h | 34 + reftable/tree_test.c | 61 + reftable/writer.c | 690 ++++++++++ reftable/writer.h | 50 + t/helper/test-reftable.c | 21 + t/helper/test-tool.c | 4 +- t/helper/test-tool.h | 2 + t/t0032-reftable-unittest.sh | 15 + 69 files changed, 10932 insertions(+), 12 deletions(-) create mode 100644 compat/.gitattributes create mode 100644 compat/zlib-uncompress2.c create mode 100644 reftable/LICENSE create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/basics_test.c create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/block_test.c create mode 100644 reftable/blocksource.c create mode 100644 reftable/blocksource.h create mode 100644 reftable/constants.h create mode 100644 reftable/dump.c create mode 100644 reftable/error.c create mode 100644 reftable/generic.c create mode 100644 reftable/generic.h create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/merged_test.c create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/pq_test.c create mode 100644 reftable/publicbasics.c create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/readwrite_test.c create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/record_test.c create mode 100644 reftable/refname.c create mode 100644 reftable/refname.h create mode 100644 reftable/refname_test.c create mode 100644 reftable/reftable-blocksource.h create mode 100644 reftable/reftable-error.h create mode 100644 reftable/reftable-generic.h create mode 100644 reftable/reftable-iterator.h create mode 100644 reftable/reftable-malloc.h create mode 100644 reftable/reftable-merged.h create mode 100644 reftable/reftable-reader.h create mode 100644 reftable/reftable-record.h create mode 100644 reftable/reftable-stack.h create mode 100644 reftable/reftable-tests.h create mode 100644 reftable/reftable-writer.h create mode 100644 reftable/reftable.c create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/stack_test.c create mode 100644 reftable/system.h create mode 100644 reftable/test_framework.c create mode 100644 reftable/test_framework.h create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100644 reftable/tree_test.c create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h create mode 100644 t/helper/test-reftable.c create mode 100755 t/t0032-reftable-unittest.sh
base-commit: c4203212e360b25a1c69467b5a8437d45a373cac
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-git-1081%2Fhanwen%2Fjust-library-v1
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-git-1081/hanwen/just-library-v1
Pull-Request: https://github.com/git/git/pull/1081
Signature · 1 line
gitgitgadget
PATCH 1/19 hash.h: provide constants for the hash IDs
From: Han-Wen Nienhuys <[email protected]>
This will simplify referencing them from code that is not deeply integrated with Git, in particular, the reftable library.
--- hash.h | 6 ++++++ object-file.c | 7 ++----- 2 files changed, 8 insertions(+), 5 deletions(-)
hash.h +6 -0
@@ -95,12 +95,18 @@ static inline void git_SHA256_Clone(git_SHA256_CTX *dst, const git_SHA256_CTX *s/* Number of algorithms supported (including unknown). */ #define GIT_HASH_NALGOS (GIT_HASH_SHA256 + 1) +/* "sha1", big-endian */ +#define GIT_SHA1_FORMAT_ID 0x73686131 +/* The length in bytes and in hex digits of an object name (SHA-1 value). */ #define GIT_SHA1_RAWSZ 20#define GIT_SHA1_HEXSZ (2 * GIT_SHA1_RAWSZ)/* The block size of SHA-1. */ #define GIT_SHA1_BLKSZ 64 +/* "s256", big-endian */ +#define GIT_SHA256_FORMAT_ID 0x73323536 +/* The length in bytes and in hex digits of an object name (SHA-256 value). */ #define GIT_SHA256_RAWSZ 32#define GIT_SHA256_HEXSZ (2 * GIT_SHA256_RAWSZ)
object-file.c +2 -5
@@ -164,7 +164,6 @@ static void git_hash_unknown_final_oid(struct object_id *oid, git_hash_ctx *ctx)BUG ("trying to finalize unknown hash" ); } -const struct git_hash_algo hash_algos[GIT_HASH_NALGOS ] = { {NULL ,
@@ -183,8 +182,7 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = { }, {"sha1" , -/* "sha1", big-endian */ -0x73686131 , +GIT_SHA1_FORMAT_ID ,GIT_SHA1_RAWSZ ,GIT_SHA1_HEXSZ , GIT_SHA1_BLKSZ,
@@ -199,8 +197,7 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = { }, {"sha256" , -/* "s256", big-endian */ -0x73323536 , +GIT_SHA256_FORMAT_ID ,GIT_SHA256_RAWSZ ,GIT_SHA256_HEXSZ , GIT_SHA256_BLKSZ,
Signature · 1 line
gitgitgadget
PATCH 2/19 reftable: RFC: add LICENSE
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The objective of this code is to be usable as a C library, so it can be reused in libgit2.
This is currently using a BSD license as it is the liberal license I could find, but this could be changed to whatever fits the stated goal above.
This code is currently imported from github.com/hanwen/reftable. Once this code lands in git.git, the C code will be removed from github.com/hanwen/reftable, and the git.git code will be the source of truth.
--- reftable/LICENSE | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 reftable/LICENSE
reftable/LICENSE new file +31 -0
@@ -0,0 +1,31 @@ +BSD License + +Copyright (c) 2020, Google LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of Google LLC nor the names of its contributors may +be used to endorse or promote products derived from this software +without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Signature · 1 line
gitgitgadget
PATCH 3/19 reftable: add error related functionality
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The reftable/ directory is structured as a library, so it cannot crash on misuse. Instead, it returns an error codes.
In addition, the error code can be used to signal conditions from lower levels of the library to be handled by higher levels of the library. For example, a transaction might legitimately write an empty reftable file, but in that case, we'd want to shortcut the transaction overhead.
--- reftable/error.c | 41 ++++++++++++++++++++++++++ reftable/reftable-error.h | 62 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 reftable/error.c create mode 100644 reftable/reftable-error.h
reftable/error.c new file +41 -0
@@ -0,0 +1,41 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "reftable-error.h" + +#include <stdio.h> + +const char *reftable_error_str (int err) +{ +static char buf[250 ]; +switch (err) { +case REFTABLE_IO_ERROR : +return "I/O error" ; +case REFTABLE_FORMAT_ERROR : +return "corrupt reftable file" ; +case REFTABLE_NOT_EXIST_ERROR : +return "file does not exist" ; +case REFTABLE_LOCK_ERROR : +return "data is outdated" ; +case REFTABLE_API_ERROR : +return "misuse of the reftable API" ; +case REFTABLE_ZLIB_ERROR : +return "zlib failure" ; +case REFTABLE_NAME_CONFLICT : +return "file/directory conflict" ; +case REFTABLE_EMPTY_TABLE_ERROR : +return "wrote empty table" ; +case REFTABLE_REFNAME_ERROR : +return "invalid refname" ; +case -1 : +return "general error" ; +default : +snprintf (buf,sizeof (buf),"unknown error code %d" , err); +return buf; + } +}
reftable/reftable-error.h new file +62 -0
62 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH 4/19 reftable: utility functions
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This commit provides basic utility classes for the reftable library.
--- Makefile | 25 +++- contrib/buildsystems/CMakeLists.txt | 14 ++- contrib/buildsystems/Generators/Vcxproj.pm | 11 +- reftable/basics.c | 128 +++++++++++++++++++++ reftable/basics.h | 60 ++++++++++ reftable/basics_test.c | 98 ++++++++++++++++ reftable/publicbasics.c | 58 ++++++++++ reftable/reftable-malloc.h | 18 +++ reftable/reftable-tests.h | 22 ++++ reftable/system.h | 24 ++++ reftable/test_framework.c | 23 ++++ reftable/test_framework.h | 53 +++++++++ t/helper/test-reftable.c | 9 ++ t/helper/test-tool.c | 3 +- t/helper/test-tool.h | 1 + t/t0032-reftable-unittest.sh | 15 +++ 16 files changed, 555 insertions(+), 7 deletions(-) create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/basics_test.c create mode 100644 reftable/publicbasics.c create mode 100644 reftable/reftable-malloc.h create mode 100644 reftable/reftable-tests.h create mode 100644 reftable/system.h create mode 100644 reftable/test_framework.c create mode 100644 reftable/test_framework.h create mode 100644 t/helper/test-reftable.c create mode 100755 t/t0032-reftable-unittest.sh
Makefile +22 -3
@@ -743,6 +743,7 @@ TEST_BUILTINS_OBJS += test-read-cache.o TEST_BUILTINS_OBJS += test-read-graph.o TEST_BUILTINS_OBJS += test-read-midx.o TEST_BUILTINS_OBJS += test-ref-store.o +TEST_BUILTINS_OBJS += test-reftable.o TEST_BUILTINS_OBJS += test-regex.o TEST_BUILTINS_OBJS += test-repository.o TEST_BUILTINS_OBJS += test-revision-walking.o
@@ -821,6 +822,8 @@ TEST_SHELL_PATH = $(SHELL_PATH) LIB_FILE = libgit.a XDIFF_LIB = xdiff/lib.a +REFTABLE_LIB = reftable/libreftable.a +REFTABLE_TEST_LIB = reftable/libreftable_test.a GENERATED_H += command-list.h GENERATED_H += config-list.h
@@ -1195,7 +1198,7 @@ THIRD_PARTY_SOURCES += compat/regex/% THIRD_PARTY_SOURCES += sha1collisiondetection/% THIRD_PARTY_SOURCES += sha1dc/% -GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) +GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) EXTLIBS = GIT_USER_AGENT = git/$(GIT_VERSION)
@@ -2446,7 +2449,15 @@ XDIFF_OBJS += xdiff/xutils.o .PHONY: xdiff-objs xdiff-objs: $(XDIFF_OBJS) +REFTABLE_OBJS += reftable/basics.o +REFTABLE_OBJS += reftable/error.o +REFTABLE_OBJS += reftable/publicbasics.o + +REFTABLE_TEST_OBJS += reftable/test_framework.o +REFTABLE_TEST_OBJS += reftable/basics_test.o + TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) + .PHONY: test-objs test-objs: $(TEST_OBJS)
@@ -2462,6 +2473,8 @@ OBJECTS += $(PROGRAM_OBJS) OBJECTS += $(TEST_OBJS) OBJECTS += $(XDIFF_OBJS) OBJECTS += $(FUZZ_OBJS) +OBJECTS += $(REFTABLE_OBJS) $(REFTABLE_TEST_OBJS) + ifndef NO_CURL OBJECTS += http.o http-walker.o remote-curl.o endif
@@ -2612,6 +2625,12 @@ $(LIB_FILE): $(LIB_OBJS) $(XDIFF_LIB): $(XDIFF_OBJS) $(QUIET_AR)$(AR) $(ARFLAGS) $@ $^ +$(REFTABLE_LIB): $(REFTABLE_OBJS) + $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ + +$(REFTABLE_TEST_LIB): $(REFTABLE_TEST_OBJS) + $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ + export DEFAULT_EDITOR DEFAULT_PAGER Documentation/GIT-EXCLUDED-PROGRAMS: FORCE
@@ -2904,7 +2923,7 @@ perf: all t/helper/test-tool$X: $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) -t/helper/test-%$X: t/helper/test-%.o GIT-LDFLAGS $(GITLIBS) +t/helper/test-%$X: t/helper/test-%.o GIT-LDFLAGS $(GITLIBS) $(REFTABLE_TEST_LIB) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(filter %.a,$^) $(LIBS) check-sha1:: t/helper/test-tool$X
@@ -3234,7 +3253,7 @@ cocciclean: clean: profile-clean coverage-clean cocciclean $(RM) *.res $(RM) $(OBJECTS) - $(RM) $(LIB_FILE) $(XDIFF_LIB) + $(RM) $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) $(REFTABLE_TEST_LIB) $(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) git$X $(RM) $(TEST_PROGRAMS) $(RM) $(FUZZ_PROGRAMS)
contrib/buildsystems/CMakeLists.txt +12 -2
@@ -640,6 +640,12 @@ parse_makefile_for_sources(libxdiff_SOURCES "XDIFF_OBJS") list(TRANSFORM libxdiff_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") add_library(xdiff STATIC ${libxdiff_SOURCES}) +#reftable +parse_makefile_for_sources(reftable_SOURCES "REFTABLE_OBJS") + +list(TRANSFORM reftable_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") +add_library(reftable STATIC ${reftable_SOURCES}) + if(WIN32) if(NOT MSVC)#use windres when compiling with gcc and clang add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/git.res
@@ -662,7 +668,7 @@ endif() #link all required libraries to common-main add_library(common-main OBJECT ${CMAKE_SOURCE_DIR}/common-main.c) -target_link_libraries(common-main libgit xdiff ${ZLIB_LIBRARIES}) +target_link_libraries(common-main libgit xdiff reftable ${ZLIB_LIBRARIES}) if(Intl_FOUND) target_link_libraries(common-main ${Intl_LIBRARIES}) endif()
@@ -902,11 +908,15 @@ if(BUILD_TESTING) add_executable(test-fake-ssh ${CMAKE_SOURCE_DIR}/t/helper/test-fake-ssh.c) target_link_libraries(test-fake-ssh common-main) +#reftable-tests +parse_makefile_for_sources(test-reftable_SOURCES "REFTABLE_TEST_OBJS") +list(TRANSFORM test-reftable_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") + #test-tool parse_makefile_for_sources(test-tool_SOURCES "TEST_BUILTINS_OBJS") list(TRANSFORM test-tool_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/t/helper/") -add_executable(test-tool ${CMAKE_SOURCE_DIR}/t/helper/test-tool.c ${test-tool_SOURCES}) +add_executable(test-tool ${CMAKE_SOURCE_DIR}/t/helper/test-tool.c ${test-tool_SOURCES} ${test-reftable_SOURCES}) target_link_libraries(test-tool common-main) set_target_properties(test-fake-ssh test-tool
contrib/buildsystems/Generators/Vcxproj.pm +10 -1
@@ -77,7 +77,7 @@ sub createProject { my $libs_release = "\n "; my $libs_debug = "\n "; if (!$static_library) { - $libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); + $libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib|reftable\/libreftable\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); $libs_debug = $libs_release; $libs_debug =~ s/zlib\.lib/zlibd\.lib/g; $libs_debug =~ s/libexpat\.lib/libexpatd\.lib/g;
@@ -232,6 +232,7 @@ EOM EOM if (!$static_library || $target =~ 'vcs-svn' || $target =~ 'xdiff') { my $uuid_libgit = $$build_structure{"LIBS_libgit_GUID"}; + my $uuid_libreftable = $$build_structure{"LIBS_reftable/libreftable_GUID"}; my $uuid_xdiff_lib = $$build_structure{"LIBS_xdiff/lib_GUID"}; print F << "EOM";
@@ -241,6 +242,14 @@ EOM <ReferenceOutputAssembly>false</ReferenceOutputAssembly> </ProjectReference> EOM + if (!($name =~ /xdiff|libreftable/)) { + print F << "EOM"; + <ProjectReference Include="$cdup\\reftable\\libreftable\\libreftable.vcxproj"> + <Project>$uuid_libreftable</Project> + <ReferenceOutputAssembly>false</ReferenceOutputAssembly> + </ProjectReference> +EOM + } if (!($name =~ 'xdiff')) { print F << "EOM"; <ProjectReference Include="$cdup\\xdiff\\lib\\xdiff_lib.vcxproj">
reftable/basics.c new file +128 -0
128 lines not on this page · raw message
reftable/basics.h new file +60 -0
60 lines not on this page · raw message
reftable/basics_test.c new file +98 -0
98 lines not on this page · raw message
reftable/publicbasics.c new file +58 -0
58 lines not on this page · raw message
reftable/reftable-malloc.h new file +18 -0
@@ -0,0 +1,18 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_H +#define REFTABLE_H + +#include <stddef.h> + +/* Overrides the functions to use for memory management. */ +void reftable_set_alloc (void *(*malloc)(size_t ), +void *(*realloc)(void *,size_t ),void (*free)(void *)); + +#endif
reftable/reftable-tests.h new file +22 -0
@@ -0,0 +1,22 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_TESTS_H +#define REFTABLE_TESTS_H + +int basics_test_main (int argc,const char **argv); +int block_test_main (int argc,const char **argv); +int merged_test_main (int argc,const char **argv); +int record_test_main (int argc,const char **argv); +int refname_test_main (int argc,const char **argv); +int reftable_test_main (int argc,const char **argv); +int stack_test_main (int argc,const char **argv); +int tree_test_main (int argc,const char **argv); +int reftable_dump_main (int argc,char *const *argv); + +#endif
reftable/system.h new file +24 -0
@@ -0,0 +1,24 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef SYSTEM_H +#define SYSTEM_H + +/* This header glues the reftable library to the rest of Git */ + +#include "git-compat-util.h" +#include "strbuf.h" +#include "hash.h" /* hash ID, sizes.*/ +#include "dir.h" /* remove_dir_recursively, for tests.*/ + +#include <zlib.h> + +struct strbuf ; +int hash_size (uint32_t id); + +#endif
reftable/test_framework.c new file +23 -0
@@ -0,0 +1,23 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "system.h" +#include "test_framework.h" + +#include "basics.h" + +void set_test_hash (uint8_t *p,int i) +{ +memset (p, (uint8_t )i,hash_size (GIT_SHA1_FORMAT_ID )); +} + +ssize_t strbuf_add_void (void *b,const void *data,size_t sz) +{ +strbuf_add (b, data, sz); +return sz; +}
reftable/test_framework.h new file +53 -0
53 lines not on this page · raw message
t/helper/test-reftable.c new file +9 -0
@@ -0,0 +1,9 @@ +#include "reftable/reftable-tests.h" +#include "test-tool.h" + +int cmd__reftable (int argc,const char **argv) +{ +basics_test_main (argc, argv); + +return 0 ; +}
t/helper/test-tool.c +2 -1
@@ -53,13 +53,14 @@ static struct test_cmd cmds[] = { { "pcre2-config", cmd__pcre2_config }, { "pkt-line", cmd__pkt_line }, { "prio-queue", cmd__prio_queue }, - { "proc-receive", cmd__proc_receive}, + { "proc-receive", cmd__proc_receive }, { "progress", cmd__progress }, { "reach", cmd__reach }, { "read-cache", cmd__read_cache }, { "read-graph", cmd__read_graph }, { "read-midx", cmd__read_midx }, { "ref-store", cmd__ref_store }, + { "reftable", cmd__reftable }, { "regex", cmd__regex }, { "repository", cmd__repository }, { "revision-walking", cmd__revision_walking },
t/helper/test-tool.h +1 -0
@@ -49,6 +49,7 @@ int cmd__read_cache(int argc, const char **argv);int cmd__read_graph (int argc,const char **argv);int cmd__read_midx (int argc,const char **argv);int cmd__ref_store (int argc,const char **argv); +int cmd__reftable (int argc,const char **argv);int cmd__regex (int argc,const char **argv);int cmd__repository (int argc,const char **argv);int cmd__revision_walking (int argc,const char **argv);
t/t0032-reftable-unittest.sh new file +15 -0
@@ -0,0 +1,15 @@ +#!/bin/sh +# +# Copyright (c) 2020 Google LLC +# + +test_description='reftable unittests' + +. ./test-lib.sh + +test_expect_success 'unittests' ' + TMPDIR=$(pwd) && export TMPDIR && + test-tool reftable +' + +test_done
Signature · 1 line
gitgitgadget
PATCH 5/19 reftable: add blocksource, an abstraction for random access reads
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The reftable format is usually used with files for storage. However, we abstract away this using the blocksource data structure. This has two advantages:
* log blocks are zlib compressed, and handling them is simplified if we can discard byte segments from within the block layer.
* for unittests, it is useful to read and write in-memory. The blocksource allows us to abstract the data away from on-disk files.
--- Makefile | 1 + reftable/blocksource.c | 148 ++++++++++++++++++++++++++++++++ reftable/blocksource.h | 22 +++++ reftable/reftable-blocksource.h | 49 +++++++++++ 4 files changed, 220 insertions(+) create mode 100644 reftable/blocksource.c create mode 100644 reftable/blocksource.h create mode 100644 reftable/reftable-blocksource.h
Makefile +1 -0
@@ -2451,6 +2451,7 @@ xdiff-objs: $(XDIFF_OBJS) REFTABLE_OBJS += reftable/basics.o REFTABLE_OBJS += reftable/error.o +REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o REFTABLE_TEST_OBJS += reftable/test_framework.o
reftable/blocksource.c new file +148 -0
148 lines not on this page · raw message
reftable/blocksource.h new file +22 -0
@@ -0,0 +1,22 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef BLOCKSOURCE_H +#define BLOCKSOURCE_H + +#include "system.h" + +struct reftable_block_source ; + +/* Create an in-memory block source for reading reftables */ +void block_source_from_strbuf (struct reftable_block_source *bs, +struct strbuf *buf); + +struct reftable_block_source malloc_block_source (void ); + +#endif
reftable/reftable-blocksource.h new file +49 -0
49 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH 6/19 reftable: (de)serialization for the polymorphic record type.
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The reftable format is structured as a sequence of blocks, and each block contains a sequence of prefix-compressed key-value records. There are 4 types of records, and they have similarities in how they must be handled. This is achieved by introducing a polymorphic 'record' type that encapsulates ref, log, index and object records.
--- Makefile | 2 + reftable/constants.h | 21 + reftable/record.c | 1212 ++++++++++++++++++++++++++++++++++++ reftable/record.h | 139 +++++ reftable/record_test.c | 412 ++++++++++++ reftable/reftable-record.h | 114 ++++ t/helper/test-reftable.c | 2 +- 7 files changed, 1901 insertions(+), 1 deletion(-) create mode 100644 reftable/constants.h create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/record_test.c create mode 100644 reftable/reftable-record.h
Makefile +2 -0
@@ -2453,7 +2453,9 @@ REFTABLE_OBJS += reftable/basics.o REFTABLE_OBJS += reftable/error.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o +REFTABLE_OBJS += reftable/record.o +REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o REFTABLE_TEST_OBJS += reftable/basics_test.o
reftable/constants.h new file +21 -0
@@ -0,0 +1,21 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef CONSTANTS_H +#define CONSTANTS_H + +#define BLOCK_TYPE_LOG 'g' +#define BLOCK_TYPE_INDEX 'i' +#define BLOCK_TYPE_REF 'r' +#define BLOCK_TYPE_OBJ 'o' +#define BLOCK_TYPE_ANY 0 + +#define MAX_RESTARTS ((1 << 16) - 1) +#define DEFAULT_BLOCK_SIZE 4096 + +#endif
reftable/record.c new file +1212 -0
1,212 lines not on this page · raw message
reftable/record.h new file +139 -0
139 lines not on this page · raw message
reftable/record_test.c new file +412 -0
412 lines not on this page · raw message
reftable/reftable-record.h new file +114 -0
114 lines not on this page · raw message
t/helper/test-reftable.c +1 -1
@@ -4,6 +4,6 @@int cmd__reftable (int argc,const char **argv) {basics_test_main (argc, argv); - +record_test_main (argc, argv);return 0 ; }
Signature · 1 line
gitgitgadget
PATCH 7/19 Provide zlib's uncompress2 from compat/zlib-compat.c
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This will be needed for reading reflog blocks in reftable.
--- Makefile | 7 +++ ci/lib.sh | 1 + compat/.gitattributes | 1 + compat/zlib-uncompress2.c | 92 +++++++++++++++++++++++++++++++++++++++ config.mak.uname | 1 + configure.ac | 13 ++++++ 6 files changed, 115 insertions(+) create mode 100644 compat/.gitattributes create mode 100644 compat/zlib-uncompress2.c
Makefile +7 -0
@@ -256,6 +256,8 @@ all:: # # Define NO_DEFLATE_BOUND if your zlib does not have deflateBound. # +# Define NO_UNCOMPRESS2 if your zlib does not have uncompress2. +# # Define NO_NORETURN if using buggy versions of gcc 4.6+ and profile feedback, # as the compiler can crash (http://gcc.gnu.org/bugzilla/show_bug.cgi?id=49299) #
@@ -1738,6 +1740,11 @@ ifdef NO_DEFLATE_BOUND BASIC_CFLAGS += -DNO_DEFLATE_BOUND endif +ifdef NO_UNCOMPRESS2 + BASIC_CFLAGS += -DNO_UNCOMPRESS2 + REFTABLE_OBJS += compat/zlib-uncompress2.o +endif + ifdef NO_POSIX_GOODIES BASIC_CFLAGS += -DNO_POSIX_GOODIES endif
ci/lib.sh +1 -0
@@ -224,6 +224,7 @@ linux-gcc-default) ;; Linux32) CC=gcc + MAKEFLAGS="$MAKEFLAGS NO_UNCOMPRESS2=1" ;; linux-musl) CC=gcc
compat/.gitattributes new file +1 -0
@@ -0,0 +1 @@ +/zlib-uncompress2.c whitespace=-indent-with-non-tab,-trailing-space
compat/zlib-uncompress2.c new file +92 -0
92 lines not on this page · raw message
config.mak.uname +1 -0
@@ -258,6 +258,7 @@ ifeq ($(uname_S),FreeBSD) FILENO_IS_A_MACRO = UnfortunatelyYes endif ifeq ($(uname_S),OpenBSD) + NO_UNCOMPRESS2 = YesPlease NO_STRCASESTR = YesPlease NO_MEMMEM = YesPlease USE_ST_TIMESPEC = YesPlease
configure.ac +13 -0
@@ -672,9 +672,22 @@ AC_LINK_IFELSE([ZLIBTEST_SRC], NO_DEFLATE_BOUND=yes]) LIBS="$old_LIBS" +AC_DEFUN([ZLIBTEST_UNCOMPRESS2_SRC], [ +AC_LANG_PROGRAM([#include <zlib.h>], + [uncompress2(NULL,NULL,NULL,NULL);])]) +AC_MSG_CHECKING([for uncompress2 in -lz]) +old_LIBS="$LIBS" +LIBS="$LIBS -lz" +AC_LINK_IFELSE([ZLIBTEST_UNCOMPRESS2_SRC], + [AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no]) + NO_UNCOMPRESS2=yes]) +LIBS="$old_LIBS" + GIT_UNSTASH_FLAGS($ZLIB_PATH) GIT_CONF_SUBST([NO_DEFLATE_BOUND]) +GIT_CONF_SUBST([NO_UNCOMPRESS2]) # # Define NEEDS_SOCKET if linking with libc is not enough (SunOS,
Signature · 1 line
gitgitgadget
PATCH 1/1 fixup! Provide zlib's uncompress2 from compat/zlib-compat.c
OpenBSD 7.0 (currently in beta) comes with zlib 1.2.11 and no longer needs this compatibility layer.
--- config.mak.uname | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
config.mak.uname +4 -1
@@ -256,7 +256,10 @@ ifeq ($(uname_S),FreeBSD) FILENO_IS_A_MACRO = UnfortunatelyYes endif ifeq ($(uname_S),OpenBSD) - NO_UNCOMPRESS2 = YesPlease + # Versions < 7.0 need compatibility layer + ifeq ($(shell expr "$(uname_R)" : "[1-6]\."),2) + NO_UNCOMPRESS2 = UnfortunatelyYes + endif NO_STRCASESTR = YesPlease NO_MEMMEM = YesPlease USE_ST_TIMESPEC = YesPlease
Signature · 1 line
2.33.0.481.g26d3bed244
PATCH 8/19 reftable: reading/writing blocks
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The reftable format is structured as a sequence of block. Within a block, records are prefix compressed, with an index of offsets for fully expand keys to enable binary search within blocks.
This commit provides the logic to read and write these blocks.
--- Makefile | 2 + reftable/block.c | 448 +++++++++++++++++++++++++++++++++++++++ reftable/block.h | 127 +++++++++++ reftable/block_test.c | 120 +++++++++++ t/helper/test-reftable.c | 1 + 5 files changed, 698 insertions(+) create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/block_test.c
Makefile +2 -0
@@ -2458,10 +2458,12 @@ xdiff-objs: $(XDIFF_OBJS) REFTABLE_OBJS += reftable/basics.o REFTABLE_OBJS += reftable/error.o +REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o REFTABLE_OBJS += reftable/record.o +REFTABLE_TEST_OBJS += reftable/block_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o REFTABLE_TEST_OBJS += reftable/basics_test.o
reftable/block.c new file +448 -0
448 lines not on this page · raw message
reftable/block.h new file +127 -0
127 lines not on this page · raw message
reftable/block_test.c new file +120 -0
120 lines not on this page · raw message
t/helper/test-reftable.c +1 -0
@@ -4,6 +4,7 @@int cmd__reftable (int argc,const char **argv) {basics_test_main (argc, argv); +block_test_main (argc, argv);record_test_main (argc, argv);return 0 ; }
Signature · 1 line
gitgitgadget
PATCH 9/19 reftable: a generic binary tree implementation
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The reftable format includes support for an (OID => ref) map. This map can speed up visibility and reachability checks. In particular, various operations along the fetch/push path within Gerrit have ben sped up by using this structure.
The map is constructed with help of a binary tree. Object IDs are hashes, so they are uniformly distributed. Hence, the tree does not attempt forced rebalancing.
--- Makefile | 4 ++- reftable/tree.c | 63 ++++++++++++++++++++++++++++++++++++++++ reftable/tree.h | 34 ++++++++++++++++++++++ reftable/tree_test.c | 61 ++++++++++++++++++++++++++++++++++++++ t/helper/test-reftable.c | 1 + 5 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100644 reftable/tree_test.c
Makefile +3 -1
@@ -2462,11 +2462,13 @@ REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o REFTABLE_OBJS += reftable/record.o +REFTABLE_OBJS += reftable/tree.o +REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o -REFTABLE_TEST_OBJS += reftable/basics_test.o +REFTABLE_TEST_OBJS += reftable/tree_test.o TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS))
reftable/tree.c new file +63 -0
63 lines not on this page · raw message
reftable/tree.h new file +34 -0
@@ -0,0 +1,34 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef TREE_H +#define TREE_H + +/* tree_node is a generic binary search tree. */ +struct tree_node { +void *key; +struct tree_node *left, *right; +}; + +/* looks for `key` in `rootp` using `compare` as comparison function. If insert +* is set, insert the key if it's not found. Else, return NULL. +*/ +struct tree_node *tree_search (void *key,struct tree_node **rootp, +int (*compare)(const void *,const void *), +int insert); + +/* performs an infix walk of the tree. */ +void infix_walk (struct tree_node *t,void (*action)(void *arg,void *key), +void *arg); + +/* +* deallocates the tree nodes recursively. Keys should be deallocated separately +* by walking over the tree. */ +void tree_free (struct tree_node *t); + +#endif
reftable/tree_test.c new file +61 -0
61 lines not on this page · raw message
t/helper/test-reftable.c +1 -0
@@ -6,5 +6,6 @@ int cmd__reftable(int argc, const char **argv)basics_test_main (argc, argv);block_test_main (argc, argv);record_test_main (argc, argv); +tree_test_main (argc, argv);return 0 ; }
Signature · 1 line
gitgitgadget
PATCH 10/19 reftable: write reftable files
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
--- Makefile | 1 + reftable/reftable-writer.h | 148 ++++++++ reftable/writer.c | 690 +++++++++++++++++++++++++++++++++++++ reftable/writer.h | 50 +++ 4 files changed, 889 insertions(+) create mode 100644 reftable/reftable-writer.h create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h
Makefile +1 -0
@@ -2463,6 +2463,7 @@ REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o REFTABLE_OBJS += reftable/record.o REFTABLE_OBJS += reftable/tree.o +REFTABLE_OBJS += reftable/writer.o REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o
reftable/reftable-writer.h new file +148 -0
148 lines not on this page · raw message
reftable/writer.c new file +690 -0
690 lines not on this page · raw message
reftable/writer.h new file +50 -0
50 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH 11/19 reftable: generic interface to tables
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
--- Makefile | 3 + reftable/generic.c | 169 +++++++++++++++++++++++++++++++++++ reftable/generic.h | 32 +++++++ reftable/reftable-generic.h | 47 ++++++++++ reftable/reftable-iterator.h | 39 ++++++++ reftable/reftable.c | 115 ++++++++++++++++++++++++ 6 files changed, 405 insertions(+) create mode 100644 reftable/generic.c create mode 100644 reftable/generic.h create mode 100644 reftable/reftable-generic.h create mode 100644 reftable/reftable-iterator.h create mode 100644 reftable/reftable.c
Makefile +3 -0
@@ -2462,6 +2462,9 @@ REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o REFTABLE_OBJS += reftable/record.o +REFTABLE_OBJS += reftable/refname.o +REFTABLE_OBJS += reftable/generic.o +REFTABLE_OBJS += reftable/stack.o REFTABLE_OBJS += reftable/tree.o REFTABLE_OBJS += reftable/writer.o
reftable/generic.c new file +169 -0
169 lines not on this page · raw message
reftable/generic.h new file +32 -0
@@ -0,0 +1,32 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef GENERIC_H +#define GENERIC_H + +#include "record.h" +#include "reftable-generic.h" + +/* generic interface to reftables */ +struct reftable_table_vtable { +int (*seek_record)(void *tab,struct reftable_iterator *it, +struct reftable_record *); +uint32_t (*hash_id)(void *tab); +uint64_t (*min_update_index)(void *tab); +uint64_t (*max_update_index)(void *tab); +}; + +struct reftable_iterator_vtable { +int (*next)(void *iter_arg,struct reftable_record *rec); +void (*close)(void *iter_arg); +}; + +void iterator_set_empty (struct reftable_iterator *it); +int iterator_next (struct reftable_iterator *it,struct reftable_record *rec); + +#endif
reftable/reftable-generic.h new file +47 -0
47 lines not on this page · raw message
reftable/reftable-iterator.h new file +39 -0
@@ -0,0 +1,39 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_ITERATOR_H +#define REFTABLE_ITERATOR_H + +#include "reftable-record.h" + +struct reftable_iterator_vtable ; + +/* iterator is the generic interface for walking over data stored in a +* reftable. +*/ +struct reftable_iterator { +struct reftable_iterator_vtable *ops; +void *iter_arg; +}; + +/* reads the next reftable_ref_record. Returns < 0 for error, 0 for OK and > 0: +* end of iteration. +*/ +int reftable_iterator_next_ref (struct reftable_iterator *it, +struct reftable_ref_record *ref); + +/* reads the next reftable_log_record. Returns < 0 for error, 0 for OK and > 0: +* end of iteration. +*/ +int reftable_iterator_next_log (struct reftable_iterator *it, +struct reftable_log_record *log); + +/* releases resources associated with an iterator. */ +void reftable_iterator_destroy (struct reftable_iterator *it); + +#endif
reftable/reftable.c new file +115 -0
115 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH 12/19 reftable: read reftable files
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This supports reading a single reftable file.
The commit introduces an abstract iterator type, which captures the usecases both of reading individual refs, and iterating over a segment of the ref namespace.
--- Makefile | 2 + reftable/iter.c | 194 +++++++++ reftable/iter.h | 69 ++++ reftable/reader.c | 801 +++++++++++++++++++++++++++++++++++++ reftable/reader.h | 66 +++ reftable/reftable-reader.h | 101 +++++ 6 files changed, 1233 insertions(+) create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/reftable-reader.h
Makefile +2 -0
@@ -2460,7 +2460,9 @@ REFTABLE_OBJS += reftable/basics.o REFTABLE_OBJS += reftable/error.o REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o +REFTABLE_OBJS += reftable/iter.o REFTABLE_OBJS += reftable/publicbasics.o +REFTABLE_OBJS += reftable/reader.o REFTABLE_OBJS += reftable/record.o REFTABLE_OBJS += reftable/refname.o REFTABLE_OBJS += reftable/generic.o
reftable/iter.c new file +194 -0
194 lines not on this page · raw message
reftable/iter.h new file +69 -0
69 lines not on this page · raw message
reftable/reader.c new file +801 -0
801 lines not on this page · raw message
reftable/reader.h new file +66 -0
66 lines not on this page · raw message
reftable/reftable-reader.h new file +101 -0
101 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH 13/19 reftable: reftable file level tests
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
With support for reading and writing files in place, we can construct files (in memory) and attempt to read them back.
Because some sections of the format are optional (eg. indices, log entries), we have to exercise this code using multiple sizes of input data
--- Makefile | 1 + reftable/readwrite_test.c | 652 ++++++++++++++++++++++++++++++++++++++ reftable/reftable-tests.h | 2 +- t/helper/test-reftable.c | 1 + 4 files changed, 655 insertions(+), 1 deletion(-) create mode 100644 reftable/readwrite_test.c
Makefile +1 -0
@@ -2473,6 +2473,7 @@ REFTABLE_OBJS += reftable/writer.o REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o REFTABLE_TEST_OBJS += reftable/record_test.o +REFTABLE_TEST_OBJS += reftable/readwrite_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o REFTABLE_TEST_OBJS += reftable/tree_test.o
reftable/readwrite_test.c new file +652 -0
652 lines not on this page · raw message
reftable/reftable-tests.h +1 -1
@@ -14,7 +14,7 @@ int block_test_main(int argc, const char **argv);int merged_test_main (int argc,const char **argv);int record_test_main (int argc,const char **argv);int refname_test_main (int argc,const char **argv); -int reftable_test_main (int argc,const char **argv); +int readwrite_test_main (int argc,const char **argv);int stack_test_main (int argc,const char **argv);int tree_test_main (int argc,const char **argv);int reftable_dump_main (int argc,char *const *argv);
t/helper/test-reftable.c +1 -0
@@ -6,6 +6,7 @@ int cmd__reftable(int argc, const char **argv)basics_test_main (argc, argv);block_test_main (argc, argv);record_test_main (argc, argv); +readwrite_test_main (argc, argv);tree_test_main (argc, argv);return 0 ; }
Signature · 1 line
gitgitgadget
PATCH 14/19 reftable: add a heap-based priority queue for reftable records
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This is needed to create a merged view multiple reftables
--- Makefile | 2 + reftable/pq.c | 115 ++++++++++++++++++++++++++++++++++++++ reftable/pq.h | 32 +++++++++++ reftable/pq_test.c | 72 ++++++++++++++++++++++++ reftable/reftable-tests.h | 1 + t/helper/test-reftable.c | 1 + 6 files changed, 223 insertions(+) create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/pq_test.c
Makefile +2 -0
@@ -2462,6 +2462,7 @@ REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/iter.o REFTABLE_OBJS += reftable/publicbasics.o +REFTABLE_OBJS += reftable/pq.o REFTABLE_OBJS += reftable/reader.o REFTABLE_OBJS += reftable/record.o REFTABLE_OBJS += reftable/refname.o
@@ -2472,6 +2473,7 @@ REFTABLE_OBJS += reftable/writer.o REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o +REFTABLE_TEST_OBJS += reftable/pq_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/readwrite_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o
reftable/pq.c new file +115 -0
115 lines not on this page · raw message
reftable/pq.h new file +32 -0
@@ -0,0 +1,32 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef PQ_H +#define PQ_H + +#include "record.h" + +struct pq_entry { +int index; +struct reftable_record rec; +}; + +struct merged_iter_pqueue { +struct pq_entry *heap; +size_t len; +size_t cap; +}; + +struct pq_entry merged_iter_pqueue_top (struct merged_iter_pqueue pq); +int merged_iter_pqueue_is_empty (struct merged_iter_pqueue pq); +void merged_iter_pqueue_check (struct merged_iter_pqueue pq); +struct pq_entry merged_iter_pqueue_remove (struct merged_iter_pqueue *pq); +void merged_iter_pqueue_add (struct merged_iter_pqueue *pq,struct pq_entry e); +void merged_iter_pqueue_release (struct merged_iter_pqueue *pq); + +#endif
reftable/pq_test.c new file +72 -0
72 lines not on this page · raw message
reftable/reftable-tests.h +1 -0
@@ -12,6 +12,7 @@ https://developers.google.com/open-source/licenses/bsdint basics_test_main (int argc,const char **argv);int block_test_main (int argc,const char **argv);int merged_test_main (int argc,const char **argv); +int pq_test_main (int argc,const char **argv);int record_test_main (int argc,const char **argv);int refname_test_main (int argc,const char **argv);int readwrite_test_main (int argc,const char **argv);
t/helper/test-reftable.c +1 -0
@@ -5,6 +5,7 @@ int cmd__reftable(int argc, const char **argv) {basics_test_main (argc, argv);block_test_main (argc, argv); +pq_test_main (argc, argv);record_test_main (argc, argv);readwrite_test_main (argc, argv);tree_test_main (argc, argv);
Signature · 1 line
gitgitgadget
PATCH 15/19 reftable: add merged table view
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This adds an abstract, read-only interface to the ref database.
This primitive is used to construct the read view of the ref database (the read view is constructed by merging several *.ref files). It also provides the mechanism to provide a unified view of the refs in the main repository and the per-worktree refs.
--- Makefile | 2 + reftable/merged.c | 362 +++++++++++++++++++++++++++++++++++++ reftable/merged.h | 35 ++++ reftable/merged_test.c | 292 ++++++++++++++++++++++++++++++ reftable/reftable-merged.h | 72 ++++++++ t/helper/test-reftable.c | 1 + 6 files changed, 764 insertions(+) create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/merged_test.c create mode 100644 reftable/reftable-merged.h
Makefile +2 -0
@@ -2462,6 +2462,7 @@ REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/iter.o REFTABLE_OBJS += reftable/publicbasics.o +REFTABLE_OBJS += reftable/merged.o REFTABLE_OBJS += reftable/pq.o REFTABLE_OBJS += reftable/reader.o REFTABLE_OBJS += reftable/record.o
@@ -2473,6 +2474,7 @@ REFTABLE_OBJS += reftable/writer.o REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o +REFTABLE_TEST_OBJS += reftable/merged_test.o REFTABLE_TEST_OBJS += reftable/pq_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/readwrite_test.o
reftable/merged.c new file +362 -0
362 lines not on this page · raw message
reftable/merged.h new file +35 -0
@@ -0,0 +1,35 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef MERGED_H +#define MERGED_H + +#include "pq.h" + +struct reftable_merged_table { +struct reftable_table *stack; +size_t stack_len; +uint32_t hash_id; +int suppress_deletions; + +uint64_t min; +uint64_t max; +}; + +struct merged_iter { +struct reftable_iterator *stack; +uint32_t hash_id; +size_t stack_len; +uint8_t typ; +int suppress_deletions; +struct merged_iter_pqueue pq; +}; + +void merged_table_release (struct reftable_merged_table *mt); + +#endif
reftable/merged_test.c new file +292 -0
292 lines not on this page · raw message
reftable/reftable-merged.h new file +72 -0
72 lines not on this page · raw message
t/helper/test-reftable.c +1 -0
@@ -5,6 +5,7 @@ int cmd__reftable(int argc, const char **argv) {basics_test_main (argc, argv);block_test_main (argc, argv); +merged_test_main (argc, argv);pq_test_main (argc, argv);record_test_main (argc, argv);readwrite_test_main (argc, argv);
Signature · 1 line
gitgitgadget
PATCH 16/19 reftable: implement refname validation
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The packed/loose format has restrictions on refnames: a and a/b cannot coexist. This limitation does not apply to reftable per se, but must be maintained for interoperability. This code adds validation routines to abort transactions that are trying to add invalid names.
--- Makefile | 1 + reftable/refname.c | 209 +++++++++++++++++++++++++++++++++++++++ reftable/refname.h | 29 ++++++ reftable/refname_test.c | 102 +++++++++++++++++++ t/helper/test-reftable.c | 1 + 5 files changed, 342 insertions(+) create mode 100644 reftable/refname.c create mode 100644 reftable/refname.h create mode 100644 reftable/refname_test.c
Makefile +1 -0
@@ -2478,6 +2478,7 @@ REFTABLE_TEST_OBJS += reftable/merged_test.o REFTABLE_TEST_OBJS += reftable/pq_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/readwrite_test.o +REFTABLE_TEST_OBJS += reftable/refname_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o REFTABLE_TEST_OBJS += reftable/tree_test.o
reftable/refname.c new file +209 -0
209 lines not on this page · raw message
reftable/refname.h new file +29 -0
@@ -0,0 +1,29 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ +#ifndef REFNAME_H +#define REFNAME_H + +#include "reftable-record.h" +#include "reftable-generic.h" + +struct modification { +struct reftable_table tab; + +char **add; +size_t add_len; + +char **del; +size_t del_len; +}; + +int validate_ref_record_addition (struct reftable_table tab, +struct reftable_ref_record *recs,size_t sz); + +int modification_validate (struct modification *mod); + +#endif
reftable/refname_test.c new file +102 -0
102 lines not on this page · raw message
t/helper/test-reftable.c +1 -0
@@ -8,6 +8,7 @@ int cmd__reftable(int argc, const char **argv)merged_test_main (argc, argv);pq_test_main (argc, argv);record_test_main (argc, argv); +refname_test_main (argc, argv);readwrite_test_main (argc, argv);tree_test_main (argc, argv);return 0 ;
Signature · 1 line
gitgitgadget
PATCH 17/19 reftable: implement stack, a mutable database of reftable files.
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
--- Makefile | 1 + reftable/reftable-stack.h | 128 ++++ reftable/stack.c | 1396 +++++++++++++++++++++++++++++++++++++ reftable/stack.h | 41 ++ reftable/stack_test.c | 949 +++++++++++++++++++++++++ t/helper/test-reftable.c | 1 + 6 files changed, 2516 insertions(+) create mode 100644 reftable/reftable-stack.h create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/stack_test.c
Makefile +1 -0
@@ -2479,6 +2479,7 @@ REFTABLE_TEST_OBJS += reftable/pq_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/readwrite_test.o REFTABLE_TEST_OBJS += reftable/refname_test.o +REFTABLE_TEST_OBJS += reftable/stack_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o REFTABLE_TEST_OBJS += reftable/tree_test.o
reftable/reftable-stack.h new file +128 -0
128 lines not on this page · raw message
reftable/stack.c new file +1396 -0
1,396 lines not on this page · raw message
reftable/stack.h new file +41 -0
41 lines not on this page · raw message
reftable/stack_test.c new file +949 -0
949 lines not on this page · raw message
t/helper/test-reftable.c +1 -0
@@ -10,6 +10,7 @@ int cmd__reftable(int argc, const char **argv)record_test_main (argc, argv);refname_test_main (argc, argv);readwrite_test_main (argc, argv); +stack_test_main (argc, argv);tree_test_main (argc, argv);return 0 ; }
Signature · 1 line
gitgitgadget
PATCH 18/19 reftable: add dump utility
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
provide a command-line utility for inspecting individual tables, and inspecting a complete ref database
--- reftable/dump.c | 107 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 reftable/dump.c
reftable/dump.c new file +107 -0
@@ -0,0 +1,107 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "git-compat-util.h" +#include "hash.h" + +#include "reftable-blocksource.h" +#include "reftable-error.h" +#include "reftable-merged.h" +#include "reftable-record.h" +#include "reftable-tests.h" +#include "reftable-writer.h" +#include "reftable-iterator.h" +#include "reftable-reader.h" +#include "reftable-stack.h" +#include "reftable-generic.h" + +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> + +static int compact_stack (const char *stackdir) +{ +struct reftable_stack *stack =NULL ; +struct reftable_write_options cfg = {0 }; + +int err =reftable_new_stack (&stack, stackdir, cfg); +if (err <0 ) + goto done; + + err =reftable_stack_compact_all (stack,NULL ); +if (err <0 ) + goto done; +done: +if (stack) { +reftable_stack_destroy (stack); + } +return err; +} + +static void print_help (void ) +{ +printf ("usage: dump [-cst] arg\n\n" +"options: \n" +" -c compact\n" +" -t dump table\n" +" -s dump stack\n" +" -6 sha256 hash format\n" +" -h this help\n" +"\n" ); +} + +int reftable_dump_main (int argc,char *const *argv) +{ +int err =0 ; +int opt_dump_table =0 ; +int opt_dump_stack =0 ; +int opt_compact =0 ; +uint32_t opt_hash_id =GIT_SHA1_FORMAT_ID ; +const char *arg =NULL , *argv0 = argv[0 ]; + +for (; argc >1 ; argv++, argc--) +if (*argv[1 ] !='-' ) +break ; +else if (!strcmp ("-t" , argv[1 ])) + opt_dump_table =1 ; +else if (!strcmp ("-6" , argv[1 ])) + opt_hash_id =GIT_SHA256_FORMAT_ID ; +else if (!strcmp ("-s" , argv[1 ])) + opt_dump_stack =1 ; +else if (!strcmp ("-c" , argv[1 ])) + opt_compact =1 ; +else if (!strcmp ("-?" , argv[1 ]) || !strcmp ("-h" , argv[1 ])) { +print_help (); +return 2 ; + } + +if (argc !=2 ) { +fprintf (stderr,"need argument\n" ); +print_help (); +return 2 ; + } + + arg = argv[1 ]; + +if (opt_dump_table) { + err =reftable_reader_print_file (arg); + }else if (opt_dump_stack) { + err =reftable_stack_print_directory (arg, opt_hash_id); + }else if (opt_compact) { + err =compact_stack (arg); + } + +if (err <0 ) { +fprintf (stderr,"%s: %s: %s\n" , argv0, arg, +reftable_error_str (err)); +return 1 ; + } +return 0 ; +}
Signature · 1 line
gitgitgadget
PATCH 19/19 Add "test-tool dump-reftable" command.
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This command dumps individual tables or a stack of of tables.
--- Makefile | 1 + t/helper/test-reftable.c | 5 +++++ t/helper/test-tool.c | 1 + t/helper/test-tool.h | 1 + 4 files changed, 8 insertions(+)
Makefile +1 -0
@@ -2474,6 +2474,7 @@ REFTABLE_OBJS += reftable/writer.o REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o +REFTABLE_TEST_OBJS += reftable/dump.o REFTABLE_TEST_OBJS += reftable/merged_test.o REFTABLE_TEST_OBJS += reftable/pq_test.o REFTABLE_TEST_OBJS += reftable/record_test.o
t/helper/test-reftable.c +5 -0
@@ -14,3 +14,8 @@ int cmd__reftable(int argc, const char **argv)tree_test_main (argc, argv);return 0 ; } + +int cmd__dump_reftable (int argc,const char **argv) +{ +return reftable_dump_main (argc, (char *const *)argv); +}
t/helper/test-tool.c +1 -0
@@ -61,6 +61,7 @@ static struct test_cmd cmds[] = { { "read-midx", cmd__read_midx }, { "ref-store", cmd__ref_store }, { "reftable", cmd__reftable }, + { "dump-reftable", cmd__dump_reftable }, { "regex", cmd__regex }, { "repository", cmd__repository }, { "revision-walking", cmd__revision_walking },
t/helper/test-tool.h +1 -0
@@ -19,6 +19,7 @@ int cmd__dump_cache_tree(int argc, const char **argv);int cmd__dump_fsmonitor (int argc,const char **argv);int cmd__dump_split_index (int argc,const char **argv);int cmd__dump_untracked_cache (int argc,const char **argv); +int cmd__dump_reftable (int argc,const char **argv);int cmd__example_decorate (int argc,const char **argv);int cmd__fast_rebase (int argc,const char **argv);int cmd__genrandom (int argc,const char **argv);
Signature · 1 line
gitgitgadget
Quoted 6 lines · Han-Wen Nienhuys via GitGitGadget
The reftable format is described in
Documentation/technical/reftable.txt.This is a fully reentrant implementation of reading and writing the reftable file format, and should be suitable for embedding in libgit2 too. It does not hook the code up to git to function as a ref storage backend yet.
Junio, per discussion with AEvar, let's kick out the full reftable topic from next, and put in this topic. This is unlikely to have conflicts with any other efforts, and is necessary for the full reftable support anyway.
Signature · 7 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
cover fixup for hn/reftable
In reply to Han-Wen Nienhuys via GitGitGadgetDEVELOPER=1 wasn't working well in OpenBSD until cb/makefile-apple-clang (which just hit next), so I missed it in my previous run.
hopefully it is not too late to fix this, otherwise.
As usual they should apply to the patch they refer to and are based on the last reroll in seen (tip 1427aef0bd as of e881a4ab6c (Merge branch 'hn/reftable' into seen, 2021-09-07)
Except from the last patch (and two other minor refactors I couldn't avoid) they could all be one single change to move the declaration for the zlib compat function to some place where it could be visible from the two files that will use it, but it was spread on multiple files to easy autosquashing
Carlo Marcelo Arenas Belón (4):
fixup! reftable: reading/writing blocks
fixup! reftable: utility functions
fixup! Provide zlib's uncompress2 from compat/zlib-compat.c fixup! reftable: utility functions
compat/zlib-uncompress2.c | 3 +++ reftable/basics.h | 1 - reftable/block.c | 9 --------- reftable/system.h | 10 +++++++++- 4 files changed, 12 insertions(+), 11 deletions(-)
Signature · 1 line
2.33.0.481.g26d3bed244
PATCH 1/4 fixup! reftable: reading/writing blocks
move the definition to "system.h" because it needs to be accessible also to the compat file.
--- reftable/block.c | 9 --------- 1 file changed, 9 deletions(-)
reftable/block.c +0 -9
@@ -15,15 +15,6 @@ license that can be found in the LICENSE file or at#include "system.h" #include <zlib.h> -#ifdef NO_UNCOMPRESS2 -/* -* This is uncompress2, which is only available in zlib >= 1.2.9 -* (released as of early 2017) -*/ -int uncompress2 (Bytef *dest,uLongf *destLen,const Bytef *source, -uLong *sourceLen); -#endif -int header_size (int version) {switch (version) {
Signature · 1 line
2.33.0.481.g26d3bed244
PATCH 2/4 fixup! reftable: utility functions
In reply to Carlo Marcelo Arenas Belónremove forward declaration for strbuf that is no longer needed once strbuf.h was included.
add a conditional declaration for the zlib function that then can be used in both the compat definition and the block.c user without triggering -Wmissing-prototypes:
compat/zlib-uncompress2.c:36:13: error: no previous prototype for function
'uncompress2' [-Werror,-Wmissing-prototypes]
int ZEXPORT uncompress2 (
^
compat/zlib-uncompress2.c:36:1: note: declare 'static' if the function is not
intended to be used outside of this translation unit
int ZEXPORT uncompress2 (
^
static
1 error generated.
gmake: *** [Makefile:2571: compat/zlib-uncompress2.o] Error 1
--- reftable/system.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-)
reftable/system.h +9 -1
@@ -18,7 +18,15 @@ license that can be found in the LICENSE file or at#include <zlib.h> -struct strbuf ; +#ifdef NO_UNCOMPRESS2 +/* +* This is uncompress2, which is only available in zlib >= 1.2.9 +* (released as of early 2017) +*/ +int uncompress2 (Bytef *dest,uLongf *destLen,const Bytef *source, +uLong *sourceLen); +#endif +int hash_size (uint32_t id);#endif
Signature · 1 line
2.33.0.481.g26d3bed244
PATCH 3/4 fixup! Provide zlib's uncompress2 from compat/zlib-compat.c
In reply to Carlo Marcelo Arenas Belónwhile at it, make sure z_conf is defined to null (usually coming from zconf.h, but optional and otherwise to fail with older zlib versions)
--- compat/zlib-uncompress2.c | 3 +++ 1 file changed, 3 insertions(+)
compat/zlib-uncompress2.c +3 -0
@@ -8,6 +8,9 @@*/ +#include "../reftable/system.h" +#define z_const +/* * Copyright (C) 1995-2003, 2010, 2014, 2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h
Signature · 1 line
2.33.0.481.g26d3bed244
PATCH 4/4 fixup! reftable: utility functions
In reply to Carlo Marcelo Arenas Belón--- reftable/basics.h | 1 - 1 file changed, 1 deletion(-)
reftable/basics.h +0 -1
@@ -54,7 +54,6 @@ void reftable_free(void *p);void *reftable_calloc (size_t sz);/* Find the longest shared prefix size of `a` and `b` */ -struct strbuf ;int common_prefix_size (struct strbuf *a,struct strbuf *b);#endif
Signature · 1 line
2.33.0.481.g26d3bed244
Re: [PATCH 0/4] fixup for hn/reftable
In reply to Carlo Marcelo Arenas BelónQuoted 4 lines · Carlo Marcelo Arenas Belón
DEVELOPER=1 wasn't working well in OpenBSD until cb/makefile-apple-clang (which just hit next), so I missed it in my previous run.
hopefully it is not too late to fix this, otherwise.
I haven't merged them to 'next', so it is easy to accept a reroll.
Quoted 20 lines
As usual they should apply to the patch they refer to and are based on the last reroll in seen (tip
1427aef0bdas ofe881a4ab6c(Merge branch 'hn/reftable' into seen, 2021-09-07)Except from the last patch (and two other minor refactors I couldn't avoid) they could all be one single change to move the declaration for the zlib compat function to some place where it could be visible from the two files that will use it, but it was spread on multiple files to easy autosquashing
Carlo Marcelo Arenas Belón (4):
fixup! reftable: reading/writing blocks
fixup! reftable: utility functions
fixup! Provide zlib's uncompress2 fromcompat/zlib-compat.cfixup! reftable: utility functionscompat/zlib-uncompress2.c | 3 +++ reftable/basics.h | 1 - reftable/block.c | 9 --------- reftable/system.h | 10 +++++++++- 4 files changed, 12 insertions(+), 11 deletions(-)
v2 cover Adds reftable library code from https://github.com/hanwen/reftable.
In reply to Han-Wen Nienhuys via GitGitGadgetThe reftable format is described in Documentation/technical/reftable.txt.
This is a fully reentrant implementation of reading and writing the reftable file format, and should be suitable for embedding in libgit2 too. It does not hook the code up to git to function as a ref storage backend yet.
v2:
* fold in OpenBSD fixes by Carlo Belón.
Han-Wen Nienhuys (19):
hash.h: provide constants for the hash IDs
reftable: RFC: add LICENSE
reftable: add error related functionality
reftable: utility functions
reftable: add blocksource, an abstraction for random access reads
reftable: (de)serialization for the polymorphic record type. Provide zlib's uncompress2 from compat/zlib-compat.c
reftable: reading/writing blocks
reftable: a generic binary tree implementation
reftable: write reftable files
reftable: generic interface to tables
reftable: read reftable files
reftable: reftable file level tests
reftable: add a heap-based priority queue for reftable records
reftable: add merged table view
reftable: implement refname validation
reftable: implement stack, a mutable database of reftable files.
reftable: add dump utility
Add "test-tool dump-reftable" command.
Makefile | 53 +- ci/lib.sh | 1 + compat/.gitattributes | 1 + compat/zlib-uncompress2.c | 95 ++ config.mak.uname | 5 + configure.ac | 13 + contrib/buildsystems/CMakeLists.txt | 14 +- contrib/buildsystems/Generators/Vcxproj.pm | 11 +- hash.h | 6 + object-file.c | 7 +- reftable/LICENSE | 31 + reftable/basics.c | 128 ++ reftable/basics.h | 60 + reftable/basics_test.c | 98 ++ reftable/block.c | 439 ++++++ reftable/block.h | 127 ++ reftable/block_test.c | 120 ++ reftable/blocksource.c | 148 +++ reftable/blocksource.h | 22 + reftable/constants.h | 21 + reftable/dump.c | 107 ++ reftable/error.c | 41 + reftable/generic.c | 169 +++ reftable/generic.h | 32 + reftable/iter.c | 194 +++ reftable/iter.h | 69 + reftable/merged.c | 362 +++++ reftable/merged.h | 35 + reftable/merged_test.c | 292 ++++ reftable/pq.c | 105 ++ reftable/pq.h | 33 + reftable/pq_test.c | 82 ++ reftable/publicbasics.c | 58 + reftable/reader.c | 801 +++++++++++ reftable/reader.h | 66 + reftable/readwrite_test.c | 652 +++++++++ reftable/record.c | 1212 +++++++++++++++++ reftable/record.h | 139 ++ reftable/record_test.c | 412 ++++++ reftable/refname.c | 209 +++ reftable/refname.h | 29 + reftable/refname_test.c | 102 ++ reftable/reftable-blocksource.h | 49 + reftable/reftable-error.h | 62 + reftable/reftable-generic.h | 47 + reftable/reftable-iterator.h | 39 + reftable/reftable-malloc.h | 18 + reftable/reftable-merged.h | 72 + reftable/reftable-reader.h | 101 ++ reftable/reftable-record.h | 114 ++ reftable/reftable-stack.h | 128 ++ reftable/reftable-tests.h | 23 + reftable/reftable-writer.h | 148 +++ reftable/reftable.c | 115 ++ reftable/stack.c | 1396 ++++++++++++++++++++ reftable/stack.h | 41 + reftable/stack_test.c | 949 +++++++++++++ reftable/system.h | 32 + reftable/test_framework.c | 23 + reftable/test_framework.h | 53 + reftable/tree.c | 63 + reftable/tree.h | 34 + reftable/tree_test.c | 61 + reftable/writer.c | 690 ++++++++++ reftable/writer.h | 50 + t/helper/test-reftable.c | 21 + t/helper/test-tool.c | 4 +- t/helper/test-tool.h | 2 + t/t0032-reftable-unittest.sh | 15 + 69 files changed, 10939 insertions(+), 12 deletions(-) create mode 100644 compat/.gitattributes create mode 100644 compat/zlib-uncompress2.c create mode 100644 reftable/LICENSE create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/basics_test.c create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/block_test.c create mode 100644 reftable/blocksource.c create mode 100644 reftable/blocksource.h create mode 100644 reftable/constants.h create mode 100644 reftable/dump.c create mode 100644 reftable/error.c create mode 100644 reftable/generic.c create mode 100644 reftable/generic.h create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/merged_test.c create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/pq_test.c create mode 100644 reftable/publicbasics.c create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/readwrite_test.c create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/record_test.c create mode 100644 reftable/refname.c create mode 100644 reftable/refname.h create mode 100644 reftable/refname_test.c create mode 100644 reftable/reftable-blocksource.h create mode 100644 reftable/reftable-error.h create mode 100644 reftable/reftable-generic.h create mode 100644 reftable/reftable-iterator.h create mode 100644 reftable/reftable-malloc.h create mode 100644 reftable/reftable-merged.h create mode 100644 reftable/reftable-reader.h create mode 100644 reftable/reftable-record.h create mode 100644 reftable/reftable-stack.h create mode 100644 reftable/reftable-tests.h create mode 100644 reftable/reftable-writer.h create mode 100644 reftable/reftable.c create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/stack_test.c create mode 100644 reftable/system.h create mode 100644 reftable/test_framework.c create mode 100644 reftable/test_framework.h create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100644 reftable/tree_test.c create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h create mode 100644 t/helper/test-reftable.c create mode 100755 t/t0032-reftable-unittest.sh
base-commit: e0a2f5cbc585657e757385ad918f167f519cfb96
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-git-1081%2Fhanwen%2Fjust-library-v2
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-git-1081/hanwen/just-library-v2
Pull-Request: https://github.com/git/git/pull/1081
Range-diff vs v1:
1: b9ee8cd8e9f = 1: 8534c69cf84 hash.h: provide constants for the hash IDs
2: 2d31c783fb5 = 2: 52968d8e83f reftable: RFC: add LICENSE
3: ea6f4f8c150 = 3: 84c8b396df5 reftable: add error related functionality
4: fd5c5c9f338 ! 4: fa58f3d73e7 reftable: utility functions
@@ Makefile: OBJECTS += $(PROGRAM_OBJS)
endif
@@ Makefile: $(LIB_FILE): $(LIB_OBJS)
$(XDIFF_LIB): $(XDIFF_OBJS)
- $(QUIET_AR)$(AR) $(ARFLAGS) $@ $^
+ $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
+$(REFTABLE_LIB): $(REFTABLE_OBJS)
+ $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
@@ reftable/system.h (new)
+
+#include <zlib.h>
+
-+struct strbuf;
++#ifdef NO_UNCOMPRESS2
++/*
++ * This is uncompress2, which is only available in zlib >= 1.2.9
++ * (released as of early 2017)
++ */
++int uncompress2(Bytef *dest, uLongf *destLen, const Bytef *source,
++ uLong *sourceLen);
++#endif
++
+int hash_size(uint32_t id);
+
+#endif
5: c11f06ed6fb = 5: e9206532126 reftable: add blocksource, an abstraction for random access reads
6: 23977331cf8 = 6: 651e195fe77 reftable: (de)serialization for the polymorphic record type.
7: aa9b8e05dbd ! 7: e595dbc6c84 Provide zlib's uncompress2 from compat/zlib-compat.c
@@ compat/zlib-uncompress2.c (new)
+
+*/
+
++#include "../reftable/system.h"
++#define z_const
++
+/*
+ * Copyright (C) 1995-2003, 2010, 2014, 2016 Jean-loup Gailly, Mark Adler
+ * For conditions of distribution and use, see copyright notice in zlib.h
@@ config.mak.uname: ifeq ($(uname_S),FreeBSD)
endif
ifeq ($(uname_S),OpenBSD)
+ NO_UNCOMPRESS2 = YesPlease
++ # Versions < 7.0 need compatibility layer
++ ifeq ($(shell expr "$(uname_R)" : "[1-6]\."),2)
++ NO_UNCOMPRESS2 = UnfortunatelyYes
++ endif
NO_STRCASESTR = YesPlease
NO_MEMMEM = YesPlease
USE_ST_TIMESPEC = YesPlease
8: 2648e05c510 ! 8: 97f7ee04886 reftable: reading/writing blocks
@@ reftable/block.c (new)
+#include "system.h"
+#include <zlib.h>
+
-+#ifdef NO_UNCOMPRESS2
-+/*
-+ * This is uncompress2, which is only available in zlib >= 1.2.9
-+ * (released as of early 2017)
-+ */
-+int uncompress2(Bytef *dest, uLongf *destLen, const Bytef *source,
-+ uLong *sourceLen);
-+#endif
-+
+int header_size(int version)
+{
+ switch (version) {
9: 49a9a00465d ! 9: 8d2e8be5795 reftable: a generic binary tree implementation
@@ reftable/tree_test.c (new)
+{
+ struct curry *c = arg;
+ if (c->last) {
-+ assert(test_compare(c->last, key) < 0);
++ EXPECT(test_compare(c->last, key) < 0);
+ }
+ c->last = key;
+}
@@ reftable/tree_test.c (new)
+ } while (i != 1);
+
+ for (i = 1; i < ARRAY_SIZE(nodes); i++) {
-+ assert(values + i == nodes[i]->key);
-+ assert(nodes[i] ==
++ EXPECT(values + i == nodes[i]->key);
++ EXPECT(nodes[i] ==
+ tree_search(values + i, &root, &test_compare, 0));
+ }
+
10: c00876afd26 = 10: 965af6ec065 reftable: write reftable files
11: 0f32d6f4c91 = 11: 4e259380fda reftable: generic interface to tables
12: a77878993aa = 12: a43d3512164 reftable: read reftable files
13: 4bbefe977ae = 13: f0e159cd853 reftable: reftable file level tests
14: 7ee46496081 ! 14: 5a5c108d601 reftable: add a heap-based priority queue for reftable records
@@ reftable/pq.c (new)
+#include "system.h"
+#include "basics.h"
+
-+static int pq_less(struct pq_entry a, struct pq_entry b)
++int pq_less(struct pq_entry *a, struct pq_entry *b)
+{
+ struct strbuf ak = STRBUF_INIT;
+ struct strbuf bk = STRBUF_INIT;
+ int cmp = 0;
-+ reftable_record_key(&a.rec, &ak);
-+ reftable_record_key(&b.rec, &bk);
++ reftable_record_key(&a->rec, &ak);
++ reftable_record_key(&b->rec, &bk);
+
+ cmp = strbuf_cmp(&ak, &bk);
+
@@ reftable/pq.c (new)
+ strbuf_release(&bk);
+
+ if (cmp == 0)
-+ return a.index > b.index;
++ return a->index > b->index;
+
+ return cmp < 0;
+}
@@ reftable/pq.c (new)
+ return pq.len == 0;
+}
+
-+void merged_iter_pqueue_check(struct merged_iter_pqueue pq)
-+{
-+ int i = 0;
-+ for (i = 1; i < pq.len; i++) {
-+ int parent = (i - 1) / 2;
-+
-+ assert(pq_less(pq.heap[parent], pq.heap[i]));
-+ }
-+}
-+
+struct pq_entry merged_iter_pqueue_remove(struct merged_iter_pqueue *pq)
+{
+ int i = 0;
@@ reftable/pq.c (new)
+ int min = i;
+ int j = 2 * i + 1;
+ int k = 2 * i + 2;
-+ if (j < pq->len && pq_less(pq->heap[j], pq->heap[i])) {
++ if (j < pq->len && pq_less(&pq->heap[j], &pq->heap[i])) {
+ min = j;
+ }
-+ if (k < pq->len && pq_less(pq->heap[k], pq->heap[min])) {
++ if (k < pq->len && pq_less(&pq->heap[k], &pq->heap[min])) {
+ min = k;
+ }
+
@@ reftable/pq.c (new)
+ i = pq->len - 1;
+ while (i > 0) {
+ int j = (i - 1) / 2;
-+ if (pq_less(pq->heap[j], pq->heap[i])) {
++ if (pq_less(&pq->heap[j], &pq->heap[i])) {
+ break;
+ }
+
@@ reftable/pq.h (new)
+struct pq_entry merged_iter_pqueue_remove(struct merged_iter_pqueue *pq);
+void merged_iter_pqueue_add(struct merged_iter_pqueue *pq, struct pq_entry e);
+void merged_iter_pqueue_release(struct merged_iter_pqueue *pq);
++int pq_less(struct pq_entry *a, struct pq_entry *b);
+
+#endif
@@ reftable/pq_test.c (new)
+#include "reftable-tests.h"
+#include "test_framework.h"
+
++void merged_iter_pqueue_check(struct merged_iter_pqueue pq)
++{
++ int i = 0;
++ for (i = 1; i < pq.len; i++) {
++ int parent = (i - 1) / 2;
++
++ assert(pq_less(&pq.heap[parent], &pq.heap[i]));
++ }
++}
++
+static void test_pq(void)
+{
+ char *names[54] = { NULL };
@@ reftable/pq_test.c (new)
+ merged_iter_pqueue_check(pq);
+
+ if (last) {
-+ assert(strcmp(last, ref->refname) < 0);
++ EXPECT(strcmp(last, ref->refname) < 0);
+ }
+ last = ref->refname;
+ ref->refname = NULL;
15: 8bc889aaae8 ! 15: 905ffa58f36 reftable: add merged table view
@@ reftable/merged_test.c (new)
+ for (i = 0; i < n; i++) {
+ uint64_t before = refs[i].update_index;
+ int n = reftable_writer_add_ref(w, &refs[i]);
-+ assert(n == 0);
-+ assert(before == refs[i].update_index);
++ EXPECT(n == 0);
++ EXPECT(before == refs[i].update_index);
+ }
+
+ err = reftable_writer_close(w);
@@ reftable/merged_test.c (new)
+ }
+ reftable_iterator_destroy(&it);
+
-+ assert(ARRAY_SIZE(want) == len);
++ EXPECT(ARRAY_SIZE(want) == len);
+ for (i = 0; i < len; i++) {
-+ assert(reftable_ref_record_equal(&want[i], &out[i],
++ EXPECT(reftable_ref_record_equal(&want[i], &out[i],
+ GIT_SHA1_RAWSZ));
+ }
+ for (i = 0; i < len; i++) {
@@ reftable/merged_test.c (new)
+ EXPECT_ERR(err);
+
+ hash_id = reftable_reader_hash_id(rd);
-+ assert(hash_id == GIT_SHA1_FORMAT_ID);
++ EXPECT(hash_id == GIT_SHA1_FORMAT_ID);
+
+ reftable_table_from_reader(&tab[0], rd);
+ err = reftable_new_merged_table(&merged, tab, 1, GIT_SHA1_FORMAT_ID);
16: 9a30c1c4666 = 16: e9598889af2 reftable: implement refname validation
17: b099c42d65d = 17: 8b0ee68c636 reftable: implement stack, a mutable database of reftable files.
18: 0170f20b5e7 = 18: a9bebe45bde reftable: add dump utility
19: 4fc6b5cbf16 = 19: 104fbc7502f Add "test-tool dump-reftable" command.Signature · 1 line
gitgitgadget
PATCH v2 1/19 hash.h: provide constants for the hash IDs
From: Han-Wen Nienhuys <[email protected]>
This will simplify referencing them from code that is not deeply integrated with Git, in particular, the reftable library.
--- hash.h | 6 ++++++ object-file.c | 7 ++----- 2 files changed, 8 insertions(+), 5 deletions(-)
hash.h +6 -0
@@ -95,12 +95,18 @@ static inline void git_SHA256_Clone(git_SHA256_CTX *dst, const git_SHA256_CTX *s/* Number of algorithms supported (including unknown). */ #define GIT_HASH_NALGOS (GIT_HASH_SHA256 + 1) +/* "sha1", big-endian */ +#define GIT_SHA1_FORMAT_ID 0x73686131 +/* The length in bytes and in hex digits of an object name (SHA-1 value). */ #define GIT_SHA1_RAWSZ 20#define GIT_SHA1_HEXSZ (2 * GIT_SHA1_RAWSZ)/* The block size of SHA-1. */ #define GIT_SHA1_BLKSZ 64 +/* "s256", big-endian */ +#define GIT_SHA256_FORMAT_ID 0x73323536 +/* The length in bytes and in hex digits of an object name (SHA-256 value). */ #define GIT_SHA256_RAWSZ 32#define GIT_SHA256_HEXSZ (2 * GIT_SHA256_RAWSZ)
object-file.c +2 -5
@@ -164,7 +164,6 @@ static void git_hash_unknown_final_oid(struct object_id *oid, git_hash_ctx *ctx)BUG ("trying to finalize unknown hash" ); } -const struct git_hash_algo hash_algos[GIT_HASH_NALGOS ] = { {NULL ,
@@ -183,8 +182,7 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = { }, {"sha1" , -/* "sha1", big-endian */ -0x73686131 , +GIT_SHA1_FORMAT_ID ,GIT_SHA1_RAWSZ ,GIT_SHA1_HEXSZ , GIT_SHA1_BLKSZ,
@@ -199,8 +197,7 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = { }, {"sha256" , -/* "s256", big-endian */ -0x73323536 , +GIT_SHA256_FORMAT_ID ,GIT_SHA256_RAWSZ ,GIT_SHA256_HEXSZ , GIT_SHA256_BLKSZ,
Signature · 1 line
gitgitgadget
PATCH v2 2/19 reftable: RFC: add LICENSE
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The objective of this code is to be usable as a C library, so it can be reused in libgit2.
This is currently using a BSD license as it is the liberal license I could find, but this could be changed to whatever fits the stated goal above.
This code is currently imported from github.com/hanwen/reftable. Once this code lands in git.git, the C code will be removed from github.com/hanwen/reftable, and the git.git code will be the source of truth.
--- reftable/LICENSE | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 reftable/LICENSE
reftable/LICENSE new file +31 -0
@@ -0,0 +1,31 @@ +BSD License + +Copyright (c) 2020, Google LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of Google LLC nor the names of its contributors may +be used to endorse or promote products derived from this software +without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Signature · 1 line
gitgitgadget
PATCH v2 3/19 reftable: add error related functionality
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The reftable/ directory is structured as a library, so it cannot crash on misuse. Instead, it returns an error codes.
In addition, the error code can be used to signal conditions from lower levels of the library to be handled by higher levels of the library. For example, a transaction might legitimately write an empty reftable file, but in that case, we'd want to shortcut the transaction overhead.
--- reftable/error.c | 41 ++++++++++++++++++++++++++ reftable/reftable-error.h | 62 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 reftable/error.c create mode 100644 reftable/reftable-error.h
reftable/error.c new file +41 -0
@@ -0,0 +1,41 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "reftable-error.h" + +#include <stdio.h> + +const char *reftable_error_str (int err) +{ +static char buf[250 ]; +switch (err) { +case REFTABLE_IO_ERROR : +return "I/O error" ; +case REFTABLE_FORMAT_ERROR : +return "corrupt reftable file" ; +case REFTABLE_NOT_EXIST_ERROR : +return "file does not exist" ; +case REFTABLE_LOCK_ERROR : +return "data is outdated" ; +case REFTABLE_API_ERROR : +return "misuse of the reftable API" ; +case REFTABLE_ZLIB_ERROR : +return "zlib failure" ; +case REFTABLE_NAME_CONFLICT : +return "file/directory conflict" ; +case REFTABLE_EMPTY_TABLE_ERROR : +return "wrote empty table" ; +case REFTABLE_REFNAME_ERROR : +return "invalid refname" ; +case -1 : +return "general error" ; +default : +snprintf (buf,sizeof (buf),"unknown error code %d" , err); +return buf; + } +}
reftable/reftable-error.h new file +62 -0
62 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v2 4/19 reftable: utility functions
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This commit provides basic utility classes for the reftable library.
--- Makefile | 25 +++- contrib/buildsystems/CMakeLists.txt | 14 ++- contrib/buildsystems/Generators/Vcxproj.pm | 11 +- reftable/basics.c | 128 +++++++++++++++++++++ reftable/basics.h | 60 ++++++++++ reftable/basics_test.c | 98 ++++++++++++++++ reftable/publicbasics.c | 58 ++++++++++ reftable/reftable-malloc.h | 18 +++ reftable/reftable-tests.h | 22 ++++ reftable/system.h | 32 ++++++ reftable/test_framework.c | 23 ++++ reftable/test_framework.h | 53 +++++++++ t/helper/test-reftable.c | 9 ++ t/helper/test-tool.c | 3 +- t/helper/test-tool.h | 1 + t/t0032-reftable-unittest.sh | 15 +++ 16 files changed, 563 insertions(+), 7 deletions(-) create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/basics_test.c create mode 100644 reftable/publicbasics.c create mode 100644 reftable/reftable-malloc.h create mode 100644 reftable/reftable-tests.h create mode 100644 reftable/system.h create mode 100644 reftable/test_framework.c create mode 100644 reftable/test_framework.h create mode 100644 t/helper/test-reftable.c create mode 100755 t/t0032-reftable-unittest.sh
Makefile +22 -3
@@ -743,6 +743,7 @@ TEST_BUILTINS_OBJS += test-read-cache.o TEST_BUILTINS_OBJS += test-read-graph.o TEST_BUILTINS_OBJS += test-read-midx.o TEST_BUILTINS_OBJS += test-ref-store.o +TEST_BUILTINS_OBJS += test-reftable.o TEST_BUILTINS_OBJS += test-regex.o TEST_BUILTINS_OBJS += test-repository.o TEST_BUILTINS_OBJS += test-revision-walking.o
@@ -821,6 +822,8 @@ TEST_SHELL_PATH = $(SHELL_PATH) LIB_FILE = libgit.a XDIFF_LIB = xdiff/lib.a +REFTABLE_LIB = reftable/libreftable.a +REFTABLE_TEST_LIB = reftable/libreftable_test.a GENERATED_H += command-list.h GENERATED_H += config-list.h
@@ -1195,7 +1198,7 @@ THIRD_PARTY_SOURCES += compat/regex/% THIRD_PARTY_SOURCES += sha1collisiondetection/% THIRD_PARTY_SOURCES += sha1dc/% -GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) +GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) EXTLIBS = GIT_USER_AGENT = git/$(GIT_VERSION)
@@ -2446,7 +2449,15 @@ XDIFF_OBJS += xdiff/xutils.o .PHONY: xdiff-objs xdiff-objs: $(XDIFF_OBJS) +REFTABLE_OBJS += reftable/basics.o +REFTABLE_OBJS += reftable/error.o +REFTABLE_OBJS += reftable/publicbasics.o + +REFTABLE_TEST_OBJS += reftable/test_framework.o +REFTABLE_TEST_OBJS += reftable/basics_test.o + TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) + .PHONY: test-objs test-objs: $(TEST_OBJS)
@@ -2462,6 +2473,8 @@ OBJECTS += $(PROGRAM_OBJS) OBJECTS += $(TEST_OBJS) OBJECTS += $(XDIFF_OBJS) OBJECTS += $(FUZZ_OBJS) +OBJECTS += $(REFTABLE_OBJS) $(REFTABLE_TEST_OBJS) + ifndef NO_CURL OBJECTS += http.o http-walker.o remote-curl.o endif
@@ -2612,6 +2625,12 @@ $(LIB_FILE): $(LIB_OBJS) $(XDIFF_LIB): $(XDIFF_OBJS) $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ +$(REFTABLE_LIB): $(REFTABLE_OBJS) + $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ + +$(REFTABLE_TEST_LIB): $(REFTABLE_TEST_OBJS) + $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ + export DEFAULT_EDITOR DEFAULT_PAGER Documentation/GIT-EXCLUDED-PROGRAMS: FORCE
@@ -2904,7 +2923,7 @@ perf: all t/helper/test-tool$X: $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) -t/helper/test-%$X: t/helper/test-%.o GIT-LDFLAGS $(GITLIBS) +t/helper/test-%$X: t/helper/test-%.o GIT-LDFLAGS $(GITLIBS) $(REFTABLE_TEST_LIB) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(filter %.a,$^) $(LIBS) check-sha1:: t/helper/test-tool$X
@@ -3234,7 +3253,7 @@ cocciclean: clean: profile-clean coverage-clean cocciclean $(RM) *.res $(RM) $(OBJECTS) - $(RM) $(LIB_FILE) $(XDIFF_LIB) + $(RM) $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) $(REFTABLE_TEST_LIB) $(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) git$X $(RM) $(TEST_PROGRAMS) $(RM) $(FUZZ_PROGRAMS)
contrib/buildsystems/CMakeLists.txt +12 -2
@@ -640,6 +640,12 @@ parse_makefile_for_sources(libxdiff_SOURCES "XDIFF_OBJS") list(TRANSFORM libxdiff_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") add_library(xdiff STATIC ${libxdiff_SOURCES}) +#reftable +parse_makefile_for_sources(reftable_SOURCES "REFTABLE_OBJS") + +list(TRANSFORM reftable_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") +add_library(reftable STATIC ${reftable_SOURCES}) + if(WIN32) if(NOT MSVC)#use windres when compiling with gcc and clang add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/git.res
@@ -662,7 +668,7 @@ endif() #link all required libraries to common-main add_library(common-main OBJECT ${CMAKE_SOURCE_DIR}/common-main.c) -target_link_libraries(common-main libgit xdiff ${ZLIB_LIBRARIES}) +target_link_libraries(common-main libgit xdiff reftable ${ZLIB_LIBRARIES}) if(Intl_FOUND) target_link_libraries(common-main ${Intl_LIBRARIES}) endif()
@@ -902,11 +908,15 @@ if(BUILD_TESTING) add_executable(test-fake-ssh ${CMAKE_SOURCE_DIR}/t/helper/test-fake-ssh.c) target_link_libraries(test-fake-ssh common-main) +#reftable-tests +parse_makefile_for_sources(test-reftable_SOURCES "REFTABLE_TEST_OBJS") +list(TRANSFORM test-reftable_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") + #test-tool parse_makefile_for_sources(test-tool_SOURCES "TEST_BUILTINS_OBJS") list(TRANSFORM test-tool_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/t/helper/") -add_executable(test-tool ${CMAKE_SOURCE_DIR}/t/helper/test-tool.c ${test-tool_SOURCES}) +add_executable(test-tool ${CMAKE_SOURCE_DIR}/t/helper/test-tool.c ${test-tool_SOURCES} ${test-reftable_SOURCES}) target_link_libraries(test-tool common-main) set_target_properties(test-fake-ssh test-tool
contrib/buildsystems/Generators/Vcxproj.pm +10 -1
@@ -77,7 +77,7 @@ sub createProject { my $libs_release = "\n "; my $libs_debug = "\n "; if (!$static_library) { - $libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); + $libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib|reftable\/libreftable\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); $libs_debug = $libs_release; $libs_debug =~ s/zlib\.lib/zlibd\.lib/g; $libs_debug =~ s/libexpat\.lib/libexpatd\.lib/g;
@@ -232,6 +232,7 @@ EOM EOM if (!$static_library || $target =~ 'vcs-svn' || $target =~ 'xdiff') { my $uuid_libgit = $$build_structure{"LIBS_libgit_GUID"}; + my $uuid_libreftable = $$build_structure{"LIBS_reftable/libreftable_GUID"}; my $uuid_xdiff_lib = $$build_structure{"LIBS_xdiff/lib_GUID"}; print F << "EOM";
@@ -241,6 +242,14 @@ EOM <ReferenceOutputAssembly>false</ReferenceOutputAssembly> </ProjectReference> EOM + if (!($name =~ /xdiff|libreftable/)) { + print F << "EOM"; + <ProjectReference Include="$cdup\\reftable\\libreftable\\libreftable.vcxproj"> + <Project>$uuid_libreftable</Project> + <ReferenceOutputAssembly>false</ReferenceOutputAssembly> + </ProjectReference> +EOM + } if (!($name =~ 'xdiff')) { print F << "EOM"; <ProjectReference Include="$cdup\\xdiff\\lib\\xdiff_lib.vcxproj">
reftable/basics.c new file +128 -0
128 lines not on this page · raw message
reftable/basics.h new file +60 -0
60 lines not on this page · raw message
reftable/basics_test.c new file +98 -0
98 lines not on this page · raw message
reftable/publicbasics.c new file +58 -0
58 lines not on this page · raw message
reftable/reftable-malloc.h new file +18 -0
@@ -0,0 +1,18 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_H +#define REFTABLE_H + +#include <stddef.h> + +/* Overrides the functions to use for memory management. */ +void reftable_set_alloc (void *(*malloc)(size_t ), +void *(*realloc)(void *,size_t ),void (*free)(void *)); + +#endif
reftable/reftable-tests.h new file +22 -0
@@ -0,0 +1,22 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_TESTS_H +#define REFTABLE_TESTS_H + +int basics_test_main (int argc,const char **argv); +int block_test_main (int argc,const char **argv); +int merged_test_main (int argc,const char **argv); +int record_test_main (int argc,const char **argv); +int refname_test_main (int argc,const char **argv); +int reftable_test_main (int argc,const char **argv); +int stack_test_main (int argc,const char **argv); +int tree_test_main (int argc,const char **argv); +int reftable_dump_main (int argc,char *const *argv); + +#endif
reftable/system.h new file +32 -0
@@ -0,0 +1,32 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef SYSTEM_H +#define SYSTEM_H + +/* This header glues the reftable library to the rest of Git */ + +#include "git-compat-util.h" +#include "strbuf.h" +#include "hash.h" /* hash ID, sizes.*/ +#include "dir.h" /* remove_dir_recursively, for tests.*/ + +#include <zlib.h> + +#ifdef NO_UNCOMPRESS2 +/* +* This is uncompress2, which is only available in zlib >= 1.2.9 +* (released as of early 2017) +*/ +int uncompress2 (Bytef *dest,uLongf *destLen,const Bytef *source, +uLong *sourceLen); +#endif + +int hash_size (uint32_t id); + +#endif
reftable/test_framework.c new file +23 -0
@@ -0,0 +1,23 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "system.h" +#include "test_framework.h" + +#include "basics.h" + +void set_test_hash (uint8_t *p,int i) +{ +memset (p, (uint8_t )i,hash_size (GIT_SHA1_FORMAT_ID )); +} + +ssize_t strbuf_add_void (void *b,const void *data,size_t sz) +{ +strbuf_add (b, data, sz); +return sz; +}
reftable/test_framework.h new file +53 -0
53 lines not on this page · raw message
t/helper/test-reftable.c new file +9 -0
@@ -0,0 +1,9 @@ +#include "reftable/reftable-tests.h" +#include "test-tool.h" + +int cmd__reftable (int argc,const char **argv) +{ +basics_test_main (argc, argv); + +return 0 ; +}
t/helper/test-tool.c +2 -1
@@ -53,13 +53,14 @@ static struct test_cmd cmds[] = { { "pcre2-config", cmd__pcre2_config }, { "pkt-line", cmd__pkt_line }, { "prio-queue", cmd__prio_queue }, - { "proc-receive", cmd__proc_receive}, + { "proc-receive", cmd__proc_receive }, { "progress", cmd__progress }, { "reach", cmd__reach }, { "read-cache", cmd__read_cache }, { "read-graph", cmd__read_graph }, { "read-midx", cmd__read_midx }, { "ref-store", cmd__ref_store }, + { "reftable", cmd__reftable }, { "regex", cmd__regex }, { "repository", cmd__repository }, { "revision-walking", cmd__revision_walking },
t/helper/test-tool.h +1 -0
@@ -49,6 +49,7 @@ int cmd__read_cache(int argc, const char **argv);int cmd__read_graph (int argc,const char **argv);int cmd__read_midx (int argc,const char **argv);int cmd__ref_store (int argc,const char **argv); +int cmd__reftable (int argc,const char **argv);int cmd__regex (int argc,const char **argv);int cmd__repository (int argc,const char **argv);int cmd__revision_walking (int argc,const char **argv);
t/t0032-reftable-unittest.sh new file +15 -0
@@ -0,0 +1,15 @@ +#!/bin/sh +# +# Copyright (c) 2020 Google LLC +# + +test_description='reftable unittests' + +. ./test-lib.sh + +test_expect_success 'unittests' ' + TMPDIR=$(pwd) && export TMPDIR && + test-tool reftable +' + +test_done
Signature · 1 line
gitgitgadget
PATCH v2 5/19 reftable: add blocksource, an abstraction for random access reads
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The reftable format is usually used with files for storage. However, we abstract away this using the blocksource data structure. This has two advantages:
* log blocks are zlib compressed, and handling them is simplified if we can discard byte segments from within the block layer.
* for unittests, it is useful to read and write in-memory. The blocksource allows us to abstract the data away from on-disk files.
--- Makefile | 1 + reftable/blocksource.c | 148 ++++++++++++++++++++++++++++++++ reftable/blocksource.h | 22 +++++ reftable/reftable-blocksource.h | 49 +++++++++++ 4 files changed, 220 insertions(+) create mode 100644 reftable/blocksource.c create mode 100644 reftable/blocksource.h create mode 100644 reftable/reftable-blocksource.h
Makefile +1 -0
@@ -2451,6 +2451,7 @@ xdiff-objs: $(XDIFF_OBJS) REFTABLE_OBJS += reftable/basics.o REFTABLE_OBJS += reftable/error.o +REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o REFTABLE_TEST_OBJS += reftable/test_framework.o
reftable/blocksource.c new file +148 -0
148 lines not on this page · raw message
reftable/blocksource.h new file +22 -0
@@ -0,0 +1,22 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef BLOCKSOURCE_H +#define BLOCKSOURCE_H + +#include "system.h" + +struct reftable_block_source ; + +/* Create an in-memory block source for reading reftables */ +void block_source_from_strbuf (struct reftable_block_source *bs, +struct strbuf *buf); + +struct reftable_block_source malloc_block_source (void ); + +#endif
reftable/reftable-blocksource.h new file +49 -0
49 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v2 6/19 reftable: (de)serialization for the polymorphic record type.
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The reftable format is structured as a sequence of blocks, and each block contains a sequence of prefix-compressed key-value records. There are 4 types of records, and they have similarities in how they must be handled. This is achieved by introducing a polymorphic 'record' type that encapsulates ref, log, index and object records.
--- Makefile | 2 + reftable/constants.h | 21 + reftable/record.c | 1212 ++++++++++++++++++++++++++++++++++++ reftable/record.h | 139 +++++ reftable/record_test.c | 412 ++++++++++++ reftable/reftable-record.h | 114 ++++ t/helper/test-reftable.c | 2 +- 7 files changed, 1901 insertions(+), 1 deletion(-) create mode 100644 reftable/constants.h create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/record_test.c create mode 100644 reftable/reftable-record.h
Makefile +2 -0
@@ -2453,7 +2453,9 @@ REFTABLE_OBJS += reftable/basics.o REFTABLE_OBJS += reftable/error.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o +REFTABLE_OBJS += reftable/record.o +REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o REFTABLE_TEST_OBJS += reftable/basics_test.o
reftable/constants.h new file +21 -0
@@ -0,0 +1,21 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef CONSTANTS_H +#define CONSTANTS_H + +#define BLOCK_TYPE_LOG 'g' +#define BLOCK_TYPE_INDEX 'i' +#define BLOCK_TYPE_REF 'r' +#define BLOCK_TYPE_OBJ 'o' +#define BLOCK_TYPE_ANY 0 + +#define MAX_RESTARTS ((1 << 16) - 1) +#define DEFAULT_BLOCK_SIZE 4096 + +#endif
reftable/record.c new file +1212 -0
1,212 lines not on this page · raw message
reftable/record.h new file +139 -0
139 lines not on this page · raw message
reftable/record_test.c new file +412 -0
412 lines not on this page · raw message
reftable/reftable-record.h new file +114 -0
114 lines not on this page · raw message
t/helper/test-reftable.c +1 -1
@@ -4,6 +4,6 @@int cmd__reftable (int argc,const char **argv) {basics_test_main (argc, argv); - +record_test_main (argc, argv);return 0 ; }
Signature · 1 line
gitgitgadget
PATCH v2 7/19 Provide zlib's uncompress2 from compat/zlib-compat.c
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This will be needed for reading reflog blocks in reftable.
--- Makefile | 7 +++ ci/lib.sh | 1 + compat/.gitattributes | 1 + compat/zlib-uncompress2.c | 95 +++++++++++++++++++++++++++++++++++++++ config.mak.uname | 5 +++ configure.ac | 13 ++++++ 6 files changed, 122 insertions(+) create mode 100644 compat/.gitattributes create mode 100644 compat/zlib-uncompress2.c
Makefile +7 -0
@@ -256,6 +256,8 @@ all:: # # Define NO_DEFLATE_BOUND if your zlib does not have deflateBound. # +# Define NO_UNCOMPRESS2 if your zlib does not have uncompress2. +# # Define NO_NORETURN if using buggy versions of gcc 4.6+ and profile feedback, # as the compiler can crash (http://gcc.gnu.org/bugzilla/show_bug.cgi?id=49299) #
@@ -1738,6 +1740,11 @@ ifdef NO_DEFLATE_BOUND BASIC_CFLAGS += -DNO_DEFLATE_BOUND endif +ifdef NO_UNCOMPRESS2 + BASIC_CFLAGS += -DNO_UNCOMPRESS2 + REFTABLE_OBJS += compat/zlib-uncompress2.o +endif + ifdef NO_POSIX_GOODIES BASIC_CFLAGS += -DNO_POSIX_GOODIES endif
ci/lib.sh +1 -0
@@ -224,6 +224,7 @@ linux-gcc-default) ;; Linux32) CC=gcc + MAKEFLAGS="$MAKEFLAGS NO_UNCOMPRESS2=1" ;; linux-musl) CC=gcc
compat/.gitattributes new file +1 -0
@@ -0,0 +1 @@ +/zlib-uncompress2.c whitespace=-indent-with-non-tab,-trailing-space
compat/zlib-uncompress2.c new file +95 -0
95 lines not on this page · raw message
config.mak.uname +5 -0
@@ -258,6 +258,11 @@ ifeq ($(uname_S),FreeBSD) FILENO_IS_A_MACRO = UnfortunatelyYes endif ifeq ($(uname_S),OpenBSD) + NO_UNCOMPRESS2 = YesPlease + # Versions < 7.0 need compatibility layer + ifeq ($(shell expr "$(uname_R)" : "[1-6]\."),2) + NO_UNCOMPRESS2 = UnfortunatelyYes + endif NO_STRCASESTR = YesPlease NO_MEMMEM = YesPlease USE_ST_TIMESPEC = YesPlease
configure.ac +13 -0
@@ -672,9 +672,22 @@ AC_LINK_IFELSE([ZLIBTEST_SRC], NO_DEFLATE_BOUND=yes]) LIBS="$old_LIBS" +AC_DEFUN([ZLIBTEST_UNCOMPRESS2_SRC], [ +AC_LANG_PROGRAM([#include <zlib.h>], + [uncompress2(NULL,NULL,NULL,NULL);])]) +AC_MSG_CHECKING([for uncompress2 in -lz]) +old_LIBS="$LIBS" +LIBS="$LIBS -lz" +AC_LINK_IFELSE([ZLIBTEST_UNCOMPRESS2_SRC], + [AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no]) + NO_UNCOMPRESS2=yes]) +LIBS="$old_LIBS" + GIT_UNSTASH_FLAGS($ZLIB_PATH) GIT_CONF_SUBST([NO_DEFLATE_BOUND]) +GIT_CONF_SUBST([NO_UNCOMPRESS2]) # # Define NEEDS_SOCKET if linking with libc is not enough (SunOS,
Signature · 1 line
gitgitgadget
Re: [PATCH v2 07/19] Provide zlib's uncompress2 from compat/zlib-compat.c
config.mak.uname ifeq ($(uname_S),FreeBSD) · line 261
Quoted 9 lines · Han-Wen Nienhuys via GitGitGadget
config.mak.uname +1
-0@@ -258,6 +258,11 @@ ifeq ($(uname_S),FreeBSD) FILENO_IS_A_MACRO = UnfortunatelyYes endif ifeq ($(uname_S),OpenBSD) + NO_UNCOMPRESS2 = YesPlease
This first line is no longer needed and is counter productive The logic in the next 4 sets this variable correctly, otherwise
config.mak.uname ifeq ($(uname_S),FreeBSD) · line 268
Quoted 7 lines
config.mak.uname +4
-0+ # Versions < 7.0 need compatibility layer + ifeq ($(shell expr "$(uname_R)" : "[1-6]\."),2) + NO_UNCOMPRESS2 = UnfortunatelyYes + endif NO_STRCASESTR = YesPlease NO_MEMMEM = YesPlease USE_ST_TIMESPEC = YesPlease
Carlo
PATCH v2 8/19 reftable: reading/writing blocks
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The reftable format is structured as a sequence of block. Within a block, records are prefix compressed, with an index of offsets for fully expand keys to enable binary search within blocks.
This commit provides the logic to read and write these blocks.
--- Makefile | 2 + reftable/block.c | 439 +++++++++++++++++++++++++++++++++++++++ reftable/block.h | 127 +++++++++++ reftable/block_test.c | 120 +++++++++++ t/helper/test-reftable.c | 1 + 5 files changed, 689 insertions(+) create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/block_test.c
Makefile +2 -0
@@ -2458,10 +2458,12 @@ xdiff-objs: $(XDIFF_OBJS) REFTABLE_OBJS += reftable/basics.o REFTABLE_OBJS += reftable/error.o +REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o REFTABLE_OBJS += reftable/record.o +REFTABLE_TEST_OBJS += reftable/block_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o REFTABLE_TEST_OBJS += reftable/basics_test.o
reftable/block.c new file +439 -0
439 lines not on this page · raw message
reftable/block.h new file +127 -0
127 lines not on this page · raw message
reftable/block_test.c new file +120 -0
120 lines not on this page · raw message
t/helper/test-reftable.c +1 -0
@@ -4,6 +4,7 @@int cmd__reftable (int argc,const char **argv) {basics_test_main (argc, argv); +block_test_main (argc, argv);record_test_main (argc, argv);return 0 ; }
Signature · 1 line
gitgitgadget
Re: [PATCH v2 08/19] reftable: reading/writing blocks
Quoted 25 lines · Han-Wen Nienhuys via GitGitGadget
From: Han-Wen Nienhuys <[email protected]>
The reftable format is structured as a sequence of block. Within a block, records are prefix compressed, with an index of offsets for fully expand keys to enable binary search within blocks.
[...]reftable/block.c new file +19
-0+if (typ ==BLOCK_TYPE_LOG ) { +int block_header_skip =4 + header_off; +uLongf dst_len = sz - block_header_skip;/* total size of dest +buffer. */ +uLongf src_len = block->len - block_header_skip; +/* Log blocks specify the *uncompressed* size in their header. +*/ +uint8_t *uncompressed =reftable_malloc (sz); + +/* Copy over the block header verbatim. It's not compressed. */ +memcpy (uncompressed, block->data, block_header_skip); + +/* Uncompress */ +if (Z_OK != +uncompress2 (uncompressed + block_header_skip, &dst_len, + block->data + block_header_skip, &src_len)) { +reftable_free (uncompressed); +return REFTABLE_ZLIB_ERROR ; + }
reftable/block.c line 229
Quoted 10 lines
reftable/block.c new file +10
-0+ +if (dst_len + block_header_skip != sz) +return REFTABLE_FORMAT_ERROR ; + +/* We're done with the input data. */ +reftable_block_done (block); + block->data = uncompressed; + block->len = sz; + block->source =malloc_block_source (); + full_block_size = src_len + block_header_skip;
I haven't tried to re-implement this, but in 07/19 we're adding an uncompress2() fallaback function, and here's the only place where it'll get used.
Looking at the uncompress2() implementation isn't this a rather trivial wrapper for something we'd get from git.git's zlib.c, i.e. you seem to just need the "avail_in" here, which we're checking in various other parts of the codebase.
I suspect that the reason for this is that this code is trying to keep at arms length from the rest of git.git, but if so some ifdef here if it's compiling in git.git seems better than having a compat interface in git.git just for doing something its zlib.c API provides.
Anyway, re Junio's ask if this is ready for "next" in the latest What's Cooking I don't think this should be a blocker, just wondering if anyone's interested in some post-cleanup at some point, and whether I'm missing something about uncompress2() v.s. zlib.c.
PATCH v2 9/19 reftable: a generic binary tree implementation
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The reftable format includes support for an (OID => ref) map. This map can speed up visibility and reachability checks. In particular, various operations along the fetch/push path within Gerrit have ben sped up by using this structure.
The map is constructed with help of a binary tree. Object IDs are hashes, so they are uniformly distributed. Hence, the tree does not attempt forced rebalancing.
--- Makefile | 4 ++- reftable/tree.c | 63 ++++++++++++++++++++++++++++++++++++++++ reftable/tree.h | 34 ++++++++++++++++++++++ reftable/tree_test.c | 61 ++++++++++++++++++++++++++++++++++++++ t/helper/test-reftable.c | 1 + 5 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100644 reftable/tree_test.c
Makefile +3 -1
@@ -2462,11 +2462,13 @@ REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o REFTABLE_OBJS += reftable/record.o +REFTABLE_OBJS += reftable/tree.o +REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o -REFTABLE_TEST_OBJS += reftable/basics_test.o +REFTABLE_TEST_OBJS += reftable/tree_test.o TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS))
reftable/tree.c new file +63 -0
63 lines not on this page · raw message
reftable/tree.h new file +34 -0
@@ -0,0 +1,34 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef TREE_H +#define TREE_H + +/* tree_node is a generic binary search tree. */ +struct tree_node { +void *key; +struct tree_node *left, *right; +}; + +/* looks for `key` in `rootp` using `compare` as comparison function. If insert +* is set, insert the key if it's not found. Else, return NULL. +*/ +struct tree_node *tree_search (void *key,struct tree_node **rootp, +int (*compare)(const void *,const void *), +int insert); + +/* performs an infix walk of the tree. */ +void infix_walk (struct tree_node *t,void (*action)(void *arg,void *key), +void *arg); + +/* +* deallocates the tree nodes recursively. Keys should be deallocated separately +* by walking over the tree. */ +void tree_free (struct tree_node *t); + +#endif
reftable/tree_test.c new file +61 -0
61 lines not on this page · raw message
t/helper/test-reftable.c +1 -0
@@ -6,5 +6,6 @@ int cmd__reftable(int argc, const char **argv)basics_test_main (argc, argv);block_test_main (argc, argv);record_test_main (argc, argv); +tree_test_main (argc, argv);return 0 ; }
Signature · 1 line
gitgitgadget
PATCH v2 10/19 reftable: write reftable files
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
--- Makefile | 1 + reftable/reftable-writer.h | 148 ++++++++ reftable/writer.c | 690 +++++++++++++++++++++++++++++++++++++ reftable/writer.h | 50 +++ 4 files changed, 889 insertions(+) create mode 100644 reftable/reftable-writer.h create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h
Makefile +1 -0
@@ -2463,6 +2463,7 @@ REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o REFTABLE_OBJS += reftable/record.o REFTABLE_OBJS += reftable/tree.o +REFTABLE_OBJS += reftable/writer.o REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o
reftable/reftable-writer.h new file +148 -0
148 lines not on this page · raw message
reftable/writer.c new file +690 -0
690 lines not on this page · raw message
reftable/writer.h new file +50 -0
50 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v2 11/19 reftable: generic interface to tables
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
--- Makefile | 3 + reftable/generic.c | 169 +++++++++++++++++++++++++++++++++++ reftable/generic.h | 32 +++++++ reftable/reftable-generic.h | 47 ++++++++++ reftable/reftable-iterator.h | 39 ++++++++ reftable/reftable.c | 115 ++++++++++++++++++++++++ 6 files changed, 405 insertions(+) create mode 100644 reftable/generic.c create mode 100644 reftable/generic.h create mode 100644 reftable/reftable-generic.h create mode 100644 reftable/reftable-iterator.h create mode 100644 reftable/reftable.c
Makefile +3 -0
@@ -2462,6 +2462,9 @@ REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o REFTABLE_OBJS += reftable/record.o +REFTABLE_OBJS += reftable/refname.o +REFTABLE_OBJS += reftable/generic.o +REFTABLE_OBJS += reftable/stack.o REFTABLE_OBJS += reftable/tree.o REFTABLE_OBJS += reftable/writer.o
reftable/generic.c new file +169 -0
169 lines not on this page · raw message
reftable/generic.h new file +32 -0
@@ -0,0 +1,32 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef GENERIC_H +#define GENERIC_H + +#include "record.h" +#include "reftable-generic.h" + +/* generic interface to reftables */ +struct reftable_table_vtable { +int (*seek_record)(void *tab,struct reftable_iterator *it, +struct reftable_record *); +uint32_t (*hash_id)(void *tab); +uint64_t (*min_update_index)(void *tab); +uint64_t (*max_update_index)(void *tab); +}; + +struct reftable_iterator_vtable { +int (*next)(void *iter_arg,struct reftable_record *rec); +void (*close)(void *iter_arg); +}; + +void iterator_set_empty (struct reftable_iterator *it); +int iterator_next (struct reftable_iterator *it,struct reftable_record *rec); + +#endif
reftable/reftable-generic.h new file +47 -0
47 lines not on this page · raw message
reftable/reftable-iterator.h new file +39 -0
@@ -0,0 +1,39 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_ITERATOR_H +#define REFTABLE_ITERATOR_H + +#include "reftable-record.h" + +struct reftable_iterator_vtable ; + +/* iterator is the generic interface for walking over data stored in a +* reftable. +*/ +struct reftable_iterator { +struct reftable_iterator_vtable *ops; +void *iter_arg; +}; + +/* reads the next reftable_ref_record. Returns < 0 for error, 0 for OK and > 0: +* end of iteration. +*/ +int reftable_iterator_next_ref (struct reftable_iterator *it, +struct reftable_ref_record *ref); + +/* reads the next reftable_log_record. Returns < 0 for error, 0 for OK and > 0: +* end of iteration. +*/ +int reftable_iterator_next_log (struct reftable_iterator *it, +struct reftable_log_record *log); + +/* releases resources associated with an iterator. */ +void reftable_iterator_destroy (struct reftable_iterator *it); + +#endif
reftable/reftable.c new file +115 -0
115 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v2 12/19 reftable: read reftable files
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This supports reading a single reftable file.
The commit introduces an abstract iterator type, which captures the usecases both of reading individual refs, and iterating over a segment of the ref namespace.
--- Makefile | 2 + reftable/iter.c | 194 +++++++++ reftable/iter.h | 69 ++++ reftable/reader.c | 801 +++++++++++++++++++++++++++++++++++++ reftable/reader.h | 66 +++ reftable/reftable-reader.h | 101 +++++ 6 files changed, 1233 insertions(+) create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/reftable-reader.h
Makefile +2 -0
@@ -2460,7 +2460,9 @@ REFTABLE_OBJS += reftable/basics.o REFTABLE_OBJS += reftable/error.o REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o +REFTABLE_OBJS += reftable/iter.o REFTABLE_OBJS += reftable/publicbasics.o +REFTABLE_OBJS += reftable/reader.o REFTABLE_OBJS += reftable/record.o REFTABLE_OBJS += reftable/refname.o REFTABLE_OBJS += reftable/generic.o
reftable/iter.c new file +194 -0
194 lines not on this page · raw message
reftable/iter.h new file +69 -0
69 lines not on this page · raw message
reftable/reader.c new file +801 -0
801 lines not on this page · raw message
reftable/reader.h new file +66 -0
66 lines not on this page · raw message
reftable/reftable-reader.h new file +101 -0
101 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v2 13/19 reftable: reftable file level tests
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
With support for reading and writing files in place, we can construct files (in memory) and attempt to read them back.
Because some sections of the format are optional (eg. indices, log entries), we have to exercise this code using multiple sizes of input data
--- Makefile | 1 + reftable/readwrite_test.c | 652 ++++++++++++++++++++++++++++++++++++++ reftable/reftable-tests.h | 2 +- t/helper/test-reftable.c | 1 + 4 files changed, 655 insertions(+), 1 deletion(-) create mode 100644 reftable/readwrite_test.c
Makefile +1 -0
@@ -2473,6 +2473,7 @@ REFTABLE_OBJS += reftable/writer.o REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o REFTABLE_TEST_OBJS += reftable/record_test.o +REFTABLE_TEST_OBJS += reftable/readwrite_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o REFTABLE_TEST_OBJS += reftable/tree_test.o
reftable/readwrite_test.c new file +652 -0
652 lines not on this page · raw message
reftable/reftable-tests.h +1 -1
@@ -14,7 +14,7 @@ int block_test_main(int argc, const char **argv);int merged_test_main (int argc,const char **argv);int record_test_main (int argc,const char **argv);int refname_test_main (int argc,const char **argv); -int reftable_test_main (int argc,const char **argv); +int readwrite_test_main (int argc,const char **argv);int stack_test_main (int argc,const char **argv);int tree_test_main (int argc,const char **argv);int reftable_dump_main (int argc,char *const *argv);
t/helper/test-reftable.c +1 -0
@@ -6,6 +6,7 @@ int cmd__reftable(int argc, const char **argv)basics_test_main (argc, argv);block_test_main (argc, argv);record_test_main (argc, argv); +readwrite_test_main (argc, argv);tree_test_main (argc, argv);return 0 ; }
Signature · 1 line
gitgitgadget
PATCH v2 14/19 reftable: add a heap-based priority queue for reftable records
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This is needed to create a merged view multiple reftables
--- Makefile | 2 + reftable/pq.c | 105 ++++++++++++++++++++++++++++++++++++++ reftable/pq.h | 33 ++++++++++++ reftable/pq_test.c | 82 +++++++++++++++++++++++++++++ reftable/reftable-tests.h | 1 + t/helper/test-reftable.c | 1 + 6 files changed, 224 insertions(+) create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/pq_test.c
Makefile +2 -0
@@ -2462,6 +2462,7 @@ REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/iter.o REFTABLE_OBJS += reftable/publicbasics.o +REFTABLE_OBJS += reftable/pq.o REFTABLE_OBJS += reftable/reader.o REFTABLE_OBJS += reftable/record.o REFTABLE_OBJS += reftable/refname.o
@@ -2472,6 +2473,7 @@ REFTABLE_OBJS += reftable/writer.o REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o +REFTABLE_TEST_OBJS += reftable/pq_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/readwrite_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o
reftable/pq.c new file +105 -0
105 lines not on this page · raw message
reftable/pq.h new file +33 -0
@@ -0,0 +1,33 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef PQ_H +#define PQ_H + +#include "record.h" + +struct pq_entry { +int index; +struct reftable_record rec; +}; + +struct merged_iter_pqueue { +struct pq_entry *heap; +size_t len; +size_t cap; +}; + +struct pq_entry merged_iter_pqueue_top (struct merged_iter_pqueue pq); +int merged_iter_pqueue_is_empty (struct merged_iter_pqueue pq); +void merged_iter_pqueue_check (struct merged_iter_pqueue pq); +struct pq_entry merged_iter_pqueue_remove (struct merged_iter_pqueue *pq); +void merged_iter_pqueue_add (struct merged_iter_pqueue *pq,struct pq_entry e); +void merged_iter_pqueue_release (struct merged_iter_pqueue *pq); +int pq_less (struct pq_entry *a,struct pq_entry *b); + +#endif
reftable/pq_test.c new file +82 -0
82 lines not on this page · raw message
reftable/reftable-tests.h +1 -0
@@ -12,6 +12,7 @@ https://developers.google.com/open-source/licenses/bsdint basics_test_main (int argc,const char **argv);int block_test_main (int argc,const char **argv);int merged_test_main (int argc,const char **argv); +int pq_test_main (int argc,const char **argv);int record_test_main (int argc,const char **argv);int refname_test_main (int argc,const char **argv);int readwrite_test_main (int argc,const char **argv);
t/helper/test-reftable.c +1 -0
@@ -5,6 +5,7 @@ int cmd__reftable(int argc, const char **argv) {basics_test_main (argc, argv);block_test_main (argc, argv); +pq_test_main (argc, argv);record_test_main (argc, argv);readwrite_test_main (argc, argv);tree_test_main (argc, argv);
Signature · 1 line
gitgitgadget
PATCH v2 15/19 reftable: add merged table view
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This adds an abstract, read-only interface to the ref database.
This primitive is used to construct the read view of the ref database (the read view is constructed by merging several *.ref files). It also provides the mechanism to provide a unified view of the refs in the main repository and the per-worktree refs.
--- Makefile | 2 + reftable/merged.c | 362 +++++++++++++++++++++++++++++++++++++ reftable/merged.h | 35 ++++ reftable/merged_test.c | 292 ++++++++++++++++++++++++++++++ reftable/reftable-merged.h | 72 ++++++++ t/helper/test-reftable.c | 1 + 6 files changed, 764 insertions(+) create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/merged_test.c create mode 100644 reftable/reftable-merged.h
Makefile +2 -0
@@ -2462,6 +2462,7 @@ REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/iter.o REFTABLE_OBJS += reftable/publicbasics.o +REFTABLE_OBJS += reftable/merged.o REFTABLE_OBJS += reftable/pq.o REFTABLE_OBJS += reftable/reader.o REFTABLE_OBJS += reftable/record.o
@@ -2473,6 +2474,7 @@ REFTABLE_OBJS += reftable/writer.o REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o +REFTABLE_TEST_OBJS += reftable/merged_test.o REFTABLE_TEST_OBJS += reftable/pq_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/readwrite_test.o
reftable/merged.c new file +362 -0
362 lines not on this page · raw message
reftable/merged.h new file +35 -0
@@ -0,0 +1,35 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef MERGED_H +#define MERGED_H + +#include "pq.h" + +struct reftable_merged_table { +struct reftable_table *stack; +size_t stack_len; +uint32_t hash_id; +int suppress_deletions; + +uint64_t min; +uint64_t max; +}; + +struct merged_iter { +struct reftable_iterator *stack; +uint32_t hash_id; +size_t stack_len; +uint8_t typ; +int suppress_deletions; +struct merged_iter_pqueue pq; +}; + +void merged_table_release (struct reftable_merged_table *mt); + +#endif
reftable/merged_test.c new file +292 -0
292 lines not on this page · raw message
reftable/reftable-merged.h new file +72 -0
72 lines not on this page · raw message
t/helper/test-reftable.c +1 -0
@@ -5,6 +5,7 @@ int cmd__reftable(int argc, const char **argv) {basics_test_main (argc, argv);block_test_main (argc, argv); +merged_test_main (argc, argv);pq_test_main (argc, argv);record_test_main (argc, argv);readwrite_test_main (argc, argv);
Signature · 1 line
gitgitgadget
PATCH v2 16/19 reftable: implement refname validation
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The packed/loose format has restrictions on refnames: a and a/b cannot coexist. This limitation does not apply to reftable per se, but must be maintained for interoperability. This code adds validation routines to abort transactions that are trying to add invalid names.
--- Makefile | 1 + reftable/refname.c | 209 +++++++++++++++++++++++++++++++++++++++ reftable/refname.h | 29 ++++++ reftable/refname_test.c | 102 +++++++++++++++++++ t/helper/test-reftable.c | 1 + 5 files changed, 342 insertions(+) create mode 100644 reftable/refname.c create mode 100644 reftable/refname.h create mode 100644 reftable/refname_test.c
Makefile +1 -0
@@ -2478,6 +2478,7 @@ REFTABLE_TEST_OBJS += reftable/merged_test.o REFTABLE_TEST_OBJS += reftable/pq_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/readwrite_test.o +REFTABLE_TEST_OBJS += reftable/refname_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o REFTABLE_TEST_OBJS += reftable/tree_test.o
reftable/refname.c new file +209 -0
209 lines not on this page · raw message
reftable/refname.h new file +29 -0
@@ -0,0 +1,29 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ +#ifndef REFNAME_H +#define REFNAME_H + +#include "reftable-record.h" +#include "reftable-generic.h" + +struct modification { +struct reftable_table tab; + +char **add; +size_t add_len; + +char **del; +size_t del_len; +}; + +int validate_ref_record_addition (struct reftable_table tab, +struct reftable_ref_record *recs,size_t sz); + +int modification_validate (struct modification *mod); + +#endif
reftable/refname_test.c new file +102 -0
102 lines not on this page · raw message
t/helper/test-reftable.c +1 -0
@@ -8,6 +8,7 @@ int cmd__reftable(int argc, const char **argv)merged_test_main (argc, argv);pq_test_main (argc, argv);record_test_main (argc, argv); +refname_test_main (argc, argv);readwrite_test_main (argc, argv);tree_test_main (argc, argv);return 0 ;
Signature · 1 line
gitgitgadget
PATCH v2 17/19 reftable: implement stack, a mutable database of reftable files.
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
--- Makefile | 1 + reftable/reftable-stack.h | 128 ++++ reftable/stack.c | 1396 +++++++++++++++++++++++++++++++++++++ reftable/stack.h | 41 ++ reftable/stack_test.c | 949 +++++++++++++++++++++++++ t/helper/test-reftable.c | 1 + 6 files changed, 2516 insertions(+) create mode 100644 reftable/reftable-stack.h create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/stack_test.c
Makefile +1 -0
@@ -2479,6 +2479,7 @@ REFTABLE_TEST_OBJS += reftable/pq_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/readwrite_test.o REFTABLE_TEST_OBJS += reftable/refname_test.o +REFTABLE_TEST_OBJS += reftable/stack_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o REFTABLE_TEST_OBJS += reftable/tree_test.o
reftable/reftable-stack.h new file +128 -0
128 lines not on this page · raw message
reftable/stack.c new file +1396 -0
1,396 lines not on this page · raw message
reftable/stack.h new file +41 -0
41 lines not on this page · raw message
reftable/stack_test.c new file +949 -0
949 lines not on this page · raw message
t/helper/test-reftable.c +1 -0
@@ -10,6 +10,7 @@ int cmd__reftable(int argc, const char **argv)record_test_main (argc, argv);refname_test_main (argc, argv);readwrite_test_main (argc, argv); +stack_test_main (argc, argv);tree_test_main (argc, argv);return 0 ; }
Signature · 1 line
gitgitgadget
PATCH v2 18/19 reftable: add dump utility
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
provide a command-line utility for inspecting individual tables, and inspecting a complete ref database
--- reftable/dump.c | 107 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 reftable/dump.c
reftable/dump.c new file +107 -0
@@ -0,0 +1,107 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "git-compat-util.h" +#include "hash.h" + +#include "reftable-blocksource.h" +#include "reftable-error.h" +#include "reftable-merged.h" +#include "reftable-record.h" +#include "reftable-tests.h" +#include "reftable-writer.h" +#include "reftable-iterator.h" +#include "reftable-reader.h" +#include "reftable-stack.h" +#include "reftable-generic.h" + +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> + +static int compact_stack (const char *stackdir) +{ +struct reftable_stack *stack =NULL ; +struct reftable_write_options cfg = {0 }; + +int err =reftable_new_stack (&stack, stackdir, cfg); +if (err <0 ) + goto done; + + err =reftable_stack_compact_all (stack,NULL ); +if (err <0 ) + goto done; +done: +if (stack) { +reftable_stack_destroy (stack); + } +return err; +} + +static void print_help (void ) +{ +printf ("usage: dump [-cst] arg\n\n" +"options: \n" +" -c compact\n" +" -t dump table\n" +" -s dump stack\n" +" -6 sha256 hash format\n" +" -h this help\n" +"\n" ); +} + +int reftable_dump_main (int argc,char *const *argv) +{ +int err =0 ; +int opt_dump_table =0 ; +int opt_dump_stack =0 ; +int opt_compact =0 ; +uint32_t opt_hash_id =GIT_SHA1_FORMAT_ID ; +const char *arg =NULL , *argv0 = argv[0 ]; + +for (; argc >1 ; argv++, argc--) +if (*argv[1 ] !='-' ) +break ; +else if (!strcmp ("-t" , argv[1 ])) + opt_dump_table =1 ; +else if (!strcmp ("-6" , argv[1 ])) + opt_hash_id =GIT_SHA256_FORMAT_ID ; +else if (!strcmp ("-s" , argv[1 ])) + opt_dump_stack =1 ; +else if (!strcmp ("-c" , argv[1 ])) + opt_compact =1 ; +else if (!strcmp ("-?" , argv[1 ]) || !strcmp ("-h" , argv[1 ])) { +print_help (); +return 2 ; + } + +if (argc !=2 ) { +fprintf (stderr,"need argument\n" ); +print_help (); +return 2 ; + } + + arg = argv[1 ]; + +if (opt_dump_table) { + err =reftable_reader_print_file (arg); + }else if (opt_dump_stack) { + err =reftable_stack_print_directory (arg, opt_hash_id); + }else if (opt_compact) { + err =compact_stack (arg); + } + +if (err <0 ) { +fprintf (stderr,"%s: %s: %s\n" , argv0, arg, +reftable_error_str (err)); +return 1 ; + } +return 0 ; +}
Signature · 1 line
gitgitgadget
PATCH v2 19/19 Add "test-tool dump-reftable" command.
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This command dumps individual tables or a stack of of tables.
--- Makefile | 1 + t/helper/test-reftable.c | 5 +++++ t/helper/test-tool.c | 1 + t/helper/test-tool.h | 1 + 4 files changed, 8 insertions(+)
Makefile +1 -0
@@ -2474,6 +2474,7 @@ REFTABLE_OBJS += reftable/writer.o REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o +REFTABLE_TEST_OBJS += reftable/dump.o REFTABLE_TEST_OBJS += reftable/merged_test.o REFTABLE_TEST_OBJS += reftable/pq_test.o REFTABLE_TEST_OBJS += reftable/record_test.o
t/helper/test-reftable.c +5 -0
@@ -14,3 +14,8 @@ int cmd__reftable(int argc, const char **argv)tree_test_main (argc, argv);return 0 ; } + +int cmd__dump_reftable (int argc,const char **argv) +{ +return reftable_dump_main (argc, (char *const *)argv); +}
t/helper/test-tool.c +1 -0
@@ -61,6 +61,7 @@ static struct test_cmd cmds[] = { { "read-midx", cmd__read_midx }, { "ref-store", cmd__ref_store }, { "reftable", cmd__reftable }, + { "dump-reftable", cmd__dump_reftable }, { "regex", cmd__regex }, { "repository", cmd__repository }, { "revision-walking", cmd__revision_walking },
t/helper/test-tool.h +1 -0
@@ -19,6 +19,7 @@ int cmd__dump_cache_tree(int argc, const char **argv);int cmd__dump_fsmonitor (int argc,const char **argv);int cmd__dump_split_index (int argc,const char **argv);int cmd__dump_untracked_cache (int argc,const char **argv); +int cmd__dump_reftable (int argc,const char **argv);int cmd__example_decorate (int argc,const char **argv);int cmd__fast_rebase (int argc,const char **argv);int cmd__genrandom (int argc,const char **argv);
Signature · 1 line
gitgitgadget
Quoted 9 lines · Han-Wen Nienhuys via GitGitGadget
The reftable format is described in
Documentation/technical/reftable.txt.This is a fully reentrant implementation of reading and writing the reftable file format, and should be suitable for embedding in libgit2 too. It does not hook the code up to git to function as a ref storage backend yet.
v2:
* fold in OpenBSD fixes by Carlo Belón.
Thanks, both. Will replace.
Quoted 5 lines · Han-Wen Nienhuys via GitGitGadget
The reftable format is described in
Documentation/technical/reftable.txt.This is a fully reentrant implementation of reading and writing the reftable file format, and should be suitable for embedding in libgit2 too. It does not hook the code up to git to function as a ref storage backend yet.
Not a question for Han-Wen, but I am wondering how much style and other consistency guidelines we have in our C code to the files in this directory.
I am guessing that rules like "no decl after statement" and "no decl in the set-up part of the for loop control" (i.e. "for (int i = 0; ..." is a no-no) should apply equally to this code, but it might be OK to deviate from rules that are only meant to help human readers [*] without affecting compilation.
Opinions?
[Footnote]
* multi-line comment style.
* asterisk sticks to identifier not to type
* not enclosing a single-statement block inside {}
* empty statement looks like
; /* empty */
Quoted 5 lines · Junio C Hamano
Quoted 2 lines · Han-Wen Nienhuys via GitGitGadget
The reftable format is described in
Documentation/technical/reftable.txt.
Quoted 7 lines
Quoted 3 lines
This is a fully reentrant implementation of reading and writing the reftable file format, and should be suitable for embedding in libgit2 too. It does not hook the code up to git to function as a ref storage backend yet.
Not a question for Han-Wen, but I am wondering how much style and other consistency guidelines we have in our C code to the files in this directory.
I am Han-Wen, but I'm not sure what you are saying here.
Quoted 7 lines
I am guessing that rules like "no decl after statement" and "no decl in the set-up part of the for loop control" (i.e. "for (int i = 0; ..." is a no-no) should apply equally to this code, but it might be OK to deviate from rules that are only meant to help human readers [*] without affecting compilation.
Opinions?
The code has a different style because I wrote it separately from Git. I'm not wedded to its current style, and most styling can easily be changed. If you have specific things that should be addressed, let me know.
Signature · 11 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Quoted 10 lines · Han-Wen Nienhuys
Quoted 9 lines · Junio C Hamano
Quoted 5 lines · Han-Wen Nienhuys via GitGitGadget
The reftable format is described in
Documentation/technical/reftable.txt.This is a fully reentrant implementation of reading and writing the reftable file format, and should be suitable for embedding in libgit2 too. It does not hook the code up to git to function as a ref storage backend yet.
Quoted 5 lines
Quoted 3 lines
Not a question for Han-Wen, but I am wondering how much style and other consistency guidelines we have in our C code to the files in this directory.
I am Han-Wen, but I'm not sure what you are saying here.
Sorry, the sentence is unreadable because I missed a verb above (insert "should apply" between "code" and "to").
I was asking for opinions on how we should treat this piece of code. We loosen "style guidelines" on borrowed code that is maintained elsewhere to ease re-importing, but code we maintain in-tree are subject to our style guide. I am not sure how this part that is meant to be reusable in other projects like libgit2 should be treated.
Quoted 6 lines
Quoted 6 lines
I am guessing that rules like "no decl after statement" and "no decl in the set-up part of the for loop control" (i.e. "for (int i = 0; ..." is a no-no) should apply equally to this code, but it might be OK to deviate from rules that are only meant to help human readers [*] without affecting compilation.
Quoted 6 lines
Quoted 1 line
Opinions?
The code has a different style because I wrote it separately from Git. I'm not wedded to its current style, and most styling can easily be changed. If you have specific things that should be addressed, let me know.
The question was for other reviewers to help us come up with what the "specific things" ought to be. I saw style differences around comments and code formatting (everything I listed in the footnote, plus, // comment which I forgot to mention) which may or may not turn out to be part of that "specific things", because they do not break compilation.
Quoted 6 lines · Junio C Hamano
Quoted 4 lines · Han-Wen Nienhuys
The code has a different style because I wrote it separately from Git. I'm not wedded to its current style, and most styling can easily be changed. If you have specific things that should be addressed, let me know.
Quoted 6 lines
The question was for other reviewers to help us come up with what the "specific things" ought to be. I saw style differences around comments and code formatting (everything I listed in the footnote, plus, // comment which I forgot to mention) which may or may not turn out to be part of that "specific things", because they do not break compilation.
they will be flagged by the compiler in pedantic mode when in gnu89 mode though
reftable/stack_test.c:49:1: warning: C++ style comments are not
allowed in ISO C
90
49 | // Work linenumber into the tempdir, so we can see which
tests forget to
| ^
reftable/stack_test.c:49:1: note: (this will be reported only once
per input file)
and are probably an easy thing to fix
Carlo
Quoted 21 lines · Carlo Arenas
Quoted 6 lines · Junio C Hamano
Quoted 4 lines · Han-Wen Nienhuys
The code has a different style because I wrote it separately from Git. I'm not wedded to its current style, and most styling can easily be changed. If you have specific things that should be addressed, let me know.
Quoted 6 lines
The question was for other reviewers to help us come up with what the "specific things" ought to be. I saw style differences around comments and code formatting (everything I listed in the footnote, plus, // comment which I forgot to mention) which may or may not turn out to be part of that "specific things", because they do not break compilation.
they will be flagged by the compiler in pedantic mode when in gnu89 mode though
reftable/stack_test.c:49:1: warning: C++ style comments are not
allowed in ISO C
90
49 | // Work linenumber into the tempdir, so we can see which
tests forget to
Quoted 5 lines
| ^
reftable/stack_test.c:49:1: note: (this will be reported only once
per input file)and are probably an easy thing to fix
I guess I still wasn't clear enough, then. Of course, there is no question that we do not tolerate compilation-breaking deviation from our coding guidelines. The "loosening" exception I was alluding to were those rules that do not break compilation but help human readers due to consistent writing. I do not think
/* this comment
* goes against our house style */
int* a, b;would be flagged by the gnu89 compiler, for example.
v3 cover Adds reftable library code from https://github.com/hanwen/reftable.
In reply to Han-Wen Nienhuys via GitGitGadgetThe reftable format is described in Documentation/technical/reftable.txt.
This is a fully reentrant implementation of reading and writing the reftable file format, and should be suitable for embedding in libgit2 too. It does not hook the code up to git to function as a ref storage backend yet.
v2:
* fold in OpenBSD fixes by Carlo Belón.
Han-Wen Nienhuys (19):
hash.h: provide constants for the hash IDs
reftable: RFC: add LICENSE
reftable: add error related functionality
reftable: utility functions
reftable: add blocksource, an abstraction for random access reads
reftable: (de)serialization for the polymorphic record type. Provide zlib's uncompress2 from compat/zlib-compat.c
reftable: reading/writing blocks
reftable: a generic binary tree implementation
reftable: write reftable files
reftable: generic interface to tables
reftable: read reftable files
reftable: reftable file level tests
reftable: add a heap-based priority queue for reftable records
reftable: add merged table view
reftable: implement refname validation
reftable: implement stack, a mutable database of reftable files.
reftable: add dump utility
Add "test-tool dump-reftable" command.
Makefile | 53 +- ci/lib.sh | 1 + compat/.gitattributes | 1 + compat/zlib-uncompress2.c | 95 ++ config.mak.uname | 4 + configure.ac | 13 + contrib/buildsystems/CMakeLists.txt | 14 +- contrib/buildsystems/Generators/Vcxproj.pm | 11 +- hash.h | 6 + object-file.c | 7 +- reftable/LICENSE | 31 + reftable/basics.c | 128 ++ reftable/basics.h | 60 + reftable/basics_test.c | 98 ++ reftable/block.c | 439 ++++++ reftable/block.h | 127 ++ reftable/block_test.c | 120 ++ reftable/blocksource.c | 148 +++ reftable/blocksource.h | 22 + reftable/constants.h | 21 + reftable/dump.c | 107 ++ reftable/error.c | 41 + reftable/generic.c | 169 +++ reftable/generic.h | 32 + reftable/iter.c | 194 +++ reftable/iter.h | 69 + reftable/merged.c | 362 +++++ reftable/merged.h | 35 + reftable/merged_test.c | 292 ++++ reftable/pq.c | 105 ++ reftable/pq.h | 33 + reftable/pq_test.c | 82 ++ reftable/publicbasics.c | 58 + reftable/reader.c | 801 +++++++++++ reftable/reader.h | 66 + reftable/readwrite_test.c | 652 +++++++++ reftable/record.c | 1212 +++++++++++++++++ reftable/record.h | 139 ++ reftable/record_test.c | 412 ++++++ reftable/refname.c | 209 +++ reftable/refname.h | 29 + reftable/refname_test.c | 102 ++ reftable/reftable-blocksource.h | 49 + reftable/reftable-error.h | 62 + reftable/reftable-generic.h | 47 + reftable/reftable-iterator.h | 39 + reftable/reftable-malloc.h | 18 + reftable/reftable-merged.h | 72 + reftable/reftable-reader.h | 101 ++ reftable/reftable-record.h | 114 ++ reftable/reftable-stack.h | 128 ++ reftable/reftable-tests.h | 23 + reftable/reftable-writer.h | 148 +++ reftable/reftable.c | 115 ++ reftable/stack.c | 1396 ++++++++++++++++++++ reftable/stack.h | 41 + reftable/stack_test.c | 949 +++++++++++++ reftable/system.h | 32 + reftable/test_framework.c | 23 + reftable/test_framework.h | 53 + reftable/tree.c | 63 + reftable/tree.h | 34 + reftable/tree_test.c | 61 + reftable/writer.c | 690 ++++++++++ reftable/writer.h | 50 + t/helper/test-reftable.c | 21 + t/helper/test-tool.c | 4 +- t/helper/test-tool.h | 2 + t/t0032-reftable-unittest.sh | 15 + 69 files changed, 10938 insertions(+), 12 deletions(-) create mode 100644 compat/.gitattributes create mode 100644 compat/zlib-uncompress2.c create mode 100644 reftable/LICENSE create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/basics_test.c create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/block_test.c create mode 100644 reftable/blocksource.c create mode 100644 reftable/blocksource.h create mode 100644 reftable/constants.h create mode 100644 reftable/dump.c create mode 100644 reftable/error.c create mode 100644 reftable/generic.c create mode 100644 reftable/generic.h create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/merged_test.c create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/pq_test.c create mode 100644 reftable/publicbasics.c create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/readwrite_test.c create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/record_test.c create mode 100644 reftable/refname.c create mode 100644 reftable/refname.h create mode 100644 reftable/refname_test.c create mode 100644 reftable/reftable-blocksource.h create mode 100644 reftable/reftable-error.h create mode 100644 reftable/reftable-generic.h create mode 100644 reftable/reftable-iterator.h create mode 100644 reftable/reftable-malloc.h create mode 100644 reftable/reftable-merged.h create mode 100644 reftable/reftable-reader.h create mode 100644 reftable/reftable-record.h create mode 100644 reftable/reftable-stack.h create mode 100644 reftable/reftable-tests.h create mode 100644 reftable/reftable-writer.h create mode 100644 reftable/reftable.c create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/stack_test.c create mode 100644 reftable/system.h create mode 100644 reftable/test_framework.c create mode 100644 reftable/test_framework.h create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100644 reftable/tree_test.c create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h create mode 100644 t/helper/test-reftable.c create mode 100755 t/t0032-reftable-unittest.sh
base-commit: ddb1055343948e0d0bc81f8d20245f1ada6430a0
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-git-1081%2Fhanwen%2Fjust-library-v3
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-git-1081/hanwen/just-library-v3
Pull-Request: https://github.com/git/git/pull/1081
Range-diff vs v2:
1: 8534c69cf84 = 1: b2f8dc32bfc hash.h: provide constants for the hash IDs
2: 52968d8e83f = 2: 2597ddfeae1 reftable: RFC: add LICENSE
3: 84c8b396df5 = 3: 41b1ff1293f reftable: add error related functionality
4: fa58f3d73e7 = 4: c03043c5e63 reftable: utility functions
5: e9206532126 = 5: 317a6aaa357 reftable: add blocksource, an abstraction for random access reads
6: 651e195fe77 = 6: 55e66b7a74f reftable: (de)serialization for the polymorphic record type.
7: e595dbc6c84 ! 7: 4f6ecd3646a Provide zlib's uncompress2 from compat/zlib-compat.c
@@ config.mak.uname: ifeq ($(uname_S),FreeBSD)
FILENO_IS_A_MACRO = UnfortunatelyYes
endif
ifeq ($(uname_S),OpenBSD)
-+ NO_UNCOMPRESS2 = YesPlease
+ # Versions < 7.0 need compatibility layer
+ ifeq ($(shell expr "$(uname_R)" : "[1-6]\."),2)
+ NO_UNCOMPRESS2 = UnfortunatelyYes
8: 97f7ee04886 = 8: 04f2d74df51 reftable: reading/writing blocks
9: 8d2e8be5795 = 9: f6c2da61208 reftable: a generic binary tree implementation
10: 965af6ec065 = 10: 277a1f662e3 reftable: write reftable files
11: 4e259380fda = 11: f97182dec1f reftable: generic interface to tables
12: a43d3512164 = 12: 13de5d03a71 reftable: read reftable files
13: f0e159cd853 = 13: 8efc3c2b6ab reftable: reftable file level tests
14: 5a5c108d601 = 14: e89b16a37e6 reftable: add a heap-based priority queue for reftable records
15: 905ffa58f36 = 15: 40a91b14be8 reftable: add merged table view
16: e9598889af2 = 16: 46c2ddf07cf reftable: implement refname validation
17: 8b0ee68c636 ! 17: 3db887bc078 reftable: implement stack, a mutable database of reftable files.
@@ reftable/stack.c (new)
+
+int read_lines(const char *filename, char ***namesp)
+{
-+ int fd = open(filename, O_RDONLY, 0644);
++ int fd = open(filename, O_RDONLY);
+ int err = 0;
+ if (fd < 0) {
+ if (errno == ENOENT) {
18: a9bebe45bde = 18: 795a15000fb reftable: add dump utility
19: 104fbc7502f = 19: 7b3215aef39 Add "test-tool dump-reftable" command.Signature · 1 line
gitgitgadget
PATCH v3 1/19 hash.h: provide constants for the hash IDs
From: Han-Wen Nienhuys <[email protected]>
This will simplify referencing them from code that is not deeply integrated with Git, in particular, the reftable library.
--- hash.h | 6 ++++++ object-file.c | 7 ++----- 2 files changed, 8 insertions(+), 5 deletions(-)
hash.h +6 -0
@@ -95,12 +95,18 @@ static inline void git_SHA256_Clone(git_SHA256_CTX *dst, const git_SHA256_CTX *s/* Number of algorithms supported (including unknown). */ #define GIT_HASH_NALGOS (GIT_HASH_SHA256 + 1) +/* "sha1", big-endian */ +#define GIT_SHA1_FORMAT_ID 0x73686131 +/* The length in bytes and in hex digits of an object name (SHA-1 value). */ #define GIT_SHA1_RAWSZ 20#define GIT_SHA1_HEXSZ (2 * GIT_SHA1_RAWSZ)/* The block size of SHA-1. */ #define GIT_SHA1_BLKSZ 64 +/* "s256", big-endian */ +#define GIT_SHA256_FORMAT_ID 0x73323536 +/* The length in bytes and in hex digits of an object name (SHA-256 value). */ #define GIT_SHA256_RAWSZ 32#define GIT_SHA256_HEXSZ (2 * GIT_SHA256_RAWSZ)
object-file.c +2 -5
@@ -165,7 +165,6 @@ static void git_hash_unknown_final_oid(struct object_id *oid, git_hash_ctx *ctx)BUG ("trying to finalize unknown hash" ); } -const struct git_hash_algo hash_algos[GIT_HASH_NALGOS ] = { {NULL ,
@@ -184,8 +183,7 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = { }, {"sha1" , -/* "sha1", big-endian */ -0x73686131 , +GIT_SHA1_FORMAT_ID ,GIT_SHA1_RAWSZ ,GIT_SHA1_HEXSZ , GIT_SHA1_BLKSZ,
@@ -200,8 +198,7 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = { }, {"sha256" , -/* "s256", big-endian */ -0x73323536 , +GIT_SHA256_FORMAT_ID ,GIT_SHA256_RAWSZ ,GIT_SHA256_HEXSZ , GIT_SHA256_BLKSZ,
Signature · 1 line
gitgitgadget
PATCH v3 2/19 reftable: RFC: add LICENSE
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The objective of this code is to be usable as a C library, so it can be reused in libgit2.
This is currently using a BSD license as it is the liberal license I could find, but this could be changed to whatever fits the stated goal above.
This code is currently imported from github.com/hanwen/reftable. Once this code lands in git.git, the C code will be removed from github.com/hanwen/reftable, and the git.git code will be the source of truth.
--- reftable/LICENSE | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 reftable/LICENSE
reftable/LICENSE new file +31 -0
@@ -0,0 +1,31 @@ +BSD License + +Copyright (c) 2020, Google LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of Google LLC nor the names of its contributors may +be used to endorse or promote products derived from this software +without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Signature · 1 line
gitgitgadget
PATCH v3 3/19 reftable: add error related functionality
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The reftable/ directory is structured as a library, so it cannot crash on misuse. Instead, it returns an error codes.
In addition, the error code can be used to signal conditions from lower levels of the library to be handled by higher levels of the library. For example, a transaction might legitimately write an empty reftable file, but in that case, we'd want to shortcut the transaction overhead.
--- reftable/error.c | 41 ++++++++++++++++++++++++++ reftable/reftable-error.h | 62 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 reftable/error.c create mode 100644 reftable/reftable-error.h
reftable/error.c new file +41 -0
@@ -0,0 +1,41 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "reftable-error.h" + +#include <stdio.h> + +const char *reftable_error_str (int err) +{ +static char buf[250 ]; +switch (err) { +case REFTABLE_IO_ERROR : +return "I/O error" ; +case REFTABLE_FORMAT_ERROR : +return "corrupt reftable file" ; +case REFTABLE_NOT_EXIST_ERROR : +return "file does not exist" ; +case REFTABLE_LOCK_ERROR : +return "data is outdated" ; +case REFTABLE_API_ERROR : +return "misuse of the reftable API" ; +case REFTABLE_ZLIB_ERROR : +return "zlib failure" ; +case REFTABLE_NAME_CONFLICT : +return "file/directory conflict" ; +case REFTABLE_EMPTY_TABLE_ERROR : +return "wrote empty table" ; +case REFTABLE_REFNAME_ERROR : +return "invalid refname" ; +case -1 : +return "general error" ; +default : +snprintf (buf,sizeof (buf),"unknown error code %d" , err); +return buf; + } +}
reftable/reftable-error.h new file +62 -0
62 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v3 4/19 reftable: utility functions
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This commit provides basic utility classes for the reftable library.
--- Makefile | 25 +++- contrib/buildsystems/CMakeLists.txt | 14 ++- contrib/buildsystems/Generators/Vcxproj.pm | 11 +- reftable/basics.c | 128 +++++++++++++++++++++ reftable/basics.h | 60 ++++++++++ reftable/basics_test.c | 98 ++++++++++++++++ reftable/publicbasics.c | 58 ++++++++++ reftable/reftable-malloc.h | 18 +++ reftable/reftable-tests.h | 22 ++++ reftable/system.h | 32 ++++++ reftable/test_framework.c | 23 ++++ reftable/test_framework.h | 53 +++++++++ t/helper/test-reftable.c | 9 ++ t/helper/test-tool.c | 3 +- t/helper/test-tool.h | 1 + t/t0032-reftable-unittest.sh | 15 +++ 16 files changed, 563 insertions(+), 7 deletions(-) create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/basics_test.c create mode 100644 reftable/publicbasics.c create mode 100644 reftable/reftable-malloc.h create mode 100644 reftable/reftable-tests.h create mode 100644 reftable/system.h create mode 100644 reftable/test_framework.c create mode 100644 reftable/test_framework.h create mode 100644 t/helper/test-reftable.c create mode 100755 t/t0032-reftable-unittest.sh
Makefile +22 -3
@@ -736,6 +736,7 @@ TEST_BUILTINS_OBJS += test-read-cache.o TEST_BUILTINS_OBJS += test-read-graph.o TEST_BUILTINS_OBJS += test-read-midx.o TEST_BUILTINS_OBJS += test-ref-store.o +TEST_BUILTINS_OBJS += test-reftable.o TEST_BUILTINS_OBJS += test-regex.o TEST_BUILTINS_OBJS += test-repository.o TEST_BUILTINS_OBJS += test-revision-walking.o
@@ -814,6 +815,8 @@ TEST_SHELL_PATH = $(SHELL_PATH) LIB_FILE = libgit.a XDIFF_LIB = xdiff/lib.a +REFTABLE_LIB = reftable/libreftable.a +REFTABLE_TEST_LIB = reftable/libreftable_test.a GENERATED_H += command-list.h GENERATED_H += config-list.h
@@ -1188,7 +1191,7 @@ THIRD_PARTY_SOURCES += compat/regex/% THIRD_PARTY_SOURCES += sha1collisiondetection/% THIRD_PARTY_SOURCES += sha1dc/% -GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) +GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) EXTLIBS = GIT_USER_AGENT = git/$(GIT_VERSION)
@@ -2424,7 +2427,15 @@ XDIFF_OBJS += xdiff/xutils.o .PHONY: xdiff-objs xdiff-objs: $(XDIFF_OBJS) +REFTABLE_OBJS += reftable/basics.o +REFTABLE_OBJS += reftable/error.o +REFTABLE_OBJS += reftable/publicbasics.o + +REFTABLE_TEST_OBJS += reftable/test_framework.o +REFTABLE_TEST_OBJS += reftable/basics_test.o + TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) + .PHONY: test-objs test-objs: $(TEST_OBJS)
@@ -2440,6 +2451,8 @@ OBJECTS += $(PROGRAM_OBJS) OBJECTS += $(TEST_OBJS) OBJECTS += $(XDIFF_OBJS) OBJECTS += $(FUZZ_OBJS) +OBJECTS += $(REFTABLE_OBJS) $(REFTABLE_TEST_OBJS) + ifndef NO_CURL OBJECTS += http.o http-walker.o remote-curl.o endif
@@ -2590,6 +2603,12 @@ $(LIB_FILE): $(LIB_OBJS) $(XDIFF_LIB): $(XDIFF_OBJS) $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ +$(REFTABLE_LIB): $(REFTABLE_OBJS) + $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ + +$(REFTABLE_TEST_LIB): $(REFTABLE_TEST_OBJS) + $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ + export DEFAULT_EDITOR DEFAULT_PAGER Documentation/GIT-EXCLUDED-PROGRAMS: FORCE
@@ -2888,7 +2907,7 @@ perf: all t/helper/test-tool$X: $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) -t/helper/test-%$X: t/helper/test-%.o GIT-LDFLAGS $(GITLIBS) +t/helper/test-%$X: t/helper/test-%.o GIT-LDFLAGS $(GITLIBS) $(REFTABLE_TEST_LIB) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(filter %.a,$^) $(LIBS) check-sha1:: t/helper/test-tool$X
@@ -3223,7 +3242,7 @@ cocciclean: clean: profile-clean coverage-clean cocciclean $(RM) *.res $(RM) $(OBJECTS) - $(RM) $(LIB_FILE) $(XDIFF_LIB) + $(RM) $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) $(REFTABLE_TEST_LIB) $(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) git$X $(RM) $(TEST_PROGRAMS) $(RM) $(FUZZ_PROGRAMS)
contrib/buildsystems/CMakeLists.txt +12 -2
@@ -640,6 +640,12 @@ parse_makefile_for_sources(libxdiff_SOURCES "XDIFF_OBJS") list(TRANSFORM libxdiff_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") add_library(xdiff STATIC ${libxdiff_SOURCES}) +#reftable +parse_makefile_for_sources(reftable_SOURCES "REFTABLE_OBJS") + +list(TRANSFORM reftable_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") +add_library(reftable STATIC ${reftable_SOURCES}) + if(WIN32) if(NOT MSVC)#use windres when compiling with gcc and clang add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/git.res
@@ -662,7 +668,7 @@ endif() #link all required libraries to common-main add_library(common-main OBJECT ${CMAKE_SOURCE_DIR}/common-main.c) -target_link_libraries(common-main libgit xdiff ${ZLIB_LIBRARIES}) +target_link_libraries(common-main libgit xdiff reftable ${ZLIB_LIBRARIES}) if(Intl_FOUND) target_link_libraries(common-main ${Intl_LIBRARIES}) endif()
@@ -902,11 +908,15 @@ if(BUILD_TESTING) add_executable(test-fake-ssh ${CMAKE_SOURCE_DIR}/t/helper/test-fake-ssh.c) target_link_libraries(test-fake-ssh common-main) +#reftable-tests +parse_makefile_for_sources(test-reftable_SOURCES "REFTABLE_TEST_OBJS") +list(TRANSFORM test-reftable_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") + #test-tool parse_makefile_for_sources(test-tool_SOURCES "TEST_BUILTINS_OBJS") list(TRANSFORM test-tool_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/t/helper/") -add_executable(test-tool ${CMAKE_SOURCE_DIR}/t/helper/test-tool.c ${test-tool_SOURCES}) +add_executable(test-tool ${CMAKE_SOURCE_DIR}/t/helper/test-tool.c ${test-tool_SOURCES} ${test-reftable_SOURCES}) target_link_libraries(test-tool common-main) set_target_properties(test-fake-ssh test-tool
contrib/buildsystems/Generators/Vcxproj.pm +10 -1
@@ -77,7 +77,7 @@ sub createProject { my $libs_release = "\n "; my $libs_debug = "\n "; if (!$static_library) { - $libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); + $libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib|reftable\/libreftable\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); $libs_debug = $libs_release; $libs_debug =~ s/zlib\.lib/zlibd\.lib/g; $libs_debug =~ s/libexpat\.lib/libexpatd\.lib/g;
@@ -232,6 +232,7 @@ EOM EOM if (!$static_library || $target =~ 'vcs-svn' || $target =~ 'xdiff') { my $uuid_libgit = $$build_structure{"LIBS_libgit_GUID"}; + my $uuid_libreftable = $$build_structure{"LIBS_reftable/libreftable_GUID"}; my $uuid_xdiff_lib = $$build_structure{"LIBS_xdiff/lib_GUID"}; print F << "EOM";
@@ -241,6 +242,14 @@ EOM <ReferenceOutputAssembly>false</ReferenceOutputAssembly> </ProjectReference> EOM + if (!($name =~ /xdiff|libreftable/)) { + print F << "EOM"; + <ProjectReference Include="$cdup\\reftable\\libreftable\\libreftable.vcxproj"> + <Project>$uuid_libreftable</Project> + <ReferenceOutputAssembly>false</ReferenceOutputAssembly> + </ProjectReference> +EOM + } if (!($name =~ 'xdiff')) { print F << "EOM"; <ProjectReference Include="$cdup\\xdiff\\lib\\xdiff_lib.vcxproj">
reftable/basics.c new file +128 -0
128 lines not on this page · raw message
reftable/basics.h new file +60 -0
60 lines not on this page · raw message
reftable/basics_test.c new file +98 -0
98 lines not on this page · raw message
reftable/publicbasics.c new file +58 -0
58 lines not on this page · raw message
reftable/reftable-malloc.h new file +18 -0
@@ -0,0 +1,18 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_H +#define REFTABLE_H + +#include <stddef.h> + +/* Overrides the functions to use for memory management. */ +void reftable_set_alloc (void *(*malloc)(size_t ), +void *(*realloc)(void *,size_t ),void (*free)(void *)); + +#endif
reftable/reftable-tests.h new file +22 -0
@@ -0,0 +1,22 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_TESTS_H +#define REFTABLE_TESTS_H + +int basics_test_main (int argc,const char **argv); +int block_test_main (int argc,const char **argv); +int merged_test_main (int argc,const char **argv); +int record_test_main (int argc,const char **argv); +int refname_test_main (int argc,const char **argv); +int reftable_test_main (int argc,const char **argv); +int stack_test_main (int argc,const char **argv); +int tree_test_main (int argc,const char **argv); +int reftable_dump_main (int argc,char *const *argv); + +#endif
reftable/system.h new file +32 -0
@@ -0,0 +1,32 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef SYSTEM_H +#define SYSTEM_H + +/* This header glues the reftable library to the rest of Git */ + +#include "git-compat-util.h" +#include "strbuf.h" +#include "hash.h" /* hash ID, sizes.*/ +#include "dir.h" /* remove_dir_recursively, for tests.*/ + +#include <zlib.h> + +#ifdef NO_UNCOMPRESS2 +/* +* This is uncompress2, which is only available in zlib >= 1.2.9 +* (released as of early 2017) +*/ +int uncompress2 (Bytef *dest,uLongf *destLen,const Bytef *source, +uLong *sourceLen); +#endif + +int hash_size (uint32_t id); + +#endif
reftable/test_framework.c new file +23 -0
@@ -0,0 +1,23 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "system.h" +#include "test_framework.h" + +#include "basics.h" + +void set_test_hash (uint8_t *p,int i) +{ +memset (p, (uint8_t )i,hash_size (GIT_SHA1_FORMAT_ID )); +} + +ssize_t strbuf_add_void (void *b,const void *data,size_t sz) +{ +strbuf_add (b, data, sz); +return sz; +}
reftable/test_framework.h new file +53 -0
53 lines not on this page · raw message
t/helper/test-reftable.c new file +9 -0
@@ -0,0 +1,9 @@ +#include "reftable/reftable-tests.h" +#include "test-tool.h" + +int cmd__reftable (int argc,const char **argv) +{ +basics_test_main (argc, argv); + +return 0 ; +}
t/helper/test-tool.c +2 -1
@@ -53,13 +53,14 @@ static struct test_cmd cmds[] = { { "pcre2-config", cmd__pcre2_config }, { "pkt-line", cmd__pkt_line }, { "prio-queue", cmd__prio_queue }, - { "proc-receive", cmd__proc_receive}, + { "proc-receive", cmd__proc_receive }, { "progress", cmd__progress }, { "reach", cmd__reach }, { "read-cache", cmd__read_cache }, { "read-graph", cmd__read_graph }, { "read-midx", cmd__read_midx }, { "ref-store", cmd__ref_store }, + { "reftable", cmd__reftable }, { "regex", cmd__regex }, { "repository", cmd__repository }, { "revision-walking", cmd__revision_walking },
t/helper/test-tool.h +1 -0
@@ -49,6 +49,7 @@ int cmd__read_cache(int argc, const char **argv);int cmd__read_graph (int argc,const char **argv);int cmd__read_midx (int argc,const char **argv);int cmd__ref_store (int argc,const char **argv); +int cmd__reftable (int argc,const char **argv);int cmd__regex (int argc,const char **argv);int cmd__repository (int argc,const char **argv);int cmd__revision_walking (int argc,const char **argv);
t/t0032-reftable-unittest.sh new file +15 -0
@@ -0,0 +1,15 @@ +#!/bin/sh +# +# Copyright (c) 2020 Google LLC +# + +test_description='reftable unittests' + +. ./test-lib.sh + +test_expect_success 'unittests' ' + TMPDIR=$(pwd) && export TMPDIR && + test-tool reftable +' + +test_done
Signature · 1 line
gitgitgadget
PATCH v3 5/19 reftable: add blocksource, an abstraction for random access reads
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The reftable format is usually used with files for storage. However, we abstract away this using the blocksource data structure. This has two advantages:
* log blocks are zlib compressed, and handling them is simplified if we can discard byte segments from within the block layer.
* for unittests, it is useful to read and write in-memory. The blocksource allows us to abstract the data away from on-disk files.
--- Makefile | 1 + reftable/blocksource.c | 148 ++++++++++++++++++++++++++++++++ reftable/blocksource.h | 22 +++++ reftable/reftable-blocksource.h | 49 +++++++++++ 4 files changed, 220 insertions(+) create mode 100644 reftable/blocksource.c create mode 100644 reftable/blocksource.h create mode 100644 reftable/reftable-blocksource.h
Makefile +1 -0
@@ -2429,6 +2429,7 @@ xdiff-objs: $(XDIFF_OBJS) REFTABLE_OBJS += reftable/basics.o REFTABLE_OBJS += reftable/error.o +REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o REFTABLE_TEST_OBJS += reftable/test_framework.o
reftable/blocksource.c new file +148 -0
148 lines not on this page · raw message
reftable/blocksource.h new file +22 -0
@@ -0,0 +1,22 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef BLOCKSOURCE_H +#define BLOCKSOURCE_H + +#include "system.h" + +struct reftable_block_source ; + +/* Create an in-memory block source for reading reftables */ +void block_source_from_strbuf (struct reftable_block_source *bs, +struct strbuf *buf); + +struct reftable_block_source malloc_block_source (void ); + +#endif
reftable/reftable-blocksource.h new file +49 -0
49 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v3 6/19 reftable: (de)serialization for the polymorphic record type.
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The reftable format is structured as a sequence of blocks, and each block contains a sequence of prefix-compressed key-value records. There are 4 types of records, and they have similarities in how they must be handled. This is achieved by introducing a polymorphic 'record' type that encapsulates ref, log, index and object records.
--- Makefile | 2 + reftable/constants.h | 21 + reftable/record.c | 1212 ++++++++++++++++++++++++++++++++++++ reftable/record.h | 139 +++++ reftable/record_test.c | 412 ++++++++++++ reftable/reftable-record.h | 114 ++++ t/helper/test-reftable.c | 2 +- 7 files changed, 1901 insertions(+), 1 deletion(-) create mode 100644 reftable/constants.h create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/record_test.c create mode 100644 reftable/reftable-record.h
Makefile +2 -0
@@ -2431,7 +2431,9 @@ REFTABLE_OBJS += reftable/basics.o REFTABLE_OBJS += reftable/error.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o +REFTABLE_OBJS += reftable/record.o +REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o REFTABLE_TEST_OBJS += reftable/basics_test.o
reftable/constants.h new file +21 -0
@@ -0,0 +1,21 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef CONSTANTS_H +#define CONSTANTS_H + +#define BLOCK_TYPE_LOG 'g' +#define BLOCK_TYPE_INDEX 'i' +#define BLOCK_TYPE_REF 'r' +#define BLOCK_TYPE_OBJ 'o' +#define BLOCK_TYPE_ANY 0 + +#define MAX_RESTARTS ((1 << 16) - 1) +#define DEFAULT_BLOCK_SIZE 4096 + +#endif
reftable/record.c new file +1212 -0
1,212 lines not on this page · raw message
reftable/record.h new file +139 -0
139 lines not on this page · raw message
reftable/record_test.c new file +412 -0
412 lines not on this page · raw message
reftable/reftable-record.h new file +114 -0
114 lines not on this page · raw message
t/helper/test-reftable.c +1 -1
@@ -4,6 +4,6 @@int cmd__reftable (int argc,const char **argv) {basics_test_main (argc, argv); - +record_test_main (argc, argv);return 0 ; }
Signature · 1 line
gitgitgadget
PATCH v3 7/19 Provide zlib's uncompress2 from compat/zlib-compat.c
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This will be needed for reading reflog blocks in reftable.
--- Makefile | 7 +++ ci/lib.sh | 1 + compat/.gitattributes | 1 + compat/zlib-uncompress2.c | 95 +++++++++++++++++++++++++++++++++++++++ config.mak.uname | 4 ++ configure.ac | 13 ++++++ 6 files changed, 121 insertions(+) create mode 100644 compat/.gitattributes create mode 100644 compat/zlib-uncompress2.c
Makefile +7 -0
@@ -256,6 +256,8 @@ all:: # # Define NO_DEFLATE_BOUND if your zlib does not have deflateBound. # +# Define NO_UNCOMPRESS2 if your zlib does not have uncompress2. +# # Define NO_NORETURN if using buggy versions of gcc 4.6+ and profile feedback, # as the compiler can crash (http://gcc.gnu.org/bugzilla/show_bug.cgi?id=49299) #
@@ -1716,6 +1718,11 @@ ifdef NO_DEFLATE_BOUND BASIC_CFLAGS += -DNO_DEFLATE_BOUND endif +ifdef NO_UNCOMPRESS2 + BASIC_CFLAGS += -DNO_UNCOMPRESS2 + REFTABLE_OBJS += compat/zlib-uncompress2.o +endif + ifdef NO_POSIX_GOODIES BASIC_CFLAGS += -DNO_POSIX_GOODIES endif
ci/lib.sh +1 -0
@@ -224,6 +224,7 @@ linux-gcc-default) ;; Linux32) CC=gcc + MAKEFLAGS="$MAKEFLAGS NO_UNCOMPRESS2=1" ;; linux-musl) CC=gcc
compat/.gitattributes new file +1 -0
@@ -0,0 +1 @@ +/zlib-uncompress2.c whitespace=-indent-with-non-tab,-trailing-space
compat/zlib-uncompress2.c new file +95 -0
95 lines not on this page · raw message
config.mak.uname +4 -0
@@ -258,6 +258,10 @@ ifeq ($(uname_S),FreeBSD) FILENO_IS_A_MACRO = UnfortunatelyYes endif ifeq ($(uname_S),OpenBSD) + # Versions < 7.0 need compatibility layer + ifeq ($(shell expr "$(uname_R)" : "[1-6]\."),2) + NO_UNCOMPRESS2 = UnfortunatelyYes + endif NO_STRCASESTR = YesPlease NO_MEMMEM = YesPlease USE_ST_TIMESPEC = YesPlease
configure.ac +13 -0
@@ -672,9 +672,22 @@ AC_LINK_IFELSE([ZLIBTEST_SRC], NO_DEFLATE_BOUND=yes]) LIBS="$old_LIBS" +AC_DEFUN([ZLIBTEST_UNCOMPRESS2_SRC], [ +AC_LANG_PROGRAM([#include <zlib.h>], + [uncompress2(NULL,NULL,NULL,NULL);])]) +AC_MSG_CHECKING([for uncompress2 in -lz]) +old_LIBS="$LIBS" +LIBS="$LIBS -lz" +AC_LINK_IFELSE([ZLIBTEST_UNCOMPRESS2_SRC], + [AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no]) + NO_UNCOMPRESS2=yes]) +LIBS="$old_LIBS" + GIT_UNSTASH_FLAGS($ZLIB_PATH) GIT_CONF_SUBST([NO_DEFLATE_BOUND]) +GIT_CONF_SUBST([NO_UNCOMPRESS2]) # # Define NEEDS_SOCKET if linking with libc is not enough (SunOS,
Signature · 1 line
gitgitgadget
PATCH v3 8/19 reftable: reading/writing blocks
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The reftable format is structured as a sequence of block. Within a block, records are prefix compressed, with an index of offsets for fully expand keys to enable binary search within blocks.
This commit provides the logic to read and write these blocks.
--- Makefile | 2 + reftable/block.c | 439 +++++++++++++++++++++++++++++++++++++++ reftable/block.h | 127 +++++++++++ reftable/block_test.c | 120 +++++++++++ t/helper/test-reftable.c | 1 + 5 files changed, 689 insertions(+) create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/block_test.c
Makefile +2 -0
@@ -2436,10 +2436,12 @@ xdiff-objs: $(XDIFF_OBJS) REFTABLE_OBJS += reftable/basics.o REFTABLE_OBJS += reftable/error.o +REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o REFTABLE_OBJS += reftable/record.o +REFTABLE_TEST_OBJS += reftable/block_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o REFTABLE_TEST_OBJS += reftable/basics_test.o
reftable/block.c new file +439 -0
439 lines not on this page · raw message
reftable/block.h new file +127 -0
127 lines not on this page · raw message
reftable/block_test.c new file +120 -0
120 lines not on this page · raw message
t/helper/test-reftable.c +1 -0
@@ -4,6 +4,7 @@int cmd__reftable (int argc,const char **argv) {basics_test_main (argc, argv); +block_test_main (argc, argv);record_test_main (argc, argv);return 0 ; }
Signature · 1 line
gitgitgadget
PATCH 1/1 squash! reftable: reading/writing blocks
Avoid a possible infinite loop in platforms where size_t > long by using instead the required type and checking for overflow before doubling.
Increase slightly the initial buffer based on the zlib(1) recommendation to reduce the likelihood of this code triggering, and add some error checking to the memory allocation.
[1] https://refspecs.linuxbase.org/LSB_3.0.0/LSB-Core-generic/LSB-Core-generic/zlib-compress2-1.html
--- reftable/block.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-)
reftable/block.c +7 -6
@@ -132,7 +132,7 @@ int block_writer_add(struct block_writer *w, struct reftable_record *rec)int block_writer_finish (struct block_writer *w) { -int i =0 ; +int i;for (i =0 ; i < w->restart_len; i++) {put_be24 (w->buf + w->next, w->restarts[i]); w->next +=3 ;
@@ -144,23 +144,24 @@ int block_writer_finish(struct block_writer *w)if (block_writer_type (w) ==BLOCK_TYPE_LOG ) {int block_header_skip =4 + w->header_off; -uint8_t *compressed =NULL ; -int zresult =0 ; +uint8_t *compressed;uLongf src_len = w->next - block_header_skip; -size_t dest_cap = src_len; +uLongf dest_cap = src_len *1.001 +12 ; compressed =reftable_malloc (dest_cap);while (1 ) { +int zresult;uLongf out_dest_len = dest_cap; zresult =compress2 (compressed, &out_dest_len, w->buf + block_header_skip, src_len,9 ); -if (zresult ==Z_BUF_ERROR ) { +if (zresult ==Z_BUF_ERROR && dest_cap <LONG_MAX ) { dest_cap *=2 ; compressed =reftable_realloc (compressed, dest_cap); -continue ; +if (compressed) +continue ; }if (Z_OK != zresult) {
Signature · 1 line
2.33.0.955.gee03ddbf0e
Re: [PATCH] squash! reftable: reading/writing blocks
Quoted 3 lines · Carlo Marcelo Arenas Belón
Avoid a possible infinite loop in platforms where
size_t> long by using instead the required type and checking for overflow before doubling.
Thanks for your attention to detail. Folded in.
I think the loop is rather hypothetical, though: the block size is limited to 2²⁴, so I think you can´t make blocks that large.
Signature · 7 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
PATCH v3 9/19 reftable: a generic binary tree implementation
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The reftable format includes support for an (OID => ref) map. This map can speed up visibility and reachability checks. In particular, various operations along the fetch/push path within Gerrit have ben sped up by using this structure.
The map is constructed with help of a binary tree. Object IDs are hashes, so they are uniformly distributed. Hence, the tree does not attempt forced rebalancing.
--- Makefile | 4 ++- reftable/tree.c | 63 ++++++++++++++++++++++++++++++++++++++++ reftable/tree.h | 34 ++++++++++++++++++++++ reftable/tree_test.c | 61 ++++++++++++++++++++++++++++++++++++++ t/helper/test-reftable.c | 1 + 5 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100644 reftable/tree_test.c
Makefile +3 -1
@@ -2440,11 +2440,13 @@ REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o REFTABLE_OBJS += reftable/record.o +REFTABLE_OBJS += reftable/tree.o +REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o -REFTABLE_TEST_OBJS += reftable/basics_test.o +REFTABLE_TEST_OBJS += reftable/tree_test.o TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS))
reftable/tree.c new file +63 -0
63 lines not on this page · raw message
reftable/tree.h new file +34 -0
@@ -0,0 +1,34 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef TREE_H +#define TREE_H + +/* tree_node is a generic binary search tree. */ +struct tree_node { +void *key; +struct tree_node *left, *right; +}; + +/* looks for `key` in `rootp` using `compare` as comparison function. If insert +* is set, insert the key if it's not found. Else, return NULL. +*/ +struct tree_node *tree_search (void *key,struct tree_node **rootp, +int (*compare)(const void *,const void *), +int insert); + +/* performs an infix walk of the tree. */ +void infix_walk (struct tree_node *t,void (*action)(void *arg,void *key), +void *arg); + +/* +* deallocates the tree nodes recursively. Keys should be deallocated separately +* by walking over the tree. */ +void tree_free (struct tree_node *t); + +#endif
reftable/tree_test.c new file +61 -0
61 lines not on this page · raw message
t/helper/test-reftable.c +1 -0
@@ -6,5 +6,6 @@ int cmd__reftable(int argc, const char **argv)basics_test_main (argc, argv);block_test_main (argc, argv);record_test_main (argc, argv); +tree_test_main (argc, argv);return 0 ; }
Signature · 1 line
gitgitgadget
PATCH v3 10/19 reftable: write reftable files
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
--- Makefile | 1 + reftable/reftable-writer.h | 148 ++++++++ reftable/writer.c | 690 +++++++++++++++++++++++++++++++++++++ reftable/writer.h | 50 +++ 4 files changed, 889 insertions(+) create mode 100644 reftable/reftable-writer.h create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h
Makefile +1 -0
@@ -2441,6 +2441,7 @@ REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o REFTABLE_OBJS += reftable/record.o REFTABLE_OBJS += reftable/tree.o +REFTABLE_OBJS += reftable/writer.o REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o
reftable/reftable-writer.h new file +148 -0
148 lines not on this page · raw message
reftable/writer.c new file +690 -0
690 lines not on this page · raw message
reftable/writer.h new file +50 -0
50 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v3 11/19 reftable: generic interface to tables
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
--- Makefile | 3 + reftable/generic.c | 169 +++++++++++++++++++++++++++++++++++ reftable/generic.h | 32 +++++++ reftable/reftable-generic.h | 47 ++++++++++ reftable/reftable-iterator.h | 39 ++++++++ reftable/reftable.c | 115 ++++++++++++++++++++++++ 6 files changed, 405 insertions(+) create mode 100644 reftable/generic.c create mode 100644 reftable/generic.h create mode 100644 reftable/reftable-generic.h create mode 100644 reftable/reftable-iterator.h create mode 100644 reftable/reftable.c
Makefile +3 -0
@@ -2440,6 +2440,9 @@ REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o REFTABLE_OBJS += reftable/record.o +REFTABLE_OBJS += reftable/refname.o +REFTABLE_OBJS += reftable/generic.o +REFTABLE_OBJS += reftable/stack.o REFTABLE_OBJS += reftable/tree.o REFTABLE_OBJS += reftable/writer.o
reftable/generic.c new file +169 -0
169 lines not on this page · raw message
reftable/generic.h new file +32 -0
@@ -0,0 +1,32 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef GENERIC_H +#define GENERIC_H + +#include "record.h" +#include "reftable-generic.h" + +/* generic interface to reftables */ +struct reftable_table_vtable { +int (*seek_record)(void *tab,struct reftable_iterator *it, +struct reftable_record *); +uint32_t (*hash_id)(void *tab); +uint64_t (*min_update_index)(void *tab); +uint64_t (*max_update_index)(void *tab); +}; + +struct reftable_iterator_vtable { +int (*next)(void *iter_arg,struct reftable_record *rec); +void (*close)(void *iter_arg); +}; + +void iterator_set_empty (struct reftable_iterator *it); +int iterator_next (struct reftable_iterator *it,struct reftable_record *rec); + +#endif
reftable/reftable-generic.h new file +47 -0
47 lines not on this page · raw message
reftable/reftable-iterator.h new file +39 -0
@@ -0,0 +1,39 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_ITERATOR_H +#define REFTABLE_ITERATOR_H + +#include "reftable-record.h" + +struct reftable_iterator_vtable ; + +/* iterator is the generic interface for walking over data stored in a +* reftable. +*/ +struct reftable_iterator { +struct reftable_iterator_vtable *ops; +void *iter_arg; +}; + +/* reads the next reftable_ref_record. Returns < 0 for error, 0 for OK and > 0: +* end of iteration. +*/ +int reftable_iterator_next_ref (struct reftable_iterator *it, +struct reftable_ref_record *ref); + +/* reads the next reftable_log_record. Returns < 0 for error, 0 for OK and > 0: +* end of iteration. +*/ +int reftable_iterator_next_log (struct reftable_iterator *it, +struct reftable_log_record *log); + +/* releases resources associated with an iterator. */ +void reftable_iterator_destroy (struct reftable_iterator *it); + +#endif
reftable/reftable.c new file +115 -0
115 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v3 12/19 reftable: read reftable files
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This supports reading a single reftable file.
The commit introduces an abstract iterator type, which captures the usecases both of reading individual refs, and iterating over a segment of the ref namespace.
--- Makefile | 2 + reftable/iter.c | 194 +++++++++ reftable/iter.h | 69 ++++ reftable/reader.c | 801 +++++++++++++++++++++++++++++++++++++ reftable/reader.h | 66 +++ reftable/reftable-reader.h | 101 +++++ 6 files changed, 1233 insertions(+) create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/reftable-reader.h
Makefile +2 -0
@@ -2438,7 +2438,9 @@ REFTABLE_OBJS += reftable/basics.o REFTABLE_OBJS += reftable/error.o REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o +REFTABLE_OBJS += reftable/iter.o REFTABLE_OBJS += reftable/publicbasics.o +REFTABLE_OBJS += reftable/reader.o REFTABLE_OBJS += reftable/record.o REFTABLE_OBJS += reftable/refname.o REFTABLE_OBJS += reftable/generic.o
reftable/iter.c new file +194 -0
194 lines not on this page · raw message
reftable/iter.h new file +69 -0
69 lines not on this page · raw message
reftable/reader.c new file +801 -0
801 lines not on this page · raw message
reftable/reader.h new file +66 -0
66 lines not on this page · raw message
reftable/reftable-reader.h new file +101 -0
101 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v3 13/19 reftable: reftable file level tests
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
With support for reading and writing files in place, we can construct files (in memory) and attempt to read them back.
Because some sections of the format are optional (eg. indices, log entries), we have to exercise this code using multiple sizes of input data
--- Makefile | 1 + reftable/readwrite_test.c | 652 ++++++++++++++++++++++++++++++++++++++ reftable/reftable-tests.h | 2 +- t/helper/test-reftable.c | 1 + 4 files changed, 655 insertions(+), 1 deletion(-) create mode 100644 reftable/readwrite_test.c
Makefile +1 -0
@@ -2451,6 +2451,7 @@ REFTABLE_OBJS += reftable/writer.o REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o REFTABLE_TEST_OBJS += reftable/record_test.o +REFTABLE_TEST_OBJS += reftable/readwrite_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o REFTABLE_TEST_OBJS += reftable/tree_test.o
reftable/readwrite_test.c new file +652 -0
652 lines not on this page · raw message
reftable/reftable-tests.h +1 -1
@@ -14,7 +14,7 @@ int block_test_main(int argc, const char **argv);int merged_test_main (int argc,const char **argv);int record_test_main (int argc,const char **argv);int refname_test_main (int argc,const char **argv); -int reftable_test_main (int argc,const char **argv); +int readwrite_test_main (int argc,const char **argv);int stack_test_main (int argc,const char **argv);int tree_test_main (int argc,const char **argv);int reftable_dump_main (int argc,char *const *argv);
t/helper/test-reftable.c +1 -0
@@ -6,6 +6,7 @@ int cmd__reftable(int argc, const char **argv)basics_test_main (argc, argv);block_test_main (argc, argv);record_test_main (argc, argv); +readwrite_test_main (argc, argv);tree_test_main (argc, argv);return 0 ; }
Signature · 1 line
gitgitgadget
PATCH v3 14/19 reftable: add a heap-based priority queue for reftable records
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This is needed to create a merged view multiple reftables
--- Makefile | 2 + reftable/pq.c | 105 ++++++++++++++++++++++++++++++++++++++ reftable/pq.h | 33 ++++++++++++ reftable/pq_test.c | 82 +++++++++++++++++++++++++++++ reftable/reftable-tests.h | 1 + t/helper/test-reftable.c | 1 + 6 files changed, 224 insertions(+) create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/pq_test.c
Makefile +2 -0
@@ -2440,6 +2440,7 @@ REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/iter.o REFTABLE_OBJS += reftable/publicbasics.o +REFTABLE_OBJS += reftable/pq.o REFTABLE_OBJS += reftable/reader.o REFTABLE_OBJS += reftable/record.o REFTABLE_OBJS += reftable/refname.o
@@ -2450,6 +2451,7 @@ REFTABLE_OBJS += reftable/writer.o REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o +REFTABLE_TEST_OBJS += reftable/pq_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/readwrite_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o
reftable/pq.c new file +105 -0
105 lines not on this page · raw message
reftable/pq.h new file +33 -0
@@ -0,0 +1,33 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef PQ_H +#define PQ_H + +#include "record.h" + +struct pq_entry { +int index; +struct reftable_record rec; +}; + +struct merged_iter_pqueue { +struct pq_entry *heap; +size_t len; +size_t cap; +}; + +struct pq_entry merged_iter_pqueue_top (struct merged_iter_pqueue pq); +int merged_iter_pqueue_is_empty (struct merged_iter_pqueue pq); +void merged_iter_pqueue_check (struct merged_iter_pqueue pq); +struct pq_entry merged_iter_pqueue_remove (struct merged_iter_pqueue *pq); +void merged_iter_pqueue_add (struct merged_iter_pqueue *pq,struct pq_entry e); +void merged_iter_pqueue_release (struct merged_iter_pqueue *pq); +int pq_less (struct pq_entry *a,struct pq_entry *b); + +#endif
reftable/pq_test.c new file +82 -0
82 lines not on this page · raw message
reftable/reftable-tests.h +1 -0
@@ -12,6 +12,7 @@ https://developers.google.com/open-source/licenses/bsdint basics_test_main (int argc,const char **argv);int block_test_main (int argc,const char **argv);int merged_test_main (int argc,const char **argv); +int pq_test_main (int argc,const char **argv);int record_test_main (int argc,const char **argv);int refname_test_main (int argc,const char **argv);int readwrite_test_main (int argc,const char **argv);
t/helper/test-reftable.c +1 -0
@@ -5,6 +5,7 @@ int cmd__reftable(int argc, const char **argv) {basics_test_main (argc, argv);block_test_main (argc, argv); +pq_test_main (argc, argv);record_test_main (argc, argv);readwrite_test_main (argc, argv);tree_test_main (argc, argv);
Signature · 1 line
gitgitgadget
PATCH v3 15/19 reftable: add merged table view
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This adds an abstract, read-only interface to the ref database.
This primitive is used to construct the read view of the ref database (the read view is constructed by merging several *.ref files). It also provides the mechanism to provide a unified view of the refs in the main repository and the per-worktree refs.
--- Makefile | 2 + reftable/merged.c | 362 +++++++++++++++++++++++++++++++++++++ reftable/merged.h | 35 ++++ reftable/merged_test.c | 292 ++++++++++++++++++++++++++++++ reftable/reftable-merged.h | 72 ++++++++ t/helper/test-reftable.c | 1 + 6 files changed, 764 insertions(+) create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/merged_test.c create mode 100644 reftable/reftable-merged.h
Makefile +2 -0
@@ -2440,6 +2440,7 @@ REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/iter.o REFTABLE_OBJS += reftable/publicbasics.o +REFTABLE_OBJS += reftable/merged.o REFTABLE_OBJS += reftable/pq.o REFTABLE_OBJS += reftable/reader.o REFTABLE_OBJS += reftable/record.o
@@ -2451,6 +2452,7 @@ REFTABLE_OBJS += reftable/writer.o REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o +REFTABLE_TEST_OBJS += reftable/merged_test.o REFTABLE_TEST_OBJS += reftable/pq_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/readwrite_test.o
reftable/merged.c new file +362 -0
362 lines not on this page · raw message
reftable/merged.h new file +35 -0
@@ -0,0 +1,35 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef MERGED_H +#define MERGED_H + +#include "pq.h" + +struct reftable_merged_table { +struct reftable_table *stack; +size_t stack_len; +uint32_t hash_id; +int suppress_deletions; + +uint64_t min; +uint64_t max; +}; + +struct merged_iter { +struct reftable_iterator *stack; +uint32_t hash_id; +size_t stack_len; +uint8_t typ; +int suppress_deletions; +struct merged_iter_pqueue pq; +}; + +void merged_table_release (struct reftable_merged_table *mt); + +#endif
reftable/merged_test.c new file +292 -0
292 lines not on this page · raw message
reftable/reftable-merged.h new file +72 -0
72 lines not on this page · raw message
t/helper/test-reftable.c +1 -0
@@ -5,6 +5,7 @@ int cmd__reftable(int argc, const char **argv) {basics_test_main (argc, argv);block_test_main (argc, argv); +merged_test_main (argc, argv);pq_test_main (argc, argv);record_test_main (argc, argv);readwrite_test_main (argc, argv);
Signature · 1 line
gitgitgadget
PATCH v3 16/19 reftable: implement refname validation
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The packed/loose format has restrictions on refnames: a and a/b cannot coexist. This limitation does not apply to reftable per se, but must be maintained for interoperability. This code adds validation routines to abort transactions that are trying to add invalid names.
--- Makefile | 1 + reftable/refname.c | 209 +++++++++++++++++++++++++++++++++++++++ reftable/refname.h | 29 ++++++ reftable/refname_test.c | 102 +++++++++++++++++++ t/helper/test-reftable.c | 1 + 5 files changed, 342 insertions(+) create mode 100644 reftable/refname.c create mode 100644 reftable/refname.h create mode 100644 reftable/refname_test.c
Makefile +1 -0
@@ -2456,6 +2456,7 @@ REFTABLE_TEST_OBJS += reftable/merged_test.o REFTABLE_TEST_OBJS += reftable/pq_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/readwrite_test.o +REFTABLE_TEST_OBJS += reftable/refname_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o REFTABLE_TEST_OBJS += reftable/tree_test.o
reftable/refname.c new file +209 -0
209 lines not on this page · raw message
reftable/refname.h new file +29 -0
@@ -0,0 +1,29 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ +#ifndef REFNAME_H +#define REFNAME_H + +#include "reftable-record.h" +#include "reftable-generic.h" + +struct modification { +struct reftable_table tab; + +char **add; +size_t add_len; + +char **del; +size_t del_len; +}; + +int validate_ref_record_addition (struct reftable_table tab, +struct reftable_ref_record *recs,size_t sz); + +int modification_validate (struct modification *mod); + +#endif
reftable/refname_test.c new file +102 -0
102 lines not on this page · raw message
t/helper/test-reftable.c +1 -0
@@ -8,6 +8,7 @@ int cmd__reftable(int argc, const char **argv)merged_test_main (argc, argv);pq_test_main (argc, argv);record_test_main (argc, argv); +refname_test_main (argc, argv);readwrite_test_main (argc, argv);tree_test_main (argc, argv);return 0 ;
Signature · 1 line
gitgitgadget
PATCH v3 17/19 reftable: implement stack, a mutable database of reftable files.
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
--- Makefile | 1 + reftable/reftable-stack.h | 128 ++++ reftable/stack.c | 1396 +++++++++++++++++++++++++++++++++++++ reftable/stack.h | 41 ++ reftable/stack_test.c | 949 +++++++++++++++++++++++++ t/helper/test-reftable.c | 1 + 6 files changed, 2516 insertions(+) create mode 100644 reftable/reftable-stack.h create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/stack_test.c
Makefile +1 -0
@@ -2457,6 +2457,7 @@ REFTABLE_TEST_OBJS += reftable/pq_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/readwrite_test.o REFTABLE_TEST_OBJS += reftable/refname_test.o +REFTABLE_TEST_OBJS += reftable/stack_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o REFTABLE_TEST_OBJS += reftable/tree_test.o
reftable/reftable-stack.h new file +128 -0
128 lines not on this page · raw message
reftable/stack.c new file +1396 -0
1,396 lines not on this page · raw message
reftable/stack.h new file +41 -0
41 lines not on this page · raw message
reftable/stack_test.c new file +949 -0
949 lines not on this page · raw message
t/helper/test-reftable.c +1 -0
@@ -10,6 +10,7 @@ int cmd__reftable(int argc, const char **argv)record_test_main (argc, argv);refname_test_main (argc, argv);readwrite_test_main (argc, argv); +stack_test_main (argc, argv);tree_test_main (argc, argv);return 0 ; }
Signature · 1 line
gitgitgadget
PATCH v3 18/19 reftable: add dump utility
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
provide a command-line utility for inspecting individual tables, and inspecting a complete ref database
--- reftable/dump.c | 107 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 reftable/dump.c
reftable/dump.c new file +107 -0
@@ -0,0 +1,107 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "git-compat-util.h" +#include "hash.h" + +#include "reftable-blocksource.h" +#include "reftable-error.h" +#include "reftable-merged.h" +#include "reftable-record.h" +#include "reftable-tests.h" +#include "reftable-writer.h" +#include "reftable-iterator.h" +#include "reftable-reader.h" +#include "reftable-stack.h" +#include "reftable-generic.h" + +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> + +static int compact_stack (const char *stackdir) +{ +struct reftable_stack *stack =NULL ; +struct reftable_write_options cfg = {0 }; + +int err =reftable_new_stack (&stack, stackdir, cfg); +if (err <0 ) + goto done; + + err =reftable_stack_compact_all (stack,NULL ); +if (err <0 ) + goto done; +done: +if (stack) { +reftable_stack_destroy (stack); + } +return err; +} + +static void print_help (void ) +{ +printf ("usage: dump [-cst] arg\n\n" +"options: \n" +" -c compact\n" +" -t dump table\n" +" -s dump stack\n" +" -6 sha256 hash format\n" +" -h this help\n" +"\n" ); +} + +int reftable_dump_main (int argc,char *const *argv) +{ +int err =0 ; +int opt_dump_table =0 ; +int opt_dump_stack =0 ; +int opt_compact =0 ; +uint32_t opt_hash_id =GIT_SHA1_FORMAT_ID ; +const char *arg =NULL , *argv0 = argv[0 ]; + +for (; argc >1 ; argv++, argc--) +if (*argv[1 ] !='-' ) +break ; +else if (!strcmp ("-t" , argv[1 ])) + opt_dump_table =1 ; +else if (!strcmp ("-6" , argv[1 ])) + opt_hash_id =GIT_SHA256_FORMAT_ID ; +else if (!strcmp ("-s" , argv[1 ])) + opt_dump_stack =1 ; +else if (!strcmp ("-c" , argv[1 ])) + opt_compact =1 ; +else if (!strcmp ("-?" , argv[1 ]) || !strcmp ("-h" , argv[1 ])) { +print_help (); +return 2 ; + } + +if (argc !=2 ) { +fprintf (stderr,"need argument\n" ); +print_help (); +return 2 ; + } + + arg = argv[1 ]; + +if (opt_dump_table) { + err =reftable_reader_print_file (arg); + }else if (opt_dump_stack) { + err =reftable_stack_print_directory (arg, opt_hash_id); + }else if (opt_compact) { + err =compact_stack (arg); + } + +if (err <0 ) { +fprintf (stderr,"%s: %s: %s\n" , argv0, arg, +reftable_error_str (err)); +return 1 ; + } +return 0 ; +}
Signature · 1 line
gitgitgadget
PATCH v3 19/19 Add "test-tool dump-reftable" command.
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This command dumps individual tables or a stack of of tables.
--- Makefile | 1 + t/helper/test-reftable.c | 5 +++++ t/helper/test-tool.c | 1 + t/helper/test-tool.h | 1 + 4 files changed, 8 insertions(+)
Makefile +1 -0
@@ -2452,6 +2452,7 @@ REFTABLE_OBJS += reftable/writer.o REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o +REFTABLE_TEST_OBJS += reftable/dump.o REFTABLE_TEST_OBJS += reftable/merged_test.o REFTABLE_TEST_OBJS += reftable/pq_test.o REFTABLE_TEST_OBJS += reftable/record_test.o
t/helper/test-reftable.c +5 -0
@@ -14,3 +14,8 @@ int cmd__reftable(int argc, const char **argv)tree_test_main (argc, argv);return 0 ; } + +int cmd__dump_reftable (int argc,const char **argv) +{ +return reftable_dump_main (argc, (char *const *)argv); +}
t/helper/test-tool.c +1 -0
@@ -61,6 +61,7 @@ static struct test_cmd cmds[] = { { "read-midx", cmd__read_midx }, { "ref-store", cmd__ref_store }, { "reftable", cmd__reftable }, + { "dump-reftable", cmd__dump_reftable }, { "regex", cmd__regex }, { "repository", cmd__repository }, { "revision-walking", cmd__revision_walking },
t/helper/test-tool.h +1 -0
@@ -19,6 +19,7 @@ int cmd__dump_cache_tree(int argc, const char **argv);int cmd__dump_fsmonitor (int argc,const char **argv);int cmd__dump_split_index (int argc,const char **argv);int cmd__dump_untracked_cache (int argc,const char **argv); +int cmd__dump_reftable (int argc,const char **argv);int cmd__example_decorate (int argc,const char **argv);int cmd__fast_rebase (int argc,const char **argv);int cmd__genrandom (int argc,const char **argv);
Signature · 1 line
gitgitgadget
Quoted 5 lines · Han-Wen Nienhuys via GitGitGadget
The reftable format is described in
Documentation/technical/reftable.txt.This is a fully reentrant implementation of reading and writing the reftable file format, and should be suitable for embedding in libgit2 too. It does not hook the code up to git to function as a ref storage backend yet.
Elsewhere you solicited comments on the "RFC - add LICENSE" step.
As long as the copyright holders of this library are happy with BSD, I do not see a problem with your plan in which the project borrows this code under the license and then owns it, giving access to others under the same license.
If somebody sees problems with it, please raise your hands ;-)
Thanks.
Quoted 8 lines · Junio C Hamano
Quoted 5 lines · Han-Wen Nienhuys via GitGitGadget
The reftable format is described in
Documentation/technical/reftable.txt.This is a fully reentrant implementation of reading and writing the reftable file format, and should be suitable for embedding in libgit2 too. It does not hook the code up to git to function as a ref storage backend yet.
Quoted 8 lines
Elsewhere you solicited comments on the "RFC - add LICENSE" step.
As long as the copyright holders of this library are happy with BSD, I do not see a problem with your plan in which the project borrows this code under the license and then owns it, giving access to others under the same license.
If somebody sees problems with it, please raise your hands ;-)
No objection, but as noted in previous discussions about it I think we should have some clarity about what it means for contributors. I.e. that when they're contributing to reftable/ it's BSD license, but for the rest of the codebase it's GPLv2.
I've just submitted a series to address that more generally:
https://lore.kernel.org/git/[email protected]/
I think with that clarification of what these in-tree subdirectory "COPYING" files mean the only thing left is for the "RFC" label to be dropped here.
From what I as not-a-lawyer think I know about how licenses work the concept that we can have BSD licensed content in-tree that can be extracted and legally used by anyone outside of it seems rather dubious.
I.e. if I author a cross-tree where "reftable/" uses some updated API, isn't the result a derived work and now GPLv2 and not "cleanly" BSD licensed? But that's a separate question, and ultimately a worry for any out-of-tree users. So I don't really care. I also believe that we've had that situation before with (I think) libgit2 taking snapshots of our "xdiff", so maybe it's already deemed to be a complete non-issue.
All I'd like is for contributors to git.git not to be confused, which I think with my series above won't be the case.
hn/reftable "fixes"
In reply to Han-Wen Nienhuys via GitGitGadgetPatch 1, was discussed before and might be solved in a better way as an alternative.
Patch 2 and 3 are "nice to have" for portability and hopefully not controversial but could be dropped, if someone feels strongly against it.
Patch 4 is not something I'd found failing anywhere, but the fact that Microsoft mentions it is only supported as an extension and it needs to be supported by the dynamic linker and I couldn't find anything clear about it in POSIX means, that is probably safer this way.
[PATCH 1/4] fixup! reftable: add a heap-based priority queue for [PATCH 2/4] fixup! reftable: implement stack, a mutable database of [PATCH 3/4] config.mak.uname: last release and snapshots of Minix [PATCH 4/4] reftable: avoid non portable compile time pointer to
PATCH 1/4 fixup! reftable: add a heap-based priority queue for reftable records
originally discussed as part of the -DNDEBUG thread[1], but still likely to break the build with the parent variable set but not used when assert gets compiled out.
moving to BUG or DIE still a better option otherwise, but this fixup will at least address that specific problem (even if the function then is a NOOP).
[1] https://lore.kernel.org/git/[email protected]/
--- reftable/pq_test.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-)
reftable/pq_test.c +2 -4
@@ -17,11 +17,9 @@ license that can be found in the LICENSE file or atvoid merged_iter_pqueue_check (struct merged_iter_pqueue pq) { -int i =0 ; +int i;for (i =1 ; i < pq.len; i++) { -int parent = (i -1 ) /2 ; - -assert (pq_less (&pq.heap[parent], &pq.heap[i])); +assert (pq_less (&pq.heap[(i -1 )/2 ], &pq.heap[i])); } }
Signature · 1 line
2.33.0.955.gee03ddbf0e
PATCH 2/4 fixup! reftable: implement stack, a mutable database of reftable files.
In reply to Carlo Marcelo Arenas BelónAvoid C++ style comments, that are not allowed in C89.
--- reftable/stack_test.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-)
reftable/stack_test.c +5 -3
@@ -46,8 +46,10 @@ static int count_dir_entries(const char *dirname)return len; } -// Work linenumber into the tempdir, so we can see which tests forget to -// cleanup. +/* +* Work linenumber into the tempdir, so we can see which tests forget to +* cleanup. +*/ static char *get_tmp_template (int linenumber) {const char *tmp =getenv ("TMPDIR" );
@@ -861,7 +863,7 @@ static void test_reftable_stack_compaction_concurrent(void)static void unclean_stack_close (struct reftable_stack *st) { -// break abstraction boundary to simulate unclean shutdown. +/* break abstraction boundary to simulate unclean shutdown */ int i =0 ;for (; i < st->readers_len; i++) {reftable_reader_free (st->readers[i]);
Signature · 1 line
2.33.0.955.gee03ddbf0e
C++(C99)-style comments in git.git
Quoted 25 lines · Carlo Marcelo Arenas Belón
Avoid C++ style comments, that are not allowed in C89.
Signed-off-by Carlo Marcelo Arenas Belón <[email protected]>--- reftable/stack_test.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-)reftable/stack_test.c +4
-2@@ -46,8 +46,10 @@ static int count_dir_entries(const char *dirname)return len; } -// Work linenumber into the tempdir, so we can see which tests forget to -// cleanup. +/* +* Work linenumber into the tempdir, so we can see which tests forget to +* cleanup. +*/ static char *get_tmp_template (int linenumber) {const char *tmp =getenv ("TMPDIR" );
reftable/stack_test.c static void test_reftable_stack_compaction_concurrent(void) · line 869
Quoted 8 lines
reftable/stack_test.c +1
-1@@ -861,7 +863,7 @@ static void test_reftable_stack_compaction_concurrent(void)static void unclean_stack_close (struct reftable_stack *st) { -// break abstraction boundary to simulate unclean shutdown. +/* break abstraction boundary to simulate unclean shutdown */ int i =0 ;for (; i < st->readers_len; i++) {reftable_reader_free (st->readers[i]);
This change looks fine, but FYI we had a C++ comment in git.git from v2.21.0 with d73019feb44 (http: add support selecting http version, 2018-11-08) until now.
It'll be removed in v2.34.0 with my 2a7f64616a3 (http: correct version check for CURL_HTTP_VERSION_2, 2021-09-13) (not the purpose of that change).
Now, granted it's "only" compiled if you don't have NO_CURL=Y, and it's on a "#" pre-processor line, not in the "main" code.
So it's possible (but I think not likely) that someone's cpp accepts it, but not their cc. Or that some platform has nobody compiling with curl (not very likely).
I'm not advocating for using "//" comments, but I think it's fair to say given the above that they're purely an asthetic coding style preference in the context of git.git, and not a matter of C89 portability that anyone cares about when building or porting git.
We already have a hard dependency on a few C99-only features, I'd think it's very unlikely that someone has say implemented designated initializers, but not "//" comment support. So for our purposes support for it should be universal.
PATCH 3/4 config.mak.uname: last release and snapshots of Minix still use zlib 1.2.3
In reply to Carlo Marcelo Arenas BelónMinix 3 uses a BSD userspace from NetBSD, but might had synced last before 8.x got branched.
While Minix itself doesn't have much activity, this is still affecting the last release (and subsequent snapshots).
--- config.mak.uname | 1 + 1 file changed, 1 insertion(+)
config.mak.uname +1 -0
@@ -517,6 +517,7 @@ ifeq ($(uname_S),Interix) endif endif ifeq ($(uname_S),Minix) + NO_UNCOMPRESS2 = YesPlease NO_IPV6 = YesPlease NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease NO_NSEC = YesPlease
Signature · 1 line
2.33.0.955.gee03ddbf0e
PATCH 4/4 reftable: avoid non portable compile time pointer to function
In reply to Carlo Marcelo Arenas BelónWhile GNU compilers (and even MSVC) allow as an extension setting at compile time the address of a dynamic function into a static variables, it trigger C4232[1] with a warning for portability.
Avoid it by resolving at runtime the address that will be used on each case, and not relying on the pointer initialization to be honour at compile time.
[1] https://docs.microsoft.com/en-us/cpp/error-messages/compiler-warnings/compiler-warning-level-4-c4232
--- reftable/publicbasics.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-)
reftable/publicbasics.c +15 -6
@@ -11,23 +11,32 @@ license that can be found in the LICENSE file or at#include "basics.h" #include "system.h" -static void *(*reftable_malloc_ptr)(size_t sz) = &malloc; -static void *(*reftable_realloc_ptr)(void *,size_t ) = &realloc; -static void (*reftable_free_ptr)(void *) = &free; +static void *(*reftable_malloc_ptr)(size_t sz); +static void *(*reftable_realloc_ptr)(void *,size_t ); +static void (*reftable_free_ptr)(void *);void *reftable_malloc (size_t sz) { -return (*reftable_malloc_ptr)(sz); +if (reftable_malloc_ptr) +return reftable_malloc_ptr (sz); +else +return malloc (sz); }void *reftable_realloc (void *p,size_t sz) { -return (*reftable_realloc_ptr)(p, sz); +if (reftable_realloc_ptr) +return reftable_realloc_ptr (p, sz); +else +return realloc (p, sz); }void reftable_free (void *p) { -reftable_free_ptr (p); +if (reftable_free_ptr) +reftable_free_ptr (p); +else +free (p); }void *reftable_calloc (size_t sz)
Signature · 1 line
2.33.0.955.gee03ddbf0e
Re: hn/reftable "fixes"
In reply to Carlo Marcelo Arenas BelónQuoted 10 lines · Carlo Marcelo Arenas Belón
Patch 1, was discussed before and might be solved in a better way as an alternative.
Patch 2 and 3 are "nice to have" for portability and hopefully not controversial but could be dropped, if someone feels strongly against it.
Patch 4 is not something I'd found failing anywhere, but the fact that Microsoft mentions it is only supported as an extension and it needs to be supported by the dynamic linker and I couldn't find anything clear about it in POSIX means, that is probably safer this way.
All of them look sensible. Thanks.
v4 cover Adds reftable library code from https://github.com/hanwen/reftable.
In reply to Han-Wen Nienhuys via GitGitGadgetThe reftable format is described in Documentation/technical/reftable.txt.
This is a fully reentrant implementation of reading and writing the reftable file format, and should be suitable for embedding in libgit2 too. It does not hook the code up to git to function as a ref storage backend yet.
v4:
* fixes by Carlo Belón.
* remove RFC from LICENSE commit.
Han-Wen Nienhuys (19):
hash.h: provide constants for the hash IDs
reftable: add LICENSE
reftable: add error related functionality
reftable: utility functions
reftable: add blocksource, an abstraction for random access reads
reftable: (de)serialization for the polymorphic record type. Provide zlib's uncompress2 from compat/zlib-compat.c
reftable: reading/writing blocks
reftable: a generic binary tree implementation
reftable: write reftable files
reftable: generic interface to tables
reftable: read reftable files
reftable: reftable file level tests
reftable: add a heap-based priority queue for reftable records
reftable: add merged table view
reftable: implement refname validation
reftable: implement stack, a mutable database of reftable files.
reftable: add dump utility
Add "test-tool dump-reftable" command.
Makefile | 53 +- ci/lib.sh | 1 + compat/.gitattributes | 1 + compat/zlib-uncompress2.c | 95 ++ config.mak.uname | 5 + configure.ac | 13 + contrib/buildsystems/CMakeLists.txt | 14 +- contrib/buildsystems/Generators/Vcxproj.pm | 11 +- hash.h | 6 + object-file.c | 7 +- reftable/LICENSE | 31 + reftable/basics.c | 128 ++ reftable/basics.h | 60 + reftable/basics_test.c | 98 ++ reftable/block.c | 437 ++++++ reftable/block.h | 127 ++ reftable/block_test.c | 120 ++ reftable/blocksource.c | 148 +++ reftable/blocksource.h | 22 + reftable/constants.h | 21 + reftable/dump.c | 107 ++ reftable/error.c | 41 + reftable/generic.c | 169 +++ reftable/generic.h | 32 + reftable/iter.c | 194 +++ reftable/iter.h | 69 + reftable/merged.c | 362 +++++ reftable/merged.h | 38 + reftable/merged_test.c | 468 +++++++ reftable/pq.c | 105 ++ reftable/pq.h | 33 + reftable/pq_test.c | 82 ++ reftable/publicbasics.c | 65 + reftable/reader.c | 801 +++++++++++ reftable/reader.h | 64 + reftable/readwrite_test.c | 652 +++++++++ reftable/record.c | 1212 +++++++++++++++++ reftable/record.h | 139 ++ reftable/record_test.c | 412 ++++++ reftable/refname.c | 209 +++ reftable/refname.h | 29 + reftable/refname_test.c | 102 ++ reftable/reftable-blocksource.h | 49 + reftable/reftable-error.h | 62 + reftable/reftable-generic.h | 47 + reftable/reftable-iterator.h | 39 + reftable/reftable-malloc.h | 18 + reftable/reftable-merged.h | 72 + reftable/reftable-reader.h | 101 ++ reftable/reftable-record.h | 114 ++ reftable/reftable-stack.h | 128 ++ reftable/reftable-tests.h | 23 + reftable/reftable-writer.h | 148 +++ reftable/reftable.c | 115 ++ reftable/stack.c | 1396 ++++++++++++++++++++ reftable/stack.h | 41 + reftable/stack_test.c | 953 +++++++++++++ reftable/system.h | 32 + reftable/test_framework.c | 23 + reftable/test_framework.h | 53 + reftable/tree.c | 63 + reftable/tree.h | 34 + reftable/tree_test.c | 61 + reftable/writer.c | 690 ++++++++++ reftable/writer.h | 50 + t/helper/test-reftable.c | 21 + t/helper/test-tool.c | 4 +- t/helper/test-tool.h | 2 + t/t0032-reftable-unittest.sh | 15 + 69 files changed, 11125 insertions(+), 12 deletions(-) create mode 100644 compat/.gitattributes create mode 100644 compat/zlib-uncompress2.c create mode 100644 reftable/LICENSE create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/basics_test.c create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/block_test.c create mode 100644 reftable/blocksource.c create mode 100644 reftable/blocksource.h create mode 100644 reftable/constants.h create mode 100644 reftable/dump.c create mode 100644 reftable/error.c create mode 100644 reftable/generic.c create mode 100644 reftable/generic.h create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/merged_test.c create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/pq_test.c create mode 100644 reftable/publicbasics.c create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/readwrite_test.c create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/record_test.c create mode 100644 reftable/refname.c create mode 100644 reftable/refname.h create mode 100644 reftable/refname_test.c create mode 100644 reftable/reftable-blocksource.h create mode 100644 reftable/reftable-error.h create mode 100644 reftable/reftable-generic.h create mode 100644 reftable/reftable-iterator.h create mode 100644 reftable/reftable-malloc.h create mode 100644 reftable/reftable-merged.h create mode 100644 reftable/reftable-reader.h create mode 100644 reftable/reftable-record.h create mode 100644 reftable/reftable-stack.h create mode 100644 reftable/reftable-tests.h create mode 100644 reftable/reftable-writer.h create mode 100644 reftable/reftable.c create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/stack_test.c create mode 100644 reftable/system.h create mode 100644 reftable/test_framework.c create mode 100644 reftable/test_framework.h create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100644 reftable/tree_test.c create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h create mode 100644 t/helper/test-reftable.c create mode 100755 t/t0032-reftable-unittest.sh
base-commit: 106298f7f9cca4158a980de149ef217751e1f943
Published-As: https://github.com/gitgitgadget/git/releases/tag/pr-git-1081%2Fhanwen%2Fjust-library-v4
Fetch-It-Via: git fetch https://github.com/gitgitgadget/git pr-git-1081/hanwen/just-library-v4
Pull-Request: https://github.com/git/git/pull/1081
Range-diff vs v3:
1: b2f8dc32bfc = 1: 9cc44f78377 hash.h: provide constants for the hash IDs 2: 2597ddfeae1 ! 2: 6a5ee5db472 reftable: RFC: add LICENSE
@@ Metadata
Author: Han-Wen Nienhuys <[email protected]>
## Commit message ##
- reftable: RFC: add LICENSE
+ reftable: add LICENSE
The objective of this code is to be usable as a C library, so it can be reused in libgit2.
3: 41b1ff1293f ! 3: bf51bc9ab0c reftable: add error related functionality
@@ Commit message
reftable: add error related functionality
The reftable/ directory is structured as a library, so it cannot
- crash on misuse. Instead, it returns an error codes.
+ crash on misuse. Instead, it returns an error code.
- In addition, the error code can be used to signal conditions from lower levels
- of the library to be handled by higher levels of the library. For example, a
- transaction might legitimately write an empty reftable file, but in that case,
- we'd want to shortcut the transaction overhead.
+ In addition to signaling errors, the error code can be used to signal
+ conditions from lower levels of the library to be handled by higher
+ levels of the library. For example, in a transaction we might
+ legitimately write an empty reftable file, but in that case, we want to
+ shortcut the transaction.
4: c03043c5e63 ! 4: 318e7ad41e5 reftable: utility functions
@@ reftable/publicbasics.c (new)
+#include "basics.h"
+#include "system.h"
+
-+static void *(*reftable_malloc_ptr)(size_t sz) = &malloc;
-+static void *(*reftable_realloc_ptr)(void *, size_t) = &realloc;
-+static void (*reftable_free_ptr)(void *) = &free;
++static void *(*reftable_malloc_ptr)(size_t sz);
++static void *(*reftable_realloc_ptr)(void *, size_t);
++static void (*reftable_free_ptr)(void *);
+
+void *reftable_malloc(size_t sz)
+{
-+ return (*reftable_malloc_ptr)(sz);
++ if (reftable_malloc_ptr)
++ return (*reftable_malloc_ptr)(sz);
++ return malloc(sz);
+}
+
+void *reftable_realloc(void *p, size_t sz)
+{
-+ return (*reftable_realloc_ptr)(p, sz);
++ if (reftable_realloc_ptr)
++ return (*reftable_realloc_ptr)(p, sz);
++ return realloc(p, sz);
+}
+
+void reftable_free(void *p)
+{
-+ reftable_free_ptr(p);
++ if (reftable_free_ptr)
++ reftable_free_ptr(p);
++ else
++ free(p);
+}
+
+void *reftable_calloc(size_t sz)
5: 317a6aaa357 = 5: 80036f93e2e reftable: add blocksource, an abstraction for random access reads
6: 55e66b7a74f = 6: f2cd2876265 reftable: (de)serialization for the polymorphic record type.
7: 4f6ecd3646a ! 7: 6cbc2f758df Provide zlib's uncompress2 from compat/zlib-compat.c
@@ config.mak.uname: ifeq ($(uname_S),FreeBSD)
NO_STRCASESTR = YesPlease
NO_MEMMEM = YesPlease
USE_ST_TIMESPEC = YesPlease
+@@ config.mak.uname: ifeq ($(uname_S),Interix)
+ endif
+ endif
+ ifeq ($(uname_S),Minix)
++ NO_UNCOMPRESS2 = YesPlease
+ NO_IPV6 = YesPlease
+ NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease
+ NO_NSEC = YesPlease ## configure.ac ##
@@ configure.ac: AC_LINK_IFELSE([ZLIBTEST_SRC],
8: 04f2d74df51 ! 8: 1002d6f526c reftable: reading/writing blocks
@@ Commit message
This commit provides the logic to read and write these blocks.
+ Helped-by: Carlo Marcelo Arenas Belón <[email protected]>
Signed-off-by: Han-Wen Nienhuys <[email protected]>
## Makefile ##
@@ reftable/block.c (new)
+
+int block_writer_finish(struct block_writer *w)
+{
-+ int i = 0;
++ int i;
+ for (i = 0; i < w->restart_len; i++) {
+ put_be24(w->buf + w->next, w->restarts[i]);
+ w->next += 3;
@@ reftable/block.c (new)
+
+ if (block_writer_type(w) == BLOCK_TYPE_LOG) {
+ int block_header_skip = 4 + w->header_off;
-+ uint8_t *compressed = NULL;
-+ int zresult = 0;
+ uLongf src_len = w->next - block_header_skip;
-+ size_t dest_cap = src_len;
++ uLongf dest_cap = src_len * 1.001 + 12;
+
-+ compressed = reftable_malloc(dest_cap);
++ uint8_t *compressed = reftable_malloc(dest_cap);
+ while (1) {
+ uLongf out_dest_len = dest_cap;
-+
-+ zresult = compress2(compressed, &out_dest_len,
-+ w->buf + block_header_skip, src_len,
-+ 9);
-+ if (zresult == Z_BUF_ERROR) {
++ int zresult = compress2(compressed, &out_dest_len,
++ w->buf + block_header_skip,
++ src_len, 9);
++ if (zresult == Z_BUF_ERROR && dest_cap < LONG_MAX) {
+ dest_cap *= 2;
+ compressed =
+ reftable_realloc(compressed, dest_cap);
-+ continue;
++ if (compressed)
++ continue;
+ }
+
+ if (Z_OK != zresult) {
9: f6c2da61208 = 9: 96e490ac32c reftable: a generic binary tree implementation
10: 277a1f662e3 = 10: daf9cc897d1 reftable: write reftable files
11: f97182dec1f = 11: c9a5813b211 reftable: generic interface to tables
12: 13de5d03a71 ! 12: 854b30268b6 reftable: read reftable files
@@ reftable/reader.c (new)
+ return 0;
+}
+
-+int reader_seek(struct reftable_reader *r, struct reftable_iterator *it,
-+ struct reftable_record *rec)
++static int reader_seek(struct reftable_reader *r, struct reftable_iterator *it,
++ struct reftable_record *rec)
+{
+ uint8_t typ = reftable_record_type(rec);
+
@@ reftable/reader.h (new)
+
+int init_reader(struct reftable_reader *r, struct reftable_block_source *source,
+ const char *name);
-+int reader_seek(struct reftable_reader *r, struct reftable_iterator *it,
-+ struct reftable_record *rec);
+void reader_close(struct reftable_reader *r);
+const char *reader_name(struct reftable_reader *r);
+
13: 8efc3c2b6ab = 13: c0d0427d52c reftable: reftable file level tests
14: e89b16a37e6 ! 14: 6e5ee043083 reftable: add a heap-based priority queue for reftable records
@@ reftable/pq_test.c (new)
+
+void merged_iter_pqueue_check(struct merged_iter_pqueue pq)
+{
-+ int i = 0;
++ int i;
+ for (i = 1; i < pq.len; i++) {
+ int parent = (i - 1) / 2;
+
-+ assert(pq_less(&pq.heap[parent], &pq.heap[i]));
++ EXPECT(pq_less(&pq.heap[parent], &pq.heap[i]));
+ }
+}
+
15: 40a91b14be8 ! 15: 0b86f6c2327 reftable: add merged table view
@@ reftable/merged.h (new)
+ struct reftable_table *stack;
+ size_t stack_len;
+ uint32_t hash_id;
++
++ /* If unset, produce deletions. This is useful for compaction. For the
++ * full stack, deletions should be produced. */
+ int suppress_deletions;
+
+ uint64_t min;
@@ reftable/merged_test.c (new)
+ reftable_writer_free(w);
+}
+
++static void write_test_log_table(struct strbuf *buf,
++ struct reftable_log_record logs[], int n,
++ uint64_t update_index)
++{
++ int i = 0;
++ int err;
++
++ struct reftable_write_options opts = {
++ .block_size = 256,
++ .exact_log_message = 1,
++ };
++ struct reftable_writer *w = NULL;
++ w = reftable_new_writer(&strbuf_add_void, buf, &opts);
++ reftable_writer_set_limits(w, update_index, update_index);
++
++ for (i = 0; i < n; i++) {
++ int err = reftable_writer_add_log(w, &logs[i]);
++ EXPECT_ERR(err);
++ }
++
++ err = reftable_writer_close(w);
++ EXPECT_ERR(err);
++
++ reftable_writer_free(w);
++}
++
+static struct reftable_merged_table *
+merged_table_from_records(struct reftable_ref_record **refs,
+ struct reftable_block_source **source,
@@ reftable/merged_test.c (new)
+ int i = 0;
+
+ EXPECT_ERR(err);
++ EXPECT(reftable_merged_table_hash_id(mt) == GIT_SHA1_FORMAT_ID);
++ EXPECT(reftable_merged_table_min_update_index(mt) == 1);
++
+ while (len < 100) { /* cap loops/recursion. */
+ struct reftable_ref_record ref = { NULL };
+ int err = reftable_iterator_next_ref(&it, &ref);
@@ reftable/merged_test.c (new)
+ reftable_free(bs);
+}
+
++static struct reftable_merged_table *
++merged_table_from_log_records(struct reftable_log_record **logs,
++ struct reftable_block_source **source,
++ struct reftable_reader ***readers, int *sizes,
++ struct strbuf *buf, int n)
++{
++ int i = 0;
++ struct reftable_merged_table *mt = NULL;
++ int err;
++ struct reftable_table *tabs =
++ reftable_calloc(n * sizeof(struct reftable_table));
++ *readers = reftable_calloc(n * sizeof(struct reftable_reader *));
++ *source = reftable_calloc(n * sizeof(**source));
++ for (i = 0; i < n; i++) {
++ write_test_log_table(&buf[i], logs[i], sizes[i], i + 1);
++ block_source_from_strbuf(&(*source)[i], &buf[i]);
++
++ err = reftable_new_reader(&(*readers)[i], &(*source)[i],
++ "name");
++ EXPECT_ERR(err);
++ reftable_table_from_reader(&tabs[i], (*readers)[i]);
++ }
++
++ err = reftable_new_merged_table(&mt, tabs, n, GIT_SHA1_FORMAT_ID);
++ EXPECT_ERR(err);
++ return mt;
++}
++
++static void test_merged_logs(void)
++{
++ uint8_t hash1[GIT_SHA1_RAWSZ] = { 1 };
++ uint8_t hash2[GIT_SHA1_RAWSZ] = { 2 };
++ uint8_t hash3[GIT_SHA1_RAWSZ] = { 3 };
++ struct reftable_log_record r1[] = {
++ {
++ .refname = "a",
++ .update_index = 2,
++ .value_type = REFTABLE_LOG_UPDATE,
++ .value.update = {
++ .old_hash = hash2,
++ /* deletion */
++ .name = "jane doe",
++ .email = "jane@invalid",
++ .message = "message2",
++ }
++ },
++ {
++ .refname = "a",
++ .update_index = 1,
++ .value_type = REFTABLE_LOG_UPDATE,
++ .value.update = {
++ .old_hash = hash1,
++ .new_hash = hash2,
++ .name = "jane doe",
++ .email = "jane@invalid",
++ .message = "message1",
++ }
++ },
++ };
++ struct reftable_log_record r2[] = {
++ {
++ .refname = "a",
++ .update_index = 3,
++ .value_type = REFTABLE_LOG_UPDATE,
++ .value.update = {
++ .new_hash = hash3,
++ .name = "jane doe",
++ .email = "jane@invalid",
++ .message = "message3",
++ }
++ },
++ };
++ struct reftable_log_record r3[] = {
++ {
++ .refname = "a",
++ .update_index = 2,
++ .value_type = REFTABLE_LOG_DELETION,
++ },
++ };
++ struct reftable_log_record want[] = {
++ r2[0],
++ r3[0],
++ r1[1],
++ };
++
++ struct reftable_log_record *logs[] = { r1, r2, r3 };
++ int sizes[3] = { 2, 1, 1 };
++ struct strbuf bufs[3] = { STRBUF_INIT, STRBUF_INIT, STRBUF_INIT };
++ struct reftable_block_source *bs = NULL;
++ struct reftable_reader **readers = NULL;
++ struct reftable_merged_table *mt = merged_table_from_log_records(
++ logs, &bs, &readers, sizes, bufs, 3);
++
++ struct reftable_iterator it = { NULL };
++ int err = reftable_merged_table_seek_log(mt, &it, "a");
++ struct reftable_log_record *out = NULL;
++ size_t len = 0;
++ size_t cap = 0;
++ int i = 0;
++
++ EXPECT_ERR(err);
++ EXPECT(reftable_merged_table_hash_id(mt) == GIT_SHA1_FORMAT_ID);
++ EXPECT(reftable_merged_table_min_update_index(mt) == 1);
++
++ while (len < 100) { /* cap loops/recursion. */
++ struct reftable_log_record log = { NULL };
++ int err = reftable_iterator_next_log(&it, &log);
++ if (err > 0) {
++ break;
++ }
++ if (len == cap) {
++ cap = 2 * cap + 1;
++ out = reftable_realloc(
++ out, sizeof(struct reftable_log_record) * cap);
++ }
++ out[len++] = log;
++ }
++ reftable_iterator_destroy(&it);
++
++ EXPECT(ARRAY_SIZE(want) == len);
++ for (i = 0; i < len; i++) {
++ EXPECT(reftable_log_record_equal(&want[i], &out[i],
++ GIT_SHA1_RAWSZ));
++ }
++
++ err = reftable_merged_table_seek_log_at(mt, &it, "a", 2);
++ EXPECT_ERR(err);
++ reftable_log_record_release(&out[0]);
++ err = reftable_iterator_next_log(&it, &out[0]);
++ EXPECT_ERR(err);
++ EXPECT(reftable_log_record_equal(&out[0], &r3[0], GIT_SHA1_RAWSZ));
++ reftable_iterator_destroy(&it);
++
++ for (i = 0; i < len; i++) {
++ reftable_log_record_release(&out[i]);
++ }
++ reftable_free(out);
++
++ for (i = 0; i < 3; i++) {
++ strbuf_release(&bufs[i]);
++ }
++ readers_destroy(readers, 3);
++ reftable_merged_table_free(mt);
++ reftable_free(bs);
++}
++
+static void test_default_write_opts(void)
+{
+ struct reftable_write_options opts = { 0 };
@@ reftable/merged_test.c (new)
+
+int merged_test_main(int argc, const char *argv[])
+{
++ RUN_TEST(test_merged_logs);
+ RUN_TEST(test_merged_between);
+ RUN_TEST(test_merged);
+ RUN_TEST(test_default_write_opts);
16: 46c2ddf07cf = 16: c600e1f975c reftable: implement refname validation
17: 3db887bc078 ! 17: 55dacc6760f reftable: implement stack, a mutable database of reftable files.
@@ reftable/stack_test.c (new)
+ return len;
+}
+
-+// Work linenumber into the tempdir, so we can see which tests forget to
-+// cleanup.
++/*
++ * Work linenumber into the tempdir, so we can see which tests forget to
++ * cleanup.
++ */
+static char *get_tmp_template(int linenumber)
+{
+ const char *tmp = getenv("TMPDIR");
@@ reftable/stack_test.c (new)
+ err = reftable_new_stack(&st, dir, cfg);
+ EXPECT_ERR(err);
+
++ st->disable_auto_compact = 1; /* call manually below for coverage. */
+ for (i = 0; i < N; i++) {
+ char name[100];
+ struct reftable_ref_record ref = {
@@ reftable/stack_test.c (new)
+ err = reftable_stack_add(st, &write_test_ref, &ref);
+ EXPECT_ERR(err);
+
++ err = reftable_stack_auto_compact(st);
+ EXPECT(i < 3 || st->merged->stack_len < 2 * fastlog2(i));
+ }
+
@@ reftable/stack_test.c (new)
+
+static void unclean_stack_close(struct reftable_stack *st)
+{
-+ // break abstraction boundary to simulate unclean shutdown.
++ /* break abstraction boundary to simulate unclean shutdown. */
+ int i = 0;
+ for (; i < st->readers_len; i++) {
+ reftable_reader_free(st->readers[i]);
18: 795a15000fb = 18: 2f825cdb2d4 reftable: add dump utility
19: 7b3215aef39 = 19: c8428cb0df8 Add "test-tool dump-reftable" command.Signature · 1 line
gitgitgadget
PATCH v4 1/19 hash.h: provide constants for the hash IDs
From: Han-Wen Nienhuys <[email protected]>
This will simplify referencing them from code that is not deeply integrated with Git, in particular, the reftable library.
--- hash.h | 6 ++++++ object-file.c | 7 ++----- 2 files changed, 8 insertions(+), 5 deletions(-)
hash.h +6 -0
@@ -95,12 +95,18 @@ static inline void git_SHA256_Clone(git_SHA256_CTX *dst, const git_SHA256_CTX *s/* Number of algorithms supported (including unknown). */ #define GIT_HASH_NALGOS (GIT_HASH_SHA256 + 1) +/* "sha1", big-endian */ +#define GIT_SHA1_FORMAT_ID 0x73686131 +/* The length in bytes and in hex digits of an object name (SHA-1 value). */ #define GIT_SHA1_RAWSZ 20#define GIT_SHA1_HEXSZ (2 * GIT_SHA1_RAWSZ)/* The block size of SHA-1. */ #define GIT_SHA1_BLKSZ 64 +/* "s256", big-endian */ +#define GIT_SHA256_FORMAT_ID 0x73323536 +/* The length in bytes and in hex digits of an object name (SHA-256 value). */ #define GIT_SHA256_RAWSZ 32#define GIT_SHA256_HEXSZ (2 * GIT_SHA256_RAWSZ)
object-file.c +2 -5
@@ -165,7 +165,6 @@ static void git_hash_unknown_final_oid(struct object_id *oid, git_hash_ctx *ctx)BUG ("trying to finalize unknown hash" ); } -const struct git_hash_algo hash_algos[GIT_HASH_NALGOS ] = { {NULL ,
@@ -184,8 +183,7 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = { }, {"sha1" , -/* "sha1", big-endian */ -0x73686131 , +GIT_SHA1_FORMAT_ID ,GIT_SHA1_RAWSZ ,GIT_SHA1_HEXSZ , GIT_SHA1_BLKSZ,
@@ -200,8 +198,7 @@ const struct git_hash_algo hash_algos[GIT_HASH_NALGOS] = { }, {"sha256" , -/* "s256", big-endian */ -0x73323536 , +GIT_SHA256_FORMAT_ID ,GIT_SHA256_RAWSZ ,GIT_SHA256_HEXSZ , GIT_SHA256_BLKSZ,
Signature · 1 line
gitgitgadget
PATCH v4 2/19 reftable: add LICENSE
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The objective of this code is to be usable as a C library, so it can be reused in libgit2.
This is currently using a BSD license as it is the liberal license I could find, but this could be changed to whatever fits the stated goal above.
This code is currently imported from github.com/hanwen/reftable. Once this code lands in git.git, the C code will be removed from github.com/hanwen/reftable, and the git.git code will be the source of truth.
--- reftable/LICENSE | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 reftable/LICENSE
reftable/LICENSE new file +31 -0
@@ -0,0 +1,31 @@ +BSD License + +Copyright (c) 2020, Google LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of Google LLC nor the names of its contributors may +be used to endorse or promote products derived from this software +without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Signature · 1 line
gitgitgadget
PATCH v4 3/19 reftable: add error related functionality
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The reftable/ directory is structured as a library, so it cannot crash on misuse. Instead, it returns an error code.
In addition to signaling errors, the error code can be used to signal conditions from lower levels of the library to be handled by higher levels of the library. For example, in a transaction we might legitimately write an empty reftable file, but in that case, we want to shortcut the transaction.
--- reftable/error.c | 41 ++++++++++++++++++++++++++ reftable/reftable-error.h | 62 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 reftable/error.c create mode 100644 reftable/reftable-error.h
reftable/error.c new file +41 -0
@@ -0,0 +1,41 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "reftable-error.h" + +#include <stdio.h> + +const char *reftable_error_str (int err) +{ +static char buf[250 ]; +switch (err) { +case REFTABLE_IO_ERROR : +return "I/O error" ; +case REFTABLE_FORMAT_ERROR : +return "corrupt reftable file" ; +case REFTABLE_NOT_EXIST_ERROR : +return "file does not exist" ; +case REFTABLE_LOCK_ERROR : +return "data is outdated" ; +case REFTABLE_API_ERROR : +return "misuse of the reftable API" ; +case REFTABLE_ZLIB_ERROR : +return "zlib failure" ; +case REFTABLE_NAME_CONFLICT : +return "file/directory conflict" ; +case REFTABLE_EMPTY_TABLE_ERROR : +return "wrote empty table" ; +case REFTABLE_REFNAME_ERROR : +return "invalid refname" ; +case -1 : +return "general error" ; +default : +snprintf (buf,sizeof (buf),"unknown error code %d" , err); +return buf; + } +}
reftable/reftable-error.h new file +62 -0
62 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v4 4/19 reftable: utility functions
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This commit provides basic utility classes for the reftable library.
--- Makefile | 25 +++- contrib/buildsystems/CMakeLists.txt | 14 ++- contrib/buildsystems/Generators/Vcxproj.pm | 11 +- reftable/basics.c | 128 +++++++++++++++++++++ reftable/basics.h | 60 ++++++++++ reftable/basics_test.c | 98 ++++++++++++++++ reftable/publicbasics.c | 65 +++++++++++ reftable/reftable-malloc.h | 18 +++ reftable/reftable-tests.h | 22 ++++ reftable/system.h | 32 ++++++ reftable/test_framework.c | 23 ++++ reftable/test_framework.h | 53 +++++++++ t/helper/test-reftable.c | 9 ++ t/helper/test-tool.c | 3 +- t/helper/test-tool.h | 1 + t/t0032-reftable-unittest.sh | 15 +++ 16 files changed, 570 insertions(+), 7 deletions(-) create mode 100644 reftable/basics.c create mode 100644 reftable/basics.h create mode 100644 reftable/basics_test.c create mode 100644 reftable/publicbasics.c create mode 100644 reftable/reftable-malloc.h create mode 100644 reftable/reftable-tests.h create mode 100644 reftable/system.h create mode 100644 reftable/test_framework.c create mode 100644 reftable/test_framework.h create mode 100644 t/helper/test-reftable.c create mode 100755 t/t0032-reftable-unittest.sh
Makefile +22 -3
@@ -736,6 +736,7 @@ TEST_BUILTINS_OBJS += test-read-cache.o TEST_BUILTINS_OBJS += test-read-graph.o TEST_BUILTINS_OBJS += test-read-midx.o TEST_BUILTINS_OBJS += test-ref-store.o +TEST_BUILTINS_OBJS += test-reftable.o TEST_BUILTINS_OBJS += test-regex.o TEST_BUILTINS_OBJS += test-repository.o TEST_BUILTINS_OBJS += test-revision-walking.o
@@ -814,6 +815,8 @@ TEST_SHELL_PATH = $(SHELL_PATH) LIB_FILE = libgit.a XDIFF_LIB = xdiff/lib.a +REFTABLE_LIB = reftable/libreftable.a +REFTABLE_TEST_LIB = reftable/libreftable_test.a GENERATED_H += command-list.h GENERATED_H += config-list.h
@@ -1188,7 +1191,7 @@ THIRD_PARTY_SOURCES += compat/regex/% THIRD_PARTY_SOURCES += sha1collisiondetection/% THIRD_PARTY_SOURCES += sha1dc/% -GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) +GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) EXTLIBS = GIT_USER_AGENT = git/$(GIT_VERSION)
@@ -2426,7 +2429,15 @@ XDIFF_OBJS += xdiff/xutils.o .PHONY: xdiff-objs xdiff-objs: $(XDIFF_OBJS) +REFTABLE_OBJS += reftable/basics.o +REFTABLE_OBJS += reftable/error.o +REFTABLE_OBJS += reftable/publicbasics.o + +REFTABLE_TEST_OBJS += reftable/test_framework.o +REFTABLE_TEST_OBJS += reftable/basics_test.o + TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) + .PHONY: test-objs test-objs: $(TEST_OBJS)
@@ -2442,6 +2453,8 @@ OBJECTS += $(PROGRAM_OBJS) OBJECTS += $(TEST_OBJS) OBJECTS += $(XDIFF_OBJS) OBJECTS += $(FUZZ_OBJS) +OBJECTS += $(REFTABLE_OBJS) $(REFTABLE_TEST_OBJS) + ifndef NO_CURL OBJECTS += http.o http-walker.o remote-curl.o endif
@@ -2591,6 +2604,12 @@ $(LIB_FILE): $(LIB_OBJS) $(XDIFF_LIB): $(XDIFF_OBJS) $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ +$(REFTABLE_LIB): $(REFTABLE_OBJS) + $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ + +$(REFTABLE_TEST_LIB): $(REFTABLE_TEST_OBJS) + $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ + export DEFAULT_EDITOR DEFAULT_PAGER Documentation/GIT-EXCLUDED-PROGRAMS: FORCE
@@ -2889,7 +2908,7 @@ perf: all t/helper/test-tool$X: $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS)) -t/helper/test-%$X: t/helper/test-%.o GIT-LDFLAGS $(GITLIBS) +t/helper/test-%$X: t/helper/test-%.o GIT-LDFLAGS $(GITLIBS) $(REFTABLE_TEST_LIB) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(filter %.a,$^) $(LIBS) check-sha1:: t/helper/test-tool$X
@@ -3224,7 +3243,7 @@ cocciclean: clean: profile-clean coverage-clean cocciclean $(RM) *.res $(RM) $(OBJECTS) - $(RM) $(LIB_FILE) $(XDIFF_LIB) + $(RM) $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) $(REFTABLE_TEST_LIB) $(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) git$X $(RM) $(TEST_PROGRAMS) $(RM) $(FUZZ_PROGRAMS)
contrib/buildsystems/CMakeLists.txt +12 -2
@@ -640,6 +640,12 @@ parse_makefile_for_sources(libxdiff_SOURCES "XDIFF_OBJS") list(TRANSFORM libxdiff_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") add_library(xdiff STATIC ${libxdiff_SOURCES}) +#reftable +parse_makefile_for_sources(reftable_SOURCES "REFTABLE_OBJS") + +list(TRANSFORM reftable_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") +add_library(reftable STATIC ${reftable_SOURCES}) + if(WIN32) if(NOT MSVC)#use windres when compiling with gcc and clang add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/git.res
@@ -662,7 +668,7 @@ endif() #link all required libraries to common-main add_library(common-main OBJECT ${CMAKE_SOURCE_DIR}/common-main.c) -target_link_libraries(common-main libgit xdiff ${ZLIB_LIBRARIES}) +target_link_libraries(common-main libgit xdiff reftable ${ZLIB_LIBRARIES}) if(Intl_FOUND) target_link_libraries(common-main ${Intl_LIBRARIES}) endif()
@@ -902,11 +908,15 @@ if(BUILD_TESTING) add_executable(test-fake-ssh ${CMAKE_SOURCE_DIR}/t/helper/test-fake-ssh.c) target_link_libraries(test-fake-ssh common-main) +#reftable-tests +parse_makefile_for_sources(test-reftable_SOURCES "REFTABLE_TEST_OBJS") +list(TRANSFORM test-reftable_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/") + #test-tool parse_makefile_for_sources(test-tool_SOURCES "TEST_BUILTINS_OBJS") list(TRANSFORM test-tool_SOURCES PREPEND "${CMAKE_SOURCE_DIR}/t/helper/") -add_executable(test-tool ${CMAKE_SOURCE_DIR}/t/helper/test-tool.c ${test-tool_SOURCES}) +add_executable(test-tool ${CMAKE_SOURCE_DIR}/t/helper/test-tool.c ${test-tool_SOURCES} ${test-reftable_SOURCES}) target_link_libraries(test-tool common-main) set_target_properties(test-fake-ssh test-tool
contrib/buildsystems/Generators/Vcxproj.pm +10 -1
@@ -77,7 +77,7 @@ sub createProject { my $libs_release = "\n "; my $libs_debug = "\n "; if (!$static_library) { - $libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); + $libs_release = join(";", sort(grep /^(?!libgit\.lib|xdiff\/lib\.lib|vcs-svn\/lib\.lib|reftable\/libreftable\.lib)/, @{$$build_structure{"$prefix${name}_LIBS"}})); $libs_debug = $libs_release; $libs_debug =~ s/zlib\.lib/zlibd\.lib/g; $libs_debug =~ s/libexpat\.lib/libexpatd\.lib/g;
@@ -232,6 +232,7 @@ EOM EOM if (!$static_library || $target =~ 'vcs-svn' || $target =~ 'xdiff') { my $uuid_libgit = $$build_structure{"LIBS_libgit_GUID"}; + my $uuid_libreftable = $$build_structure{"LIBS_reftable/libreftable_GUID"}; my $uuid_xdiff_lib = $$build_structure{"LIBS_xdiff/lib_GUID"}; print F << "EOM";
@@ -241,6 +242,14 @@ EOM <ReferenceOutputAssembly>false</ReferenceOutputAssembly> </ProjectReference> EOM + if (!($name =~ /xdiff|libreftable/)) { + print F << "EOM"; + <ProjectReference Include="$cdup\\reftable\\libreftable\\libreftable.vcxproj"> + <Project>$uuid_libreftable</Project> + <ReferenceOutputAssembly>false</ReferenceOutputAssembly> + </ProjectReference> +EOM + } if (!($name =~ 'xdiff')) { print F << "EOM"; <ProjectReference Include="$cdup\\xdiff\\lib\\xdiff_lib.vcxproj">
reftable/basics.c new file +128 -0
128 lines not on this page · raw message
reftable/basics.h new file +60 -0
60 lines not on this page · raw message
reftable/basics_test.c new file +98 -0
98 lines not on this page · raw message
reftable/publicbasics.c new file +65 -0
65 lines not on this page · raw message
reftable/reftable-malloc.h new file +18 -0
@@ -0,0 +1,18 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_H +#define REFTABLE_H + +#include <stddef.h> + +/* Overrides the functions to use for memory management. */ +void reftable_set_alloc (void *(*malloc)(size_t ), +void *(*realloc)(void *,size_t ),void (*free)(void *)); + +#endif
reftable/reftable-tests.h new file +22 -0
@@ -0,0 +1,22 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_TESTS_H +#define REFTABLE_TESTS_H + +int basics_test_main (int argc,const char **argv); +int block_test_main (int argc,const char **argv); +int merged_test_main (int argc,const char **argv); +int record_test_main (int argc,const char **argv); +int refname_test_main (int argc,const char **argv); +int reftable_test_main (int argc,const char **argv); +int stack_test_main (int argc,const char **argv); +int tree_test_main (int argc,const char **argv); +int reftable_dump_main (int argc,char *const *argv); + +#endif
reftable/system.h new file +32 -0
@@ -0,0 +1,32 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef SYSTEM_H +#define SYSTEM_H + +/* This header glues the reftable library to the rest of Git */ + +#include "git-compat-util.h" +#include "strbuf.h" +#include "hash.h" /* hash ID, sizes.*/ +#include "dir.h" /* remove_dir_recursively, for tests.*/ + +#include <zlib.h> + +#ifdef NO_UNCOMPRESS2 +/* +* This is uncompress2, which is only available in zlib >= 1.2.9 +* (released as of early 2017) +*/ +int uncompress2 (Bytef *dest,uLongf *destLen,const Bytef *source, +uLong *sourceLen); +#endif + +int hash_size (uint32_t id); + +#endif
reftable/test_framework.c new file +23 -0
@@ -0,0 +1,23 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "system.h" +#include "test_framework.h" + +#include "basics.h" + +void set_test_hash (uint8_t *p,int i) +{ +memset (p, (uint8_t )i,hash_size (GIT_SHA1_FORMAT_ID )); +} + +ssize_t strbuf_add_void (void *b,const void *data,size_t sz) +{ +strbuf_add (b, data, sz); +return sz; +}
reftable/test_framework.h new file +53 -0
53 lines not on this page · raw message
t/helper/test-reftable.c new file +9 -0
@@ -0,0 +1,9 @@ +#include "reftable/reftable-tests.h" +#include "test-tool.h" + +int cmd__reftable (int argc,const char **argv) +{ +basics_test_main (argc, argv); + +return 0 ; +}
t/helper/test-tool.c +2 -1
@@ -53,13 +53,14 @@ static struct test_cmd cmds[] = { { "pcre2-config", cmd__pcre2_config }, { "pkt-line", cmd__pkt_line }, { "prio-queue", cmd__prio_queue }, - { "proc-receive", cmd__proc_receive}, + { "proc-receive", cmd__proc_receive }, { "progress", cmd__progress }, { "reach", cmd__reach }, { "read-cache", cmd__read_cache }, { "read-graph", cmd__read_graph }, { "read-midx", cmd__read_midx }, { "ref-store", cmd__ref_store }, + { "reftable", cmd__reftable }, { "regex", cmd__regex }, { "repository", cmd__repository }, { "revision-walking", cmd__revision_walking },
t/helper/test-tool.h +1 -0
@@ -49,6 +49,7 @@ int cmd__read_cache(int argc, const char **argv);int cmd__read_graph (int argc,const char **argv);int cmd__read_midx (int argc,const char **argv);int cmd__ref_store (int argc,const char **argv); +int cmd__reftable (int argc,const char **argv);int cmd__regex (int argc,const char **argv);int cmd__repository (int argc,const char **argv);int cmd__revision_walking (int argc,const char **argv);
t/t0032-reftable-unittest.sh new file +15 -0
@@ -0,0 +1,15 @@ +#!/bin/sh +# +# Copyright (c) 2020 Google LLC +# + +test_description='reftable unittests' + +. ./test-lib.sh + +test_expect_success 'unittests' ' + TMPDIR=$(pwd) && export TMPDIR && + test-tool reftable +' + +test_done
Signature · 1 line
gitgitgadget
PATCH v4 5/19 reftable: add blocksource, an abstraction for random access reads
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The reftable format is usually used with files for storage. However, we abstract away this using the blocksource data structure. This has two advantages:
* log blocks are zlib compressed, and handling them is simplified if we can discard byte segments from within the block layer.
* for unittests, it is useful to read and write in-memory. The blocksource allows us to abstract the data away from on-disk files.
--- Makefile | 1 + reftable/blocksource.c | 148 ++++++++++++++++++++++++++++++++ reftable/blocksource.h | 22 +++++ reftable/reftable-blocksource.h | 49 +++++++++++ 4 files changed, 220 insertions(+) create mode 100644 reftable/blocksource.c create mode 100644 reftable/blocksource.h create mode 100644 reftable/reftable-blocksource.h
Makefile +1 -0
@@ -2431,6 +2431,7 @@ xdiff-objs: $(XDIFF_OBJS) REFTABLE_OBJS += reftable/basics.o REFTABLE_OBJS += reftable/error.o +REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o REFTABLE_TEST_OBJS += reftable/test_framework.o
reftable/blocksource.c new file +148 -0
148 lines not on this page · raw message
reftable/blocksource.h new file +22 -0
@@ -0,0 +1,22 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef BLOCKSOURCE_H +#define BLOCKSOURCE_H + +#include "system.h" + +struct reftable_block_source ; + +/* Create an in-memory block source for reading reftables */ +void block_source_from_strbuf (struct reftable_block_source *bs, +struct strbuf *buf); + +struct reftable_block_source malloc_block_source (void ); + +#endif
reftable/reftable-blocksource.h new file +49 -0
49 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v4 6/19 reftable: (de)serialization for the polymorphic record type.
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The reftable format is structured as a sequence of blocks, and each block contains a sequence of prefix-compressed key-value records. There are 4 types of records, and they have similarities in how they must be handled. This is achieved by introducing a polymorphic 'record' type that encapsulates ref, log, index and object records.
--- Makefile | 2 + reftable/constants.h | 21 + reftable/record.c | 1212 ++++++++++++++++++++++++++++++++++++ reftable/record.h | 139 +++++ reftable/record_test.c | 412 ++++++++++++ reftable/reftable-record.h | 114 ++++ t/helper/test-reftable.c | 2 +- 7 files changed, 1901 insertions(+), 1 deletion(-) create mode 100644 reftable/constants.h create mode 100644 reftable/record.c create mode 100644 reftable/record.h create mode 100644 reftable/record_test.c create mode 100644 reftable/reftable-record.h
Makefile +2 -0
@@ -2433,7 +2433,9 @@ REFTABLE_OBJS += reftable/basics.o REFTABLE_OBJS += reftable/error.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o +REFTABLE_OBJS += reftable/record.o +REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o REFTABLE_TEST_OBJS += reftable/basics_test.o
reftable/constants.h new file +21 -0
@@ -0,0 +1,21 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef CONSTANTS_H +#define CONSTANTS_H + +#define BLOCK_TYPE_LOG 'g' +#define BLOCK_TYPE_INDEX 'i' +#define BLOCK_TYPE_REF 'r' +#define BLOCK_TYPE_OBJ 'o' +#define BLOCK_TYPE_ANY 0 + +#define MAX_RESTARTS ((1 << 16) - 1) +#define DEFAULT_BLOCK_SIZE 4096 + +#endif
reftable/record.c new file +1212 -0
1,212 lines not on this page · raw message
reftable/record.h new file +139 -0
139 lines not on this page · raw message
reftable/record_test.c new file +412 -0
412 lines not on this page · raw message
reftable/reftable-record.h new file +114 -0
114 lines not on this page · raw message
t/helper/test-reftable.c +1 -1
@@ -4,6 +4,6 @@int cmd__reftable (int argc,const char **argv) {basics_test_main (argc, argv); - +record_test_main (argc, argv);return 0 ; }
Signature · 1 line
gitgitgadget
PATCH v4 7/19 Provide zlib's uncompress2 from compat/zlib-compat.c
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This will be needed for reading reflog blocks in reftable.
--- Makefile | 7 +++ ci/lib.sh | 1 + compat/.gitattributes | 1 + compat/zlib-uncompress2.c | 95 +++++++++++++++++++++++++++++++++++++++ config.mak.uname | 5 +++ configure.ac | 13 ++++++ 6 files changed, 122 insertions(+) create mode 100644 compat/.gitattributes create mode 100644 compat/zlib-uncompress2.c
Makefile +7 -0
@@ -256,6 +256,8 @@ all:: # # Define NO_DEFLATE_BOUND if your zlib does not have deflateBound. # +# Define NO_UNCOMPRESS2 if your zlib does not have uncompress2. +# # Define NO_NORETURN if using buggy versions of gcc 4.6+ and profile feedback, # as the compiler can crash (http://gcc.gnu.org/bugzilla/show_bug.cgi?id=49299) #
@@ -1718,6 +1720,11 @@ ifdef NO_DEFLATE_BOUND BASIC_CFLAGS += -DNO_DEFLATE_BOUND endif +ifdef NO_UNCOMPRESS2 + BASIC_CFLAGS += -DNO_UNCOMPRESS2 + REFTABLE_OBJS += compat/zlib-uncompress2.o +endif + ifdef NO_POSIX_GOODIES BASIC_CFLAGS += -DNO_POSIX_GOODIES endif
ci/lib.sh +1 -0
@@ -224,6 +224,7 @@ linux-gcc-default) ;; Linux32) CC=gcc + MAKEFLAGS="$MAKEFLAGS NO_UNCOMPRESS2=1" ;; linux-musl) CC=gcc
compat/.gitattributes new file +1 -0
@@ -0,0 +1 @@ +/zlib-uncompress2.c whitespace=-indent-with-non-tab,-trailing-space
compat/zlib-uncompress2.c new file +95 -0
95 lines not on this page · raw message
config.mak.uname +5 -0
@@ -258,6 +258,10 @@ ifeq ($(uname_S),FreeBSD) FILENO_IS_A_MACRO = UnfortunatelyYes endif ifeq ($(uname_S),OpenBSD) + # Versions < 7.0 need compatibility layer + ifeq ($(shell expr "$(uname_R)" : "[1-6]\."),2) + NO_UNCOMPRESS2 = UnfortunatelyYes + endif NO_STRCASESTR = YesPlease NO_MEMMEM = YesPlease USE_ST_TIMESPEC = YesPlease
@@ -513,6 +517,7 @@ ifeq ($(uname_S),Interix) endif endif ifeq ($(uname_S),Minix) + NO_UNCOMPRESS2 = YesPlease NO_IPV6 = YesPlease NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease NO_NSEC = YesPlease
configure.ac +13 -0
@@ -672,9 +672,22 @@ AC_LINK_IFELSE([ZLIBTEST_SRC], NO_DEFLATE_BOUND=yes]) LIBS="$old_LIBS" +AC_DEFUN([ZLIBTEST_UNCOMPRESS2_SRC], [ +AC_LANG_PROGRAM([#include <zlib.h>], + [uncompress2(NULL,NULL,NULL,NULL);])]) +AC_MSG_CHECKING([for uncompress2 in -lz]) +old_LIBS="$LIBS" +LIBS="$LIBS -lz" +AC_LINK_IFELSE([ZLIBTEST_UNCOMPRESS2_SRC], + [AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no]) + NO_UNCOMPRESS2=yes]) +LIBS="$old_LIBS" + GIT_UNSTASH_FLAGS($ZLIB_PATH) GIT_CONF_SUBST([NO_DEFLATE_BOUND]) +GIT_CONF_SUBST([NO_UNCOMPRESS2]) # # Define NEEDS_SOCKET if linking with libc is not enough (SunOS,
Signature · 1 line
gitgitgadget
PATCH v4 8/19 reftable: reading/writing blocks
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The reftable format is structured as a sequence of block. Within a block, records are prefix compressed, with an index of offsets for fully expand keys to enable binary search within blocks.
This commit provides the logic to read and write these blocks.
--- Makefile | 2 + reftable/block.c | 437 +++++++++++++++++++++++++++++++++++++++ reftable/block.h | 127 ++++++++++++ reftable/block_test.c | 120 +++++++++++ t/helper/test-reftable.c | 1 + 5 files changed, 687 insertions(+) create mode 100644 reftable/block.c create mode 100644 reftable/block.h create mode 100644 reftable/block_test.c
Makefile +2 -0
@@ -2438,10 +2438,12 @@ xdiff-objs: $(XDIFF_OBJS) REFTABLE_OBJS += reftable/basics.o REFTABLE_OBJS += reftable/error.o +REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o REFTABLE_OBJS += reftable/record.o +REFTABLE_TEST_OBJS += reftable/block_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o REFTABLE_TEST_OBJS += reftable/basics_test.o
reftable/block.c new file +437 -0
437 lines not on this page · raw message
reftable/block.h new file +127 -0
127 lines not on this page · raw message
reftable/block_test.c new file +120 -0
120 lines not on this page · raw message
t/helper/test-reftable.c +1 -0
@@ -4,6 +4,7 @@int cmd__reftable (int argc,const char **argv) {basics_test_main (argc, argv); +block_test_main (argc, argv);record_test_main (argc, argv);return 0 ; }
Signature · 1 line
gitgitgadget
PATCH v4 9/19 reftable: a generic binary tree implementation
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The reftable format includes support for an (OID => ref) map. This map can speed up visibility and reachability checks. In particular, various operations along the fetch/push path within Gerrit have ben sped up by using this structure.
The map is constructed with help of a binary tree. Object IDs are hashes, so they are uniformly distributed. Hence, the tree does not attempt forced rebalancing.
--- Makefile | 4 ++- reftable/tree.c | 63 ++++++++++++++++++++++++++++++++++++++++ reftable/tree.h | 34 ++++++++++++++++++++++ reftable/tree_test.c | 61 ++++++++++++++++++++++++++++++++++++++ t/helper/test-reftable.c | 1 + 5 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 reftable/tree.c create mode 100644 reftable/tree.h create mode 100644 reftable/tree_test.c
Makefile +3 -1
@@ -2442,11 +2442,13 @@ REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o REFTABLE_OBJS += reftable/record.o +REFTABLE_OBJS += reftable/tree.o +REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o -REFTABLE_TEST_OBJS += reftable/basics_test.o +REFTABLE_TEST_OBJS += reftable/tree_test.o TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS))
reftable/tree.c new file +63 -0
63 lines not on this page · raw message
reftable/tree.h new file +34 -0
@@ -0,0 +1,34 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef TREE_H +#define TREE_H + +/* tree_node is a generic binary search tree. */ +struct tree_node { +void *key; +struct tree_node *left, *right; +}; + +/* looks for `key` in `rootp` using `compare` as comparison function. If insert +* is set, insert the key if it's not found. Else, return NULL. +*/ +struct tree_node *tree_search (void *key,struct tree_node **rootp, +int (*compare)(const void *,const void *), +int insert); + +/* performs an infix walk of the tree. */ +void infix_walk (struct tree_node *t,void (*action)(void *arg,void *key), +void *arg); + +/* +* deallocates the tree nodes recursively. Keys should be deallocated separately +* by walking over the tree. */ +void tree_free (struct tree_node *t); + +#endif
reftable/tree_test.c new file +61 -0
61 lines not on this page · raw message
t/helper/test-reftable.c +1 -0
@@ -6,5 +6,6 @@ int cmd__reftable(int argc, const char **argv)basics_test_main (argc, argv);block_test_main (argc, argv);record_test_main (argc, argv); +tree_test_main (argc, argv);return 0 ; }
Signature · 1 line
gitgitgadget
PATCH v4 10/19 reftable: write reftable files
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
--- Makefile | 1 + reftable/reftable-writer.h | 148 ++++++++ reftable/writer.c | 690 +++++++++++++++++++++++++++++++++++++ reftable/writer.h | 50 +++ 4 files changed, 889 insertions(+) create mode 100644 reftable/reftable-writer.h create mode 100644 reftable/writer.c create mode 100644 reftable/writer.h
Makefile +1 -0
@@ -2443,6 +2443,7 @@ REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o REFTABLE_OBJS += reftable/record.o REFTABLE_OBJS += reftable/tree.o +REFTABLE_OBJS += reftable/writer.o REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o
reftable/reftable-writer.h new file +148 -0
148 lines not on this page · raw message
reftable/writer.c new file +690 -0
690 lines not on this page · raw message
reftable/writer.h new file +50 -0
50 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v4 11/19 reftable: generic interface to tables
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
--- Makefile | 3 + reftable/generic.c | 169 +++++++++++++++++++++++++++++++++++ reftable/generic.h | 32 +++++++ reftable/reftable-generic.h | 47 ++++++++++ reftable/reftable-iterator.h | 39 ++++++++ reftable/reftable.c | 115 ++++++++++++++++++++++++ 6 files changed, 405 insertions(+) create mode 100644 reftable/generic.c create mode 100644 reftable/generic.h create mode 100644 reftable/reftable-generic.h create mode 100644 reftable/reftable-iterator.h create mode 100644 reftable/reftable.c
Makefile +3 -0
@@ -2442,6 +2442,9 @@ REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/publicbasics.o REFTABLE_OBJS += reftable/record.o +REFTABLE_OBJS += reftable/refname.o +REFTABLE_OBJS += reftable/generic.o +REFTABLE_OBJS += reftable/stack.o REFTABLE_OBJS += reftable/tree.o REFTABLE_OBJS += reftable/writer.o
reftable/generic.c new file +169 -0
169 lines not on this page · raw message
reftable/generic.h new file +32 -0
@@ -0,0 +1,32 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef GENERIC_H +#define GENERIC_H + +#include "record.h" +#include "reftable-generic.h" + +/* generic interface to reftables */ +struct reftable_table_vtable { +int (*seek_record)(void *tab,struct reftable_iterator *it, +struct reftable_record *); +uint32_t (*hash_id)(void *tab); +uint64_t (*min_update_index)(void *tab); +uint64_t (*max_update_index)(void *tab); +}; + +struct reftable_iterator_vtable { +int (*next)(void *iter_arg,struct reftable_record *rec); +void (*close)(void *iter_arg); +}; + +void iterator_set_empty (struct reftable_iterator *it); +int iterator_next (struct reftable_iterator *it,struct reftable_record *rec); + +#endif
reftable/reftable-generic.h new file +47 -0
47 lines not on this page · raw message
reftable/reftable-iterator.h new file +39 -0
@@ -0,0 +1,39 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_ITERATOR_H +#define REFTABLE_ITERATOR_H + +#include "reftable-record.h" + +struct reftable_iterator_vtable ; + +/* iterator is the generic interface for walking over data stored in a +* reftable. +*/ +struct reftable_iterator { +struct reftable_iterator_vtable *ops; +void *iter_arg; +}; + +/* reads the next reftable_ref_record. Returns < 0 for error, 0 for OK and > 0: +* end of iteration. +*/ +int reftable_iterator_next_ref (struct reftable_iterator *it, +struct reftable_ref_record *ref); + +/* reads the next reftable_log_record. Returns < 0 for error, 0 for OK and > 0: +* end of iteration. +*/ +int reftable_iterator_next_log (struct reftable_iterator *it, +struct reftable_log_record *log); + +/* releases resources associated with an iterator. */ +void reftable_iterator_destroy (struct reftable_iterator *it); + +#endif
reftable/reftable.c new file +115 -0
115 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v4 12/19 reftable: read reftable files
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This supports reading a single reftable file.
The commit introduces an abstract iterator type, which captures the usecases both of reading individual refs, and iterating over a segment of the ref namespace.
--- Makefile | 2 + reftable/iter.c | 194 +++++++++ reftable/iter.h | 69 ++++ reftable/reader.c | 801 +++++++++++++++++++++++++++++++++++++ reftable/reader.h | 64 +++ reftable/reftable-reader.h | 101 +++++ 6 files changed, 1231 insertions(+) create mode 100644 reftable/iter.c create mode 100644 reftable/iter.h create mode 100644 reftable/reader.c create mode 100644 reftable/reader.h create mode 100644 reftable/reftable-reader.h
Makefile +2 -0
@@ -2440,7 +2440,9 @@ REFTABLE_OBJS += reftable/basics.o REFTABLE_OBJS += reftable/error.o REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o +REFTABLE_OBJS += reftable/iter.o REFTABLE_OBJS += reftable/publicbasics.o +REFTABLE_OBJS += reftable/reader.o REFTABLE_OBJS += reftable/record.o REFTABLE_OBJS += reftable/refname.o REFTABLE_OBJS += reftable/generic.o
reftable/iter.c new file +194 -0
194 lines not on this page · raw message
reftable/iter.h new file +69 -0
69 lines not on this page · raw message
reftable/reader.c new file +801 -0
801 lines not on this page · raw message
reftable/reader.h new file +64 -0
64 lines not on this page · raw message
reftable/reftable-reader.h new file +101 -0
101 lines not on this page · raw message
Signature · 1 line
gitgitgadget
PATCH v4 13/19 reftable: reftable file level tests
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
With support for reading and writing files in place, we can construct files (in memory) and attempt to read them back.
Because some sections of the format are optional (eg. indices, log entries), we have to exercise this code using multiple sizes of input data
--- Makefile | 1 + reftable/readwrite_test.c | 652 ++++++++++++++++++++++++++++++++++++++ reftable/reftable-tests.h | 2 +- t/helper/test-reftable.c | 1 + 4 files changed, 655 insertions(+), 1 deletion(-) create mode 100644 reftable/readwrite_test.c
Makefile +1 -0
@@ -2453,6 +2453,7 @@ REFTABLE_OBJS += reftable/writer.o REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o REFTABLE_TEST_OBJS += reftable/record_test.o +REFTABLE_TEST_OBJS += reftable/readwrite_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o REFTABLE_TEST_OBJS += reftable/tree_test.o
reftable/readwrite_test.c new file +652 -0
652 lines not on this page · raw message
reftable/reftable-tests.h +1 -1
@@ -14,7 +14,7 @@ int block_test_main(int argc, const char **argv);int merged_test_main (int argc,const char **argv);int record_test_main (int argc,const char **argv);int refname_test_main (int argc,const char **argv); -int reftable_test_main (int argc,const char **argv); +int readwrite_test_main (int argc,const char **argv);int stack_test_main (int argc,const char **argv);int tree_test_main (int argc,const char **argv);int reftable_dump_main (int argc,char *const *argv);
t/helper/test-reftable.c +1 -0
@@ -6,6 +6,7 @@ int cmd__reftable(int argc, const char **argv)basics_test_main (argc, argv);block_test_main (argc, argv);record_test_main (argc, argv); +readwrite_test_main (argc, argv);tree_test_main (argc, argv);return 0 ; }
Signature · 1 line
gitgitgadget
PATCH v4 14/19 reftable: add a heap-based priority queue for reftable records
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This is needed to create a merged view multiple reftables
--- Makefile | 2 + reftable/pq.c | 105 ++++++++++++++++++++++++++++++++++++++ reftable/pq.h | 33 ++++++++++++ reftable/pq_test.c | 82 +++++++++++++++++++++++++++++ reftable/reftable-tests.h | 1 + t/helper/test-reftable.c | 1 + 6 files changed, 224 insertions(+) create mode 100644 reftable/pq.c create mode 100644 reftable/pq.h create mode 100644 reftable/pq_test.c
Makefile +2 -0
@@ -2442,6 +2442,7 @@ REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/iter.o REFTABLE_OBJS += reftable/publicbasics.o +REFTABLE_OBJS += reftable/pq.o REFTABLE_OBJS += reftable/reader.o REFTABLE_OBJS += reftable/record.o REFTABLE_OBJS += reftable/refname.o
@@ -2452,6 +2453,7 @@ REFTABLE_OBJS += reftable/writer.o REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o +REFTABLE_TEST_OBJS += reftable/pq_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/readwrite_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o
reftable/pq.c new file +105 -0
105 lines not on this page · raw message
reftable/pq.h new file +33 -0
@@ -0,0 +1,33 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef PQ_H +#define PQ_H + +#include "record.h" + +struct pq_entry { +int index; +struct reftable_record rec; +}; + +struct merged_iter_pqueue { +struct pq_entry *heap; +size_t len; +size_t cap; +}; + +struct pq_entry merged_iter_pqueue_top (struct merged_iter_pqueue pq); +int merged_iter_pqueue_is_empty (struct merged_iter_pqueue pq); +void merged_iter_pqueue_check (struct merged_iter_pqueue pq); +struct pq_entry merged_iter_pqueue_remove (struct merged_iter_pqueue *pq); +void merged_iter_pqueue_add (struct merged_iter_pqueue *pq,struct pq_entry e); +void merged_iter_pqueue_release (struct merged_iter_pqueue *pq); +int pq_less (struct pq_entry *a,struct pq_entry *b); + +#endif
reftable/pq_test.c new file +82 -0
82 lines not on this page · raw message
reftable/reftable-tests.h +1 -0
@@ -12,6 +12,7 @@ https://developers.google.com/open-source/licenses/bsdint basics_test_main (int argc,const char **argv);int block_test_main (int argc,const char **argv);int merged_test_main (int argc,const char **argv); +int pq_test_main (int argc,const char **argv);int record_test_main (int argc,const char **argv);int refname_test_main (int argc,const char **argv);int readwrite_test_main (int argc,const char **argv);
t/helper/test-reftable.c +1 -0
@@ -5,6 +5,7 @@ int cmd__reftable(int argc, const char **argv) {basics_test_main (argc, argv);block_test_main (argc, argv); +pq_test_main (argc, argv);record_test_main (argc, argv);readwrite_test_main (argc, argv);tree_test_main (argc, argv);
Signature · 1 line
gitgitgadget
PATCH v4 15/19 reftable: add merged table view
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This adds an abstract, read-only interface to the ref database.
This primitive is used to construct the read view of the ref database (the read view is constructed by merging several *.ref files). It also provides the mechanism to provide a unified view of the refs in the main repository and the per-worktree refs.
--- Makefile | 2 + reftable/merged.c | 362 ++++++++++++++++++++++++++++ reftable/merged.h | 38 +++ reftable/merged_test.c | 468 +++++++++++++++++++++++++++++++++++++ reftable/reftable-merged.h | 72 ++++++ t/helper/test-reftable.c | 1 + 6 files changed, 943 insertions(+) create mode 100644 reftable/merged.c create mode 100644 reftable/merged.h create mode 100644 reftable/merged_test.c create mode 100644 reftable/reftable-merged.h
Makefile +2 -0
@@ -2442,6 +2442,7 @@ REFTABLE_OBJS += reftable/block.o REFTABLE_OBJS += reftable/blocksource.o REFTABLE_OBJS += reftable/iter.o REFTABLE_OBJS += reftable/publicbasics.o +REFTABLE_OBJS += reftable/merged.o REFTABLE_OBJS += reftable/pq.o REFTABLE_OBJS += reftable/reader.o REFTABLE_OBJS += reftable/record.o
@@ -2453,6 +2454,7 @@ REFTABLE_OBJS += reftable/writer.o REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o +REFTABLE_TEST_OBJS += reftable/merged_test.o REFTABLE_TEST_OBJS += reftable/pq_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/readwrite_test.o
reftable/merged.c new file +362 -0
362 lines not on this page · raw message
reftable/merged.h new file +38 -0
@@ -0,0 +1,38 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef MERGED_H +#define MERGED_H + +#include "pq.h" + +struct reftable_merged_table { +struct reftable_table *stack; +size_t stack_len; +uint32_t hash_id; + +/* If unset, produce deletions. This is useful for compaction. For the +* full stack, deletions should be produced. */ +int suppress_deletions; + +uint64_t min; +uint64_t max; +}; + +struct merged_iter { +struct reftable_iterator *stack; +uint32_t hash_id; +size_t stack_len; +uint8_t typ; +int suppress_deletions; +struct merged_iter_pqueue pq; +}; + +void merged_table_release (struct reftable_merged_table *mt); + +#endif
reftable/merged_test.c new file +468 -0
468 lines not on this page · raw message
reftable/reftable-merged.h new file +72 -0
72 lines not on this page · raw message
t/helper/test-reftable.c +1 -0
@@ -5,6 +5,7 @@ int cmd__reftable(int argc, const char **argv) {basics_test_main (argc, argv);block_test_main (argc, argv); +merged_test_main (argc, argv);pq_test_main (argc, argv);record_test_main (argc, argv);readwrite_test_main (argc, argv);
Signature · 1 line
gitgitgadget
PATCH 1/1 reftable tests: use C syntax compatible with old xlc
Change code added in 1ae2b8cda84 (reftable: add merged table view, 2021-10-07) to be compatible with older versions of AIX's IBM xlc compiler. Version V12.1 of it (on gcc111.fsffrance.org) will hard error with:
"reftable/merged_test.c", line 211.19: 1506-196 (S) Initialization between types "char*" and "struct reftable_ref_record" is not allowed. "reftable/merged_test.c", line 212.19: 1506-196 (S) Initialization between types "unsigned long long" and "struct reftable_ref_record" is not allowed. "reftable/merged_test.c", line 213.19: 1506-196 (S) Initialization between types "enum {...}" and "struct reftable_ref_record" is not allowed. "reftable/merged_test.c", line 214.19: 1506-196 (S) Initialization between types "unsigned char*" and "struct reftable_ref_record" is not allowed. "reftable/merged_test.c", line 349.19: 1506-196 (S) Initialization between types "char*" and "struct reftable_log_record" is not allowed. "reftable/merged_test.c", line 350.19: 1506-196 (S) Initialization between types "unsigned long long" and "struct reftable_log_record" is not allowed. "reftable/merged_test.c", line 351.19: 1506-196 (S) Initialization between types "enum {...}" and "struct reftable_log_record" is not allowed.
Its newer V13.1.3 sibling (on gcc119.fsffrance.org, a AIX 7.2 box) will compile the pre-image without issues. Let's not make git's sources incompatible with this older AIX 7.1 compiler.
Perhaps there's a better way to do this, but just duplicating the earlier struct values declared earlier in these functions works, and is probably the least bad solution.
--- reftable/merged_test.c | 74 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 8 deletions(-)
reftable/merged_test.c +66 -8
@@ -206,12 +206,38 @@ static void test_merged(void) .value.val1 = hash1, }, }; - +/* +* We don't use { r2[0], r3[0], ... } for compatibility with +* older IBM xlc. +*/ struct reftable_ref_record want[] = { - r2[0 ], - r1[1 ], - r3[0 ], - r3[1 ], +/* Same as r2[0] */ + { + .refname ="a" , + .update_index =2 , + .value_type =REFTABLE_REF_DELETION , + }, +/* Same as r1[1] */ + { + + .refname ="b" , + .update_index =1 , + .value_type =REFTABLE_REF_VAL1 , + .value.val1 = hash1, + }, +/* Same as r3[0..1] */ + { + .refname ="c" , + .update_index =3 , + .value_type =REFTABLE_REF_VAL1 , + .value.val1 = hash2, + }, + { + .refname ="d" , + .update_index =3 , + .value_type =REFTABLE_REF_VAL1 , + .value.val1 = hash1, + }, };struct reftable_ref_record *refs[] = { r1, r2, r3 };
@@ -345,10 +371,42 @@ static void test_merged_logs(void) .value_type = REFTABLE_LOG_DELETION, }, }; +/* +* We don't use { r2[0], r3[0], ... } for compatibility with +* older IBM xlc. +*/ struct reftable_log_record want[] = { - r2[0 ], - r3[0 ], - r1[1 ], +/* Same as r2[0] */ + { + .refname ="a" , + .update_index =3 , + .value_type =REFTABLE_LOG_UPDATE , + .value.update = { + .new_hash = hash3, + .name ="jane doe" , + .email ="jane@invalid" , + .message ="message3" , + } + }, +/* Same as r3[0] */ + { + .refname ="a" , + .update_index =2 , + .value_type =REFTABLE_LOG_DELETION , + }, +/* Same as r1[1] */ + { + .refname ="a" , + .update_index =1 , + .value_type =REFTABLE_LOG_UPDATE , + .value.update = { + .old_hash = hash1, + .new_hash = hash2, + .name ="jane doe" , + .email ="jane@invalid" , + .message ="message1" , + } + }, };struct reftable_log_record *logs[] = { r1, r2, r3 };
Signature · 1 line
2.35.0.rc0.850.gcc6bf5af6b1
Re: [PATCH] reftable tests: use C syntax compatible with old xlc
Quoted 5 lines · Ævar Arnfjörð Bjarmason
Change code added in
1ae2b8cda84(reftable: add merged table view, 2021-10-07) to be compatible with older versions of AIX's IBM xlc compiler. Version V12.1 of it (on gcc111.fsffrance.org) will hard error with:
Quoted 7 lines
"
reftable/merged_test.c", line 211.19: 1506-196 (S) Initialization between types "char*" and "structreftable_ref_record" is not allowed. "reftable/merged_test.c", line 212.19: 1506-196 (S) Initialization between types "unsigned long long" and "structreftable_ref_record" is not allowed. "reftable/merged_test.c", line 213.19: 1506-196 (S) Initialization between types "enum {...}" and "structreftable_ref_record" is not allowed. "reftable/merged_test.c", line 214.19: 1506-196 (S) Initialization between types "unsigned char*" and "structreftable_ref_record" is not allowed. "reftable/merged_test.c", line 349.19: 1506-196 (S) Initialization between types "char*" and "structreftable_log_record" is not allowed. "reftable/merged_test.c", line 350.19: 1506-196 (S) Initialization between types "unsigned long long" and "structreftable_log_record" is not allowed. "reftable/merged_test.c", line 351.19: 1506-196 (S) Initialization between types "enum {...}" and "structreftable_log_record" is not allowed.
Weird. What C standard does xlc implement?
Quoted 3 lines
Perhaps there's a better way to do this, but just duplicating the earlier struct values declared earlier in these functions works, and is probably the least bad solution.
I'd rather not duplicate anything. Can't you do
struct foobar *want = { &r[0], &r[2] .. }
Signature · 11 lines
Han-Wen Nienhuys - Google Munich I work 80%. Don't expect answers from me on Fridays. -- Google Germany GmbH, Erika-Mann-Strasse 33, 80636 Munich Registergericht und -nummer: Hamburg, HRB 86891 Sitz der Gesellschaft: Hamburg Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Re: [PATCH] reftable tests: use C syntax compatible with old xlc
Quoted 15 lines · Han-Wen Nienhuys
Quoted 5 lines · Ævar Arnfjörð Bjarmason
Change code added in
1ae2b8cda84(reftable: add merged table view, 2021-10-07) to be compatible with older versions of AIX's IBM xlc compiler. Version V12.1 of it (on gcc111.fsffrance.org) will hard error with:Quoted 7 lines
"
reftable/merged_test.c", line 211.19: 1506-196 (S) Initialization between types "char*" and "structreftable_ref_record" is not allowed. "reftable/merged_test.c", line 212.19: 1506-196 (S) Initialization between types "unsigned long long" and "structreftable_ref_record" is not allowed. "reftable/merged_test.c", line 213.19: 1506-196 (S) Initialization between types "enum {...}" and "structreftable_ref_record" is not allowed. "reftable/merged_test.c", line 214.19: 1506-196 (S) Initialization between types "unsigned char*" and "structreftable_ref_record" is not allowed. "reftable/merged_test.c", line 349.19: 1506-196 (S) Initialization between types "char*" and "structreftable_log_record" is not allowed. "reftable/merged_test.c", line 350.19: 1506-196 (S) Initialization between types "unsigned long long" and "structreftable_log_record" is not allowed. "reftable/merged_test.c", line 351.19: 1506-196 (S) Initialization between types "enum {...}" and "structreftable_log_record" is not allowed.
Quoted 1 line
Weird. What C standard does xlc implement?
I don't know. Your guess (and searching through IBM's website) is as good as mine.
AFAICT it mostly implements the C99 semantics, but doesn't grok the interpolation of structs-within-structs
Quoted 7 lines
Quoted 3 lines
Perhaps there's a better way to do this, but just duplicating the earlier struct values declared earlier in these functions works, and is probably the least bad solution.
I'd rather not duplicate anything. Can't you do
struct foobar *want = { &r[0], &r[2] .. }
Maybe I'm just not understanding what you mean, but this:
struct reftable_ref_record *want = {
&r2[0],
&r1[1],
&r3[0],
&r3[1],
};Gives us the predictable compiler error on gcc/clang, nevermind xlc. Turn that into:
struct reftable_ref_record want[] = {
&r2[0],
&r1[1],
&r3[0],
&r3[1],
};And you can get gcc/clang to emulate what that xlc version (mis)parses that as. I.e. it takes it as a reference to the nth element in that struct.
Re: [PATCH] reftable tests: use C syntax compatible with old xlc
In reply to Han-Wen NienhuysQuoted 10 lines · Han-Wen Nienhuys
Quoted 6 lines · Ævar Arnfjörð Bjarmason
Change code added in
1ae2b8cda84(reftable: add merged table view, 2021-10-07) to be compatible with older versions of AIX's IBM xlc compiler. Version V12.1 of it (on gcc111.fsffrance.org) will hard error with:"
reftable/merged_test.c", line 211.19: 1506-196 (S) Initialization between types "char*" and "structreftable_ref_record" is not allowed....
Weird. What C standard does xlc implement?
Interesting. It is complaining about this thing:
struct reftable_ref_record r2[] = { {
.refname = "a",
.update_index = 2,
.value_type = REFTABLE_REF_DELETION,
} };
...
struct reftable_ref_record want[] = {
r2[0],
r1[1],
r3[0],
r3[1],
};and somehow assuming that r2[0] corresponds to the first member of a "struct reftable_ref_record" structure, which is "char *refname". Of course, you cannot initialize a character pointer with a whole struct, which is *not* what the code wants to do. I would understand if it were written incorrectly like so
BAD: struct reftable_ref_record want[] = {
BAD: { r2[0] },
BAD: { r1[1] },
BAD: { r3[0] },
BAD: { r3[1] },
BAD: };
but that is not what we fed the compiler. But apparently the compiler is confused.
I wonder if it helps to use designated initializer on the latter, i.e.
struct reftable_ref_record want[] = {
[0] = r2[0],
[1] = r1[1],
[2] = r3[0],
[3] = r3[1],
};PATCH v4 16/19 reftable: implement refname validation
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
The packed/loose format has restrictions on refnames: a and a/b cannot coexist. This limitation does not apply to reftable per se, but must be maintained for interoperability. This code adds validation routines to abort transactions that are trying to add invalid names.
--- Makefile | 1 + reftable/refname.c | 209 +++++++++++++++++++++++++++++++++++++++ reftable/refname.h | 29 ++++++ reftable/refname_test.c | 102 +++++++++++++++++++ t/helper/test-reftable.c | 1 + 5 files changed, 342 insertions(+) create mode 100644 reftable/refname.c create mode 100644 reftable/refname.h create mode 100644 reftable/refname_test.c
Makefile +1 -0
@@ -2458,6 +2458,7 @@ REFTABLE_TEST_OBJS += reftable/merged_test.o REFTABLE_TEST_OBJS += reftable/pq_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/readwrite_test.o +REFTABLE_TEST_OBJS += reftable/refname_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o REFTABLE_TEST_OBJS += reftable/tree_test.o
reftable/refname.c new file +209 -0
209 lines not on this page · raw message
reftable/refname.h new file +29 -0
@@ -0,0 +1,29 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ +#ifndef REFNAME_H +#define REFNAME_H + +#include "reftable-record.h" +#include "reftable-generic.h" + +struct modification { +struct reftable_table tab; + +char **add; +size_t add_len; + +char **del; +size_t del_len; +}; + +int validate_ref_record_addition (struct reftable_table tab, +struct reftable_ref_record *recs,size_t sz); + +int modification_validate (struct modification *mod); + +#endif
reftable/refname_test.c new file +102 -0
102 lines not on this page · raw message
t/helper/test-reftable.c +1 -0
@@ -8,6 +8,7 @@ int cmd__reftable(int argc, const char **argv)merged_test_main (argc, argv);pq_test_main (argc, argv);record_test_main (argc, argv); +refname_test_main (argc, argv);readwrite_test_main (argc, argv);tree_test_main (argc, argv);return 0 ;
Signature · 1 line
gitgitgadget
PATCH v4 17/19 reftable: implement stack, a mutable database of reftable files.
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
--- Makefile | 1 + reftable/reftable-stack.h | 128 ++++ reftable/stack.c | 1396 +++++++++++++++++++++++++++++++++++++ reftable/stack.h | 41 ++ reftable/stack_test.c | 953 +++++++++++++++++++++++++ t/helper/test-reftable.c | 1 + 6 files changed, 2520 insertions(+) create mode 100644 reftable/reftable-stack.h create mode 100644 reftable/stack.c create mode 100644 reftable/stack.h create mode 100644 reftable/stack_test.c
Makefile +1 -0
@@ -2459,6 +2459,7 @@ REFTABLE_TEST_OBJS += reftable/pq_test.o REFTABLE_TEST_OBJS += reftable/record_test.o REFTABLE_TEST_OBJS += reftable/readwrite_test.o REFTABLE_TEST_OBJS += reftable/refname_test.o +REFTABLE_TEST_OBJS += reftable/stack_test.o REFTABLE_TEST_OBJS += reftable/test_framework.o REFTABLE_TEST_OBJS += reftable/tree_test.o
reftable/reftable-stack.h new file +128 -0
128 lines not on this page · raw message
reftable/stack.c new file +1396 -0
1,396 lines not on this page · raw message
reftable/stack.h new file +41 -0
41 lines not on this page · raw message
reftable/stack_test.c new file +953 -0
953 lines not on this page · raw message
t/helper/test-reftable.c +1 -0
@@ -10,6 +10,7 @@ int cmd__reftable(int argc, const char **argv)record_test_main (argc, argv);refname_test_main (argc, argv);readwrite_test_main (argc, argv); +stack_test_main (argc, argv);tree_test_main (argc, argv);return 0 ; }
Signature · 1 line
gitgitgadget
PATCH v4 18/19 reftable: add dump utility
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
provide a command-line utility for inspecting individual tables, and inspecting a complete ref database
--- reftable/dump.c | 107 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 reftable/dump.c
reftable/dump.c new file +107 -0
@@ -0,0 +1,107 @@ +/* +Copyright 2020 Google LLC + +Use of this source code is governed by a BSD-style +license that can be found in the LICENSE file or at +https://developers.google.com/open-source/licenses/bsd +*/ + +#include "git-compat-util.h" +#include "hash.h" + +#include "reftable-blocksource.h" +#include "reftable-error.h" +#include "reftable-merged.h" +#include "reftable-record.h" +#include "reftable-tests.h" +#include "reftable-writer.h" +#include "reftable-iterator.h" +#include "reftable-reader.h" +#include "reftable-stack.h" +#include "reftable-generic.h" + +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> + +static int compact_stack (const char *stackdir) +{ +struct reftable_stack *stack =NULL ; +struct reftable_write_options cfg = {0 }; + +int err =reftable_new_stack (&stack, stackdir, cfg); +if (err <0 ) + goto done; + + err =reftable_stack_compact_all (stack,NULL ); +if (err <0 ) + goto done; +done: +if (stack) { +reftable_stack_destroy (stack); + } +return err; +} + +static void print_help (void ) +{ +printf ("usage: dump [-cst] arg\n\n" +"options: \n" +" -c compact\n" +" -t dump table\n" +" -s dump stack\n" +" -6 sha256 hash format\n" +" -h this help\n" +"\n" ); +} + +int reftable_dump_main (int argc,char *const *argv) +{ +int err =0 ; +int opt_dump_table =0 ; +int opt_dump_stack =0 ; +int opt_compact =0 ; +uint32_t opt_hash_id =GIT_SHA1_FORMAT_ID ; +const char *arg =NULL , *argv0 = argv[0 ]; + +for (; argc >1 ; argv++, argc--) +if (*argv[1 ] !='-' ) +break ; +else if (!strcmp ("-t" , argv[1 ])) + opt_dump_table =1 ; +else if (!strcmp ("-6" , argv[1 ])) + opt_hash_id =GIT_SHA256_FORMAT_ID ; +else if (!strcmp ("-s" , argv[1 ])) + opt_dump_stack =1 ; +else if (!strcmp ("-c" , argv[1 ])) + opt_compact =1 ; +else if (!strcmp ("-?" , argv[1 ]) || !strcmp ("-h" , argv[1 ])) { +print_help (); +return 2 ; + } + +if (argc !=2 ) { +fprintf (stderr,"need argument\n" ); +print_help (); +return 2 ; + } + + arg = argv[1 ]; + +if (opt_dump_table) { + err =reftable_reader_print_file (arg); + }else if (opt_dump_stack) { + err =reftable_stack_print_directory (arg, opt_hash_id); + }else if (opt_compact) { + err =compact_stack (arg); + } + +if (err <0 ) { +fprintf (stderr,"%s: %s: %s\n" , argv0, arg, +reftable_error_str (err)); +return 1 ; + } +return 0 ; +}
Signature · 1 line
gitgitgadget
PATCH v4 19/19 Add "test-tool dump-reftable" command.
In reply to Han-Wen Nienhuys via GitGitGadgetFrom: Han-Wen Nienhuys <[email protected]>
This command dumps individual tables or a stack of of tables.
--- Makefile | 1 + t/helper/test-reftable.c | 5 +++++ t/helper/test-tool.c | 1 + t/helper/test-tool.h | 1 + 4 files changed, 8 insertions(+)
Makefile +1 -0
@@ -2454,6 +2454,7 @@ REFTABLE_OBJS += reftable/writer.o REFTABLE_TEST_OBJS += reftable/basics_test.o REFTABLE_TEST_OBJS += reftable/block_test.o +REFTABLE_TEST_OBJS += reftable/dump.o REFTABLE_TEST_OBJS += reftable/merged_test.o REFTABLE_TEST_OBJS += reftable/pq_test.o REFTABLE_TEST_OBJS += reftable/record_test.o
t/helper/test-reftable.c +5 -0
@@ -14,3 +14,8 @@ int cmd__reftable(int argc, const char **argv)tree_test_main (argc, argv);return 0 ; } + +int cmd__dump_reftable (int argc,const char **argv) +{ +return reftable_dump_main (argc, (char *const *)argv); +}
t/helper/test-tool.c +1 -0
@@ -61,6 +61,7 @@ static struct test_cmd cmds[] = { { "read-midx", cmd__read_midx }, { "ref-store", cmd__ref_store }, { "reftable", cmd__reftable }, + { "dump-reftable", cmd__dump_reftable }, { "regex", cmd__regex }, { "repository", cmd__repository }, { "revision-walking", cmd__revision_walking },
t/helper/test-tool.h +1 -0
@@ -19,6 +19,7 @@ int cmd__dump_cache_tree(int argc, const char **argv);int cmd__dump_fsmonitor (int argc,const char **argv);int cmd__dump_split_index (int argc,const char **argv);int cmd__dump_untracked_cache (int argc,const char **argv); +int cmd__dump_reftable (int argc,const char **argv);int cmd__example_decorate (int argc,const char **argv);int cmd__fast_rebase (int argc,const char **argv);int cmd__genrandom (int argc,const char **argv);
Signature · 1 line
gitgitgadget