Add 'ls-files --debug-json' to dump the index in json
v2 cover Add 'ls-files --debug-json' to dump the index in json
v2 main changes:
- --json is renamed --debug-json
- json field names now use '_' instead of '.' to be friendlier to some languages. I stick to underscore_name instead of camelCase because the former is closer to what we use
- extension location is printed, in case you need to decode the extension by yourself (previously only the size is printed)
- all extensions are printed in the same order they appear in the file (previously eoie and ieot are printed first because that's how we parse)
- resolve undo extension is reorganized a bit to be easier to read
- tests added. Example json files are in t/t3011
Interdiff
Documentation/git-ls-files.txt +5 -5
@@ -60,11 +60,6 @@ OPTIONS --stage:: Show staged contents' mode bits, object name and stage number in the output. ---json:: - Dump the entire index content in JSON format. This is for - debugging purposes and the JSON structure may change from time - to time. - --directory:: If a whole directory is classified as "other", show just its name (with a trailing slash) and not its whole contents.
@@ -167,6 +162,11 @@ a space) at the start of each line: possible for manual inspection; the exact format may change at any time. +--debug-json:: + Dump the entire index content in JSON format. This is for + debugging purposes. The JSON structure is subject to change. + Note that the strings are not always valid UTF-8. + --eol:: Show <eolinfo> and <eolattr> of files. <eolinfo> is the file content identification used by Git when
builtin/ls-files.c +11 -5
@@ -545,8 +545,6 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix) N_("show staged contents' object name in the output")),OPT_BOOL ('k' ,"killed" , &show_killed,N_ ("show files on the filesystem that need to be removed" )), -OPT_BOOL (0 ,"json" , &show_json, -N_ ("dump index content in json format" )),OPT_BIT (0 ,"directory" , &dir.flags,N_ ("show 'other' directories' names only" ), DIR_SHOW_OTHER_DIRECTORIES),
@@ -581,6 +579,8 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix) N_("pretend that paths removed since <tree-ish> are still present")),OPT__ABBREV (&abbrev),OPT_BOOL (0 ,"debug" , &debug_mode,N_ ("show debugging data" )), +OPT_BOOL (0 ,"debug-json" , &show_json, +N_ ("dump index content in JSON format" )), OPT_END() };
@@ -636,7 +636,7 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix) "--error-unmatch");parse_pathspec (&pathspec,0 , -PATHSPEC_PREFER_CWD , + show_json ?PATHSPEC_PREFER_FULL :PATHSPEC_PREFER_CWD , prefix, argv);/*
@@ -668,8 +668,14 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix) show_cached =1 ;if (show_json && (show_stage || show_deleted || show_others || show_unmerged || show_killed || show_modified || - show_cached || show_resolve_undo || with_tree)) -die (_ ("--show-json cannot be used with other --show- options, or --with-tree" )); + show_cached || pathspec.nr)) +die (_ ("--debug-json cannot be used with other file selection options" )); +if (show_json && show_resolve_undo) +die (_ ("--debug-json cannot be used with %s" ),"--resolve-undo" ); +if (show_json && with_tree) +die (_ ("--debug-json cannot be used with %s" ),"--with-tree" ); +if (show_json && debug_mode) +die (_ ("--debug-json cannot be used with %s" ),"--debug" );if (with_tree) {/*
cache-tree.c +1 -6
@@ -599,18 +599,13 @@ struct cache_tree *cache_tree_read(const char *buffer, unsigned long size,struct cache_tree *ret;if (jw) { -jw_object_inline_begin_object (jw,"cache-tree" ); -jw_object_intmax (jw,"ext-size" , size);jw_object_inline_begin_object (jw,"root" ); }if (buffer[0 ]) ret =NULL ;/* not the whole tree */ else ret =read_one (&buffer, &size, jw); -if (jw) { -jw_end (jw);/* root */ -jw_end (jw);/* cache-tree */ - } +jw_end_gently (jw);return ret; }
dir.c +7 -6
@@ -2902,13 +2902,15 @@ struct untracked_cache *read_untracked_extension(const void *data, uc->exclude_per_dir =xstrdup (exclude_per_dir);if (jw) { -jw_object_inline_begin_object (jw,"untracked-cache" ); -jw_object_intmax (jw,"ext-size" , sz);jw_object_string (jw,"ident" , ident); -jw_object_oid_stat (jw,"info/exclude" , &uc->ss_info_exclude); -jw_object_oid_stat (jw,"excludes-file" , &uc->ss_excludes_file); +jw_object_oid_stat (jw,"info_exclude" , &uc->ss_info_exclude); +jw_object_oid_stat (jw,"excludes_file" , &uc->ss_excludes_file);jw_object_intmax (jw,"flags" , uc->dir_flags); -jw_object_string (jw,"excludes-per-dir" , uc->exclude_per_dir); +if (uc->dir_flags &DIR_SHOW_OTHER_DIRECTORIES ) +jw_object_bool (jw,"show_other_directories" ,1 ); +if (uc->dir_flags &DIR_HIDE_EMPTY_DIRECTORIES ) +jw_object_bool (jw,"hide_empty_directories" ,1 ); +jw_object_string (jw,"excludes_per_dir" , uc->exclude_per_dir); }/* NUL after exclude_per_dir is covered by sizeof(*ouc) */
@@ -2968,7 +2970,6 @@ struct untracked_cache *read_untracked_extension(const void *data,free_untracked_cache (uc); uc =NULL ; } -jw_end_gently (jw);return uc; }
fsmonitor.c +1 -4
@@ -52,12 +52,9 @@ int read_fsmonitor_extension(struct index_state *istate, const void *data, istate->fsmonitor_dirty = fsmonitor_dirty;if (istate->jw) { -jw_object_inline_begin_object (istate->jw,"fsmonitor" );jw_object_intmax (istate->jw,"version" , hdr_version); -jw_object_intmax (istate->jw,"last-update" , istate->fsmonitor_last_update); +jw_object_intmax (istate->jw,"last_update" , istate->fsmonitor_last_update);jw_object_ewah (istate->jw,"dirty" , fsmonitor_dirty); -jw_object_intmax (istate->jw,"ext-size" , sz); -jw_end (istate->jw); }trace_printf_key (&trace_fsmonitor,"read fsmonitor extension successful" );return 0 ;
json-writer.c +15 -9
@@ -203,19 +203,25 @@ void jw_object_null(struct json_writer *jw, const char *key)strbuf_addstr (&jw->json,"null" ); } +void jw_object_filemode (struct json_writer *jw,const char *key,mode_t mode) +{ +object_common (jw, key); +strbuf_addf (&jw->json,"\"%06o\"" , mode); +} +void jw_object_stat_data (struct json_writer *jw,const char *name,const struct stat_data *sd) {jw_object_inline_begin_object (jw, name); -jw_object_intmax (jw,"st_ctime.sec" , sd->sd_ctime.sec); -jw_object_intmax (jw,"st_ctime.nsec" , sd->sd_ctime.nsec); -jw_object_intmax (jw,"st_mtime.sec" , sd->sd_mtime.sec); -jw_object_intmax (jw,"st_mtime.nsec" , sd->sd_mtime.nsec); -jw_object_intmax (jw,"st_dev" , sd->sd_dev); -jw_object_intmax (jw,"st_ino" , sd->sd_ino); -jw_object_intmax (jw,"st_uid" , sd->sd_uid); -jw_object_intmax (jw,"st_gid" , sd->sd_gid); -jw_object_intmax (jw,"st_size" , sd->sd_size); +jw_object_intmax (jw,"ctime_sec" , sd->sd_ctime.sec); +jw_object_intmax (jw,"ctime_nsec" , sd->sd_ctime.nsec); +jw_object_intmax (jw,"mtime_sec" , sd->sd_mtime.sec); +jw_object_intmax (jw,"mtime_nsec" , sd->sd_mtime.nsec); +jw_object_intmax (jw,"device" , sd->sd_dev); +jw_object_intmax (jw,"inode" , sd->sd_ino); +jw_object_intmax (jw,"uid" , sd->sd_uid); +jw_object_intmax (jw,"gid" , sd->sd_gid); +jw_object_intmax (jw,"size" , sd->sd_size);jw_end (jw); }
json-writer.h +3 -0
@@ -42,8 +42,10 @@* of the given strings. */ +#include "git-compat-util.h" #include "strbuf.h" +struct ewah_bitmap ;struct stat_data ;struct json_writer
@@ -83,6 +85,7 @@ void jw_object_true(struct json_writer *jw, const char *key);void jw_object_false (struct json_writer *jw,const char *key);void jw_object_bool (struct json_writer *jw,const char *key,int value);void jw_object_null (struct json_writer *jw,const char *key); +void jw_object_filemode (struct json_writer *jw,const char *key,mode_t value);void jw_object_stat_data (struct json_writer *jw,const char *key,const struct stat_data *sd); void jw_object_ewah(struct json_writer *jw, const char *key,
read-cache.c +87 -78
317 lines not on this page · raw message
resolve-undo.c +25 -31
75 lines not on this page · raw message
split-index.c +2 -6
@@ -17,7 +17,6 @@ int read_link_extension(struct index_state *istate, {const unsigned char *data = data_;struct split_index *si; -unsigned long original_sz = sz;int ret;if (sz < the_hash_algo->rawsz)
@@ -42,12 +41,9 @@ int read_link_extension(struct index_state *istate,return error ("garbage at the end of link extension" ); done:if (istate->jw) { -jw_object_inline_begin_object (istate->jw,"split-index" );jw_object_string (istate->jw,"oid" ,oid_to_hex (&si->base_oid)); -jw_object_ewah (istate->jw,"delete-bitmap" , si->delete_bitmap); -jw_object_ewah (istate->jw,"replace-bitmap" , si->replace_bitmap); -jw_object_intmax (istate->jw,"ext-size" , original_sz); -jw_end (istate->jw); +jw_object_ewah (istate->jw,"delete_bitmap" , si->delete_bitmap); +jw_object_ewah (istate->jw,"replace_bitmap" , si->replace_bitmap); }return 0 ; }
t/t3008-ls-files-lazy-init-name-hash.sh +1 -7
@@ -4,15 +4,9 @@ test_description='Test the lazy init name hash with various folder structures' . ./test-lib.sh -if test 1 -eq $($GIT_BUILD_DIR/t/helper/test-tool online-cpus) -then - skip_all='skipping lazy-init tests, single cpu' - test_done -fi - LAZY_THREAD_COST=2000 -test_expect_success 'no buffer overflow in lazy_init_name_hash' ' +test_expect_success !SINGLE_CPU 'no buffer overflow in lazy_init_name_hash' ' ( test_seq $LAZY_THREAD_COST | sed "s/^/a_/" && echo b/b/b &&
t/t3011-ls-files-json.sh new file +106 -0
106 lines not on this page · raw message
t/t3011/basic new file +124 -0
124 lines not on this page · raw message
t/t3011/eoie new file +107 -0
107 lines not on this page · raw message
t/t3011/fsmonitor new file +38 -0
@@ -0,0 +1,38 @@ +{ + "version": 2, + "oid": <string>, + "mtime_sec": <number>, + "mtime_nsec": <number>, + "entries": [ + { + "id": 0, + "name": "one", + "mode": "100644", + "flags": 0, + "oid": <string>, + "stat": { + "ctime_sec": <number>, + "ctime_nsec": <number>, + "mtime_sec": <number>, + "mtime_nsec": <number>, + "device": <number>, + "inode": <number>, + "uid": <number>, + "gid": <number>, + "size": 4 + }, + "file_offset": <number> + } + ], + "extensions": { + "FSMN": { + "file_offset": <number>, + "ext_size": <number>, + "version": 1, + "last_update": <number>, + "dirty": [ + 0 + ] + } + } +}
t/t3011/rerere new file +66 -0
66 lines not on this page · raw message
t/t3011/split-index new file +39 -0
@@ -0,0 +1,39 @@ +{ + "version": 2, + "oid": <string>, + "mtime_sec": <number>, + "mtime_nsec": <number>, + "entries": [ + { + "id": 0, + "name": "", + "mode": "100644", + "flags": 0, + "oid": <string>, + "stat": { + "ctime_sec": <number>, + "ctime_nsec": <number>, + "mtime_sec": <number>, + "mtime_nsec": <number>, + "device": <number>, + "inode": <number>, + "uid": <number>, + "gid": <number>, + "size": 4 + }, + "file_offset": <number> + } + ], + "extensions": { + "link": { + "file_offset": <number>, + "ext_size": <number>, + "oid": <string>, + "delete_bitmap": [ + ], + "replace_bitmap": [ + 0 + ] + } + } +}
t/test-lib.sh +4 -0
@@ -1611,3 +1611,7 @@ test_lazy_prereq REBASE_P ' test_lazy_prereq FAIL_PREREQS ' test -n "$GIT_TEST_FAIL_PREREQS" ' + +test_lazy_prereq SINGLE_CPU ' + test "$(test-tool online-cpus)" -eq 1 +'
Nguyễn Thái Ngọc Duy (10):
ls-files: add --json to dump the index read-cache.c: dump common extension info in json cache-tree.c: dump "TREE" extension as json dir.c: dump "UNTR" extension as json split-index.c: dump "link" extension as json fsmonitor.c: dump "FSMN" extension as json resolve-undo.c: dump "REUC" extension as json read-cache.c: dump "EOIE" extension as json read-cache.c: dump "IEOT" extension as json
t3008: use the new SINGLE_CPU prereq
Documentation/git-ls-files.txt | 5 + builtin/ls-files.c | 38 ++++- cache-tree.c | 36 ++++- cache-tree.h | 5 +- cache.h | 2 + dir.c | 57 +++++++- dir.h | 4 +- fsmonitor.c | 6 + json-writer.c | 36 +++++ json-writer.h | 32 +++++ read-cache.c | 178 ++++++++++++++++++++---- resolve-undo.c | 30 +++- resolve-undo.h | 4 +- split-index.c | 9 +- t/t3008-ls-files-lazy-init-name-hash.sh | 8 +- t/t3011-ls-files-json.sh (new +x) | 106 ++++++++++++++ t/t3011/basic (new) | 124 +++++++++++++++++ t/t3011/eoie (new) | 107 ++++++++++++++ t/t3011/fsmonitor (new) | 38 +++++ t/t3011/rerere (new) | 66 +++++++++ t/t3011/split-index (new) | 39 ++++++ t/test-lib.sh | 4 + 22 files changed, 884 insertions(+), 50 deletions(-) create mode 100755 t/t3011-ls-files-json.sh create mode 100644 t/t3011/basic create mode 100644 t/t3011/eoie create mode 100644 t/t3011/fsmonitor create mode 100644 t/t3011/rerere create mode 100644 t/t3011/split-index
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
PATCH v2 1/10 ls-files: add --json to dump the index
So far we don't have a command to basically dump the index file out, with all its glory details. Checking some info, for example, stat time, usually involves either writing new code or firing up "xxd" and decoding values by yourself.
This --json is supposed to help that. It dumps the index in a human readable format but also easy to be processed with tools. And it will print almost enough info to reconstruct the index later.
In this patch we only dump the main part, not extensions. But at the end of the series, the entire index is dumped. The end result could be very verbose even on a small repository such as git.git.
--- Documentation/git-ls-files.txt | 5 +++ builtin/ls-files.c | 38 +++++++++++++--- cache.h | 2 + json-writer.c | 22 ++++++++++ json-writer.h | 23 ++++++++++ read-cache.c | 72 ++++++++++++++++++++++++++++++- t/t3011-ls-files-json.sh (new +x) | 44 +++++++++++++++++++ t/t3011/basic (new) | 67 ++++++++++++++++++++++++++++ 8 files changed, 265 insertions(+), 8 deletions(-)
Documentation/git-ls-files.txt +5 -0
@@ -162,6 +162,11 @@ a space) at the start of each line: possible for manual inspection; the exact format may change at any time. +--debug-json:: + Dump the entire index content in JSON format. This is for + debugging purposes. The JSON structure is subject to change. + Note that the strings are not always valid UTF-8. + --eol:: Show <eolinfo> and <eolattr> of files. <eolinfo> is the file content identification used by Git when
builtin/ls-files.c +32 -6
@@ -8,6 +8,7 @@#include "cache.h" #include "repository.h" #include "config.h" +#include "json-writer.h" #include "quote.h" #include "dir.h" #include "builtin.h"
@@ -31,6 +32,7 @@ static int show_modified;static int show_killed;static int show_valid_bit;static int show_fsmonitor_bit; +static int show_json;static int line_terminator ='\n' ;static int debug_mode;static int show_eol;
@@ -577,6 +579,8 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix) N_("pretend that paths removed since <tree-ish> are still present")),OPT__ABBREV (&abbrev),OPT_BOOL (0 ,"debug" , &debug_mode,N_ ("show debugging data" )), +OPT_BOOL (0 ,"debug-json" , &show_json, +N_ ("dump index content in JSON format" )), OPT_END() };
@@ -632,7 +636,7 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix) "--error-unmatch");parse_pathspec (&pathspec,0 , -PATHSPEC_PREFER_CWD , + show_json ?PATHSPEC_PREFER_FULL :PATHSPEC_PREFER_CWD , prefix, argv);/*
@@ -660,8 +664,18 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)/* With no flags, we default to showing the cached files */ if (!(show_stage || show_deleted || show_others || show_unmerged || - show_killed || show_modified || show_resolve_undo)) + show_killed || show_modified || show_resolve_undo || show_json)) show_cached =1 ; +if (show_json && (show_stage || show_deleted || show_others || + show_unmerged || show_killed || show_modified || + show_cached || pathspec.nr)) +die (_ ("--debug-json cannot be used with other file selection options" )); +if (show_json && show_resolve_undo) +die (_ ("--debug-json cannot be used with %s" ),"--resolve-undo" ); +if (show_json && with_tree) +die (_ ("--debug-json cannot be used with %s" ),"--with-tree" ); +if (show_json && debug_mode) +die (_ ("--debug-json cannot be used with %s" ),"--debug" );if (with_tree) {/*
@@ -673,10 +687,22 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)overlay_tree_on_index (the_repository->index, with_tree, max_prefix); } -show_files (the_repository, &dir); - -if (show_resolve_undo) -show_ru_info (the_repository->index); +if (!show_json) { +show_files (the_repository, &dir); + +if (show_resolve_undo) +show_ru_info (the_repository->index); + }else { +struct json_writer jw =JSON_WRITER_INIT ; + +discard_index (the_repository->index); + the_repository->index->jw = &jw; +if (repo_read_index (the_repository) <0 ) +die ("index file corrupt" ); +puts (jw.json.buf); + the_repository->index->jw =NULL ; +jw_release (&jw); + }if (ps_matched) {int bad;
cache.h +2 -0
@@ -326,6 +326,7 @@ static inline unsigned int canon_mode(unsigned int mode)#define UNTRACKED_CHANGED (1 << 7)#define FSMONITOR_CHANGED (1 << 8) +struct json_writer ;struct split_index ;struct untracked_cache ;
@@ -350,6 +351,7 @@ struct index_state {uint64_t fsmonitor_last_update;struct ewah_bitmap *fsmonitor_dirty;struct mem_pool *ce_mem_pool; +struct json_writer *jw; };/* Name hashing */
json-writer.c +22 -0
@@ -202,6 +202,28 @@ void jw_object_null(struct json_writer *jw, const char *key)strbuf_addstr (&jw->json,"null" ); } +void jw_object_filemode (struct json_writer *jw,const char *key,mode_t mode) +{ +object_common (jw, key); +strbuf_addf (&jw->json,"\"%06o\"" , mode); +} + +void jw_object_stat_data (struct json_writer *jw,const char *name, +const struct stat_data *sd) +{ +jw_object_inline_begin_object (jw, name); +jw_object_intmax (jw,"ctime_sec" , sd->sd_ctime.sec); +jw_object_intmax (jw,"ctime_nsec" , sd->sd_ctime.nsec); +jw_object_intmax (jw,"mtime_sec" , sd->sd_mtime.sec); +jw_object_intmax (jw,"mtime_nsec" , sd->sd_mtime.nsec); +jw_object_intmax (jw,"device" , sd->sd_dev); +jw_object_intmax (jw,"inode" , sd->sd_ino); +jw_object_intmax (jw,"uid" , sd->sd_uid); +jw_object_intmax (jw,"gid" , sd->sd_gid); +jw_object_intmax (jw,"size" , sd->sd_size); +jw_end (jw); +} +static void increase_indent (struct strbuf *sb,const struct json_writer *jw,int indent)
json-writer.h +23 -0
@@ -42,8 +42,11 @@* of the given strings. */ +#include "git-compat-util.h" #include "strbuf.h" +struct stat_data ; +struct json_writer {/*
@@ -81,6 +84,9 @@ void jw_object_true(struct json_writer *jw, const char *key);void jw_object_false (struct json_writer *jw,const char *key);void jw_object_bool (struct json_writer *jw,const char *key,int value);void jw_object_null (struct json_writer *jw,const char *key); +void jw_object_filemode (struct json_writer *jw,const char *key,mode_t value); +void jw_object_stat_data (struct json_writer *jw,const char *key, +const struct stat_data *sd);void jw_object_sub_jw (struct json_writer *jw,const char *key,const struct json_writer *value);
@@ -104,4 +110,21 @@ void jw_array_inline_begin_array(struct json_writer *jw);int jw_is_terminated (const struct json_writer *jw);void jw_end (struct json_writer *jw); +/* +* These _gently versions accept NULL json_writer to reduce too much +* branching at the call site. +*/ +static inline void jw_object_inline_begin_array_gently (struct json_writer *jw, +const char *name) +{ +if (jw) +jw_object_inline_begin_array (jw, name); +} + +static inline void jw_end_gently (struct json_writer *jw) +{ +if (jw) +jw_end (jw); +} +#endif /* JSON_WRITER_H */
read-cache.c +70 -2
132 lines not on this page · raw message
t/t3011-ls-files-json.sh new file +44 -0
44 lines not on this page · raw message
t/t3011/basic new file +67 -0
67 lines not on this page · raw message
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
PATCH v2 2/10 read-cache.c: dump common extension info in json
In reply to Nguyễn Thái Ngọc Duy--- read-cache.c | 49 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 14 deletions(-)
read-cache.c +35 -14
@@ -1694,8 +1694,26 @@ static int verify_hdr(const struct cache_header *hdr, unsigned long size) }static int read_index_extension (struct index_state *istate, -const char *ext,const char *data,unsigned long sz) +const char *map, +unsigned long *offset) { +int ret =0 ; +const char *ext = map + *offset; +uint32_t sz =get_be32 (ext +4 ); +const char *data = ext +8 ; + +if (istate->jw) { +char buf[5 ]; + +memcpy (buf, ext,4 ); + buf[4 ] ='\0' ; +jw_object_inline_begin_object (istate->jw, buf); + +jw_object_intmax (istate->jw,"file_offset" , *offset); +jw_object_intmax (istate->jw,"ext_size" , sz); + } + *offset += sz +8 ; +switch (CACHE_EXT (ext)) {case CACHE_EXT_TREE : istate->cache_tree =cache_tree_read (data, sz);
@@ -1704,8 +1722,7 @@ static int read_index_extension(struct index_state *istate, istate->resolve_undo =resolve_undo_read (data, sz);break ;case CACHE_EXT_LINK : -if (read_link_extension (istate, data, sz)) -return -1 ; + ret =read_link_extension (istate, data, sz);break ;case CACHE_EXT_UNTRACKED : istate->untracked =read_untracked_extension (data, sz);
@@ -1719,12 +1736,14 @@ static int read_index_extension(struct index_state *istate,break ;default :if (*ext <'A' ||'Z' < *ext) -return error (_ ("index uses %.4s extension, which we do not understand" ), + ret =error (_ ("index uses %.4s extension, which we do not understand" ), ext); -fprintf_ln (stderr,_ ("ignoring %.4s extension" ), ext); +else +fprintf_ln (stderr,_ ("ignoring %.4s extension" ), ext);break ; } -return 0 ; +jw_end_gently (istate->jw); +return ret; } static struct cache_entry *create_from_disk(struct mem_pool *ce_mem_pool,
@@ -1930,25 +1949,27 @@ static void *load_index_extensions(void *_data) {struct load_index_extensions *p = _data;unsigned long src_offset = p->src_offset; +int dump_json =0 ;while (src_offset <= p->mmap_size - the_hash_algo->rawsz -8 ) { -/* After an array of active_nr index entries, +if (p->istate->jw && !dump_json) { +jw_object_inline_begin_object (p->istate->jw,"extensions" ); + dump_json =1 ; + } +/* +* After an array of active_nr index entries, * there can be arbitrary number of extended * sections, each of which is prefixed with * extension name (4-byte) and section length * in 4-byte network byte order. */ -uint32_t extsize =get_be32 (p->mmap + src_offset +4 ); -if (read_index_extension (p->istate, - p->mmap + src_offset, - p->mmap + src_offset +8 , - extsize) <0 ) { +if (read_index_extension (p->istate, p->mmap, &src_offset) <0 ) {munmap ((void *)p->mmap, p->mmap_size);die (_ ("index file corrupt" )); } - src_offset +=8 ; - src_offset += extsize; } +if (dump_json) +jw_end (p->istate->jw);return NULL ; }
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
PATCH v2 3/10 cache-tree.c: dump "TREE" extension as json
In reply to Nguyễn Thái Ngọc Duy--- cache-tree.c | 36 +++++++++++++++++++++++++++++++----- cache-tree.h | 5 ++++- read-cache.c | 2 +- t/t3011-ls-files-json.sh | 4 +++- t/t3011/basic | 20 +++++++++++++++++++- 5 files changed, 58 insertions(+), 9 deletions(-)
cache-tree.c +31 -5
@@ -3,6 +3,7 @@#include "tree.h" #include "tree-walk.h" #include "cache-tree.h" +#include "json-writer.h" #include "object-store.h" #include "replace-object.h"
@@ -492,7 +493,8 @@ void cache_tree_write(struct strbuf *sb, struct cache_tree *root)write_one (sb, root,"" ,0 ); } -static struct cache_tree *read_one (const char **buffer,unsigned long *size_p) +static struct cache_tree *read_one (const char **buffer,unsigned long *size_p, +struct json_writer *jw) {const char *buf = *buffer;unsigned long size = *size_p;
@@ -546,6 +548,15 @@ static struct cache_tree *read_one(const char **buffer, unsigned long *size_p) *buffer, subtree_nr);#endif +if (jw) { +if (it->entry_count >=0 ) { +jw_object_string (jw,"oid" ,oid_to_hex (&it->oid)); +jw_object_intmax (jw,"entry_count" , it->entry_count); + }else { +jw_object_null (jw,"oid" ); + } +jw_object_inline_begin_array (jw,"subdirs" ); + }/* * Just a heuristic -- we do not add directories that often but * we do not want to have to extend it immediately when we do,
@@ -559,12 +570,18 @@ static struct cache_tree *read_one(const char **buffer, unsigned long *size_p)struct cache_tree_sub *subtree;const char *name = buf; - sub =read_one (&buf, &size); +if (jw) { +jw_array_inline_begin_object (jw); +jw_object_string (jw,"name" , name); + } + sub =read_one (&buf, &size, jw); +jw_end_gently (jw);if (!sub) goto free_return; subtree =cache_tree_sub (it, name); subtree->cache_tree = sub; } +jw_end_gently (jw);if (subtree_nr != it->subtree_nr)die ("cache-tree: internal error" ); *buffer = buf;
@@ -576,11 +593,20 @@ static struct cache_tree *read_one(const char **buffer, unsigned long *size_p)return NULL ; } -struct cache_tree *cache_tree_read (const char *buffer,unsigned long size) +struct cache_tree *cache_tree_read (const char *buffer,unsigned long size, +struct json_writer *jw) { +struct cache_tree *ret; + +if (jw) { +jw_object_inline_begin_object (jw,"root" ); + }if (buffer[0 ]) -return NULL ;/* not the whole tree */ -return read_one (&buffer, &size); + ret =NULL ;/* not the whole tree */ +else + ret =read_one (&buffer, &size, jw); +jw_end_gently (jw); +return ret; }static struct cache_tree *cache_tree_find (struct cache_tree *it,const char *path)
cache-tree.h +4 -1
@@ -6,6 +6,8 @@#include "tree-walk.h" struct cache_tree ; +struct json_writer ; +struct cache_tree_sub {struct cache_tree *cache_tree;int count;/* internally used by update_one() */
@@ -28,7 +30,8 @@ void cache_tree_invalidate_path(struct index_state *, const char *);struct cache_tree_sub *cache_tree_sub (struct cache_tree *,const char *);void cache_tree_write (struct strbuf *,struct cache_tree *root); -struct cache_tree *cache_tree_read (const char *buffer,unsigned long size); +struct cache_tree *cache_tree_read (const char *buffer,unsigned long size, +struct json_writer *jw);int cache_tree_fully_valid (struct cache_tree *);int cache_tree_update (struct index_state *,int );
read-cache.c +1 -1
@@ -1716,7 +1716,7 @@ static int read_index_extension(struct index_state *istate,switch (CACHE_EXT (ext)) {case CACHE_EXT_TREE : - istate->cache_tree =cache_tree_read (data, sz); + istate->cache_tree =cache_tree_read (data, sz, istate->jw);break ;case CACHE_EXT_RESOLVE_UNDO : istate->resolve_undo =resolve_undo_read (data, sz);
t/t3011-ls-files-json.sh +3 -1
@@ -29,6 +29,8 @@ test_expect_success 'setup' ' echo 2 >sub/two && git add sub/two && + git commit -m first && + echo intent-to-add >ita && git add -N ita &&
@@ -37,7 +39,7 @@ test_expect_success 'setup' ' strip_string oid ident ' -test_expect_success 'ls-files --json, main entries' ' +test_expect_success 'ls-files --json, main entries and TREE' ' compare_json basic '
t/t3011/basic +19 -1
@@ -63,5 +63,23 @@ }, "file_offset": <number> } - ] + ], + "extensions": { + "TREE": { + "file_offset": <number>, + "ext_size": <number>, + "root": { + "oid": null, + "subdirs": [ + { + "name": "sub", + "oid": <string>, + "entry_count": 1, + "subdirs": [ + ] + } + ] + } + } + } }
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
PATCH v2 4/10 dir.c: dump "UNTR" extension as json
In reply to Nguyễn Thái Ngọc DuyThe big part of UNTR extension is dumped at the end instead of dumping as soon as we read it, because we actually "patch" some fields in untracked_cache_dir with EWAH bitmaps at the end.
--- dir.c | 57 +++++++++++++++++++++++++++++++++++++++- dir.h | 4 ++- json-writer.h | 6 +++++ read-cache.c | 2 +- t/t3011-ls-files-json.sh | 3 ++- t/t3011/basic | 39 +++++++++++++++++++++++++++ 6 files changed, 107 insertions(+), 4 deletions(-)
dir.c +56 -1
@@ -19,6 +19,7 @@#include "varint.h" #include "ewah/ewok.h" #include "fsmonitor.h" +#include "json-writer.h" #include "submodule-config.h" /*
@@ -2826,7 +2827,42 @@ static void load_oid_stat(struct oid_stat *oid_stat, const unsigned char *data, oid_stat->valid =1 ; } -struct untracked_cache *read_untracked_extension (const void *data,unsigned long sz) +static void jw_object_oid_stat (struct json_writer *jw,const char *key, +const struct oid_stat *oid_stat) +{ +jw_object_inline_begin_object (jw, key); +jw_object_bool (jw,"valid" , oid_stat->valid); +jw_object_string (jw,"oid" ,oid_to_hex (&oid_stat->oid)); +jw_object_stat_data (jw,"stat" , &oid_stat->stat); +jw_end (jw); +} + +static void jw_object_untracked_cache_dir (struct json_writer *jw, +const struct untracked_cache_dir *ucd) +{ +int i; + +jw_object_bool (jw,"valid" , ucd->valid); +jw_object_bool (jw,"check-only" , ucd->check_only); +jw_object_stat_data (jw,"stat" , &ucd->stat_data); +jw_object_string (jw,"exclude-oid" ,oid_to_hex (&ucd->exclude_oid)); +jw_object_inline_begin_array (jw,"untracked" ); +for (i =0 ; i < ucd->untracked_nr; i++) +jw_array_string (jw, ucd->untracked[i]); +jw_end (jw); + +jw_object_inline_begin_object (jw,"dirs" ); +for (i =0 ; i < ucd->dirs_nr; i++) { +jw_object_inline_begin_object (jw, ucd->dirs[i]->name); +jw_object_untracked_cache_dir (jw, ucd->dirs[i]); +jw_end (jw); + } +jw_end (jw); +} + +struct untracked_cache *read_untracked_extension (const void *data, +unsigned long sz, +struct json_writer *jw) {struct untracked_cache *uc;struct read_data rd;
@@ -2864,6 +2900,19 @@ struct untracked_cache *read_untracked_extension(const void *data, unsigned long uc->dir_flags =get_be32 (next +ouc_offset (dir_flags)); exclude_per_dir = (const char *)next + exclude_per_dir_offset; uc->exclude_per_dir =xstrdup (exclude_per_dir); + +if (jw) { +jw_object_string (jw,"ident" , ident); +jw_object_oid_stat (jw,"info_exclude" , &uc->ss_info_exclude); +jw_object_oid_stat (jw,"excludes_file" , &uc->ss_excludes_file); +jw_object_intmax (jw,"flags" , uc->dir_flags); +if (uc->dir_flags &DIR_SHOW_OTHER_DIRECTORIES ) +jw_object_bool (jw,"show_other_directories" ,1 ); +if (uc->dir_flags &DIR_HIDE_EMPTY_DIRECTORIES ) +jw_object_bool (jw,"hide_empty_directories" ,1 ); +jw_object_string (jw,"excludes_per_dir" , uc->exclude_per_dir); + } +/* NUL after exclude_per_dir is covered by sizeof(*ouc) */ next += exclude_per_dir_offset +strlen (exclude_per_dir) +1 ;if (next >= end)
@@ -2905,6 +2954,12 @@ struct untracked_cache *read_untracked_extension(const void *data, unsigned longewah_each_bit (rd.sha1_valid, read_oid, &rd); next = rd.data; +if (jw) { +jw_object_inline_begin_object (jw,"root" ); +jw_object_untracked_cache_dir (jw, uc->root); +jw_end (jw); + } + done:free (rd.ucd);ewah_free (rd.valid);
dir.h +3 -1
@@ -6,6 +6,8 @@#include "cache.h" #include "strbuf.h" +struct json_writer ; +struct dir_entry {unsigned int len;char name[FLEX_ARRAY ];/* more */
@@ -362,7 +364,7 @@ void untracked_cache_remove_from_index(struct index_state *, const char *);void untracked_cache_add_to_index (struct index_state *,const char *);void free_untracked_cache (struct untracked_cache *); -struct untracked_cache *read_untracked_extension (const void *data,unsigned long sz); +struct untracked_cache *read_untracked_extension (const void *data,unsigned long sz,struct json_writer *jw);void write_untracked_extension (struct strbuf *out,struct untracked_cache *untracked);void add_untracked_cache (struct index_state *istate);void remove_untracked_cache (struct index_state *istate);
json-writer.h +6 -0
@@ -121,6 +121,12 @@ static inline void jw_object_inline_begin_array_gently(struct json_writer *jw,jw_object_inline_begin_array (jw, name); } +static inline void jw_array_inline_begin_object_gently (struct json_writer *jw) +{ +if (jw) +jw_array_inline_begin_object (jw); +} +static inline void jw_end_gently (struct json_writer *jw) {if (jw)
read-cache.c +1 -1
@@ -1725,7 +1725,7 @@ static int read_index_extension(struct index_state *istate, ret =read_link_extension (istate, data, sz);break ;case CACHE_EXT_UNTRACKED : - istate->untracked =read_untracked_extension (data, sz); + istate->untracked =read_untracked_extension (data, sz, istate->jw);break ;case CACHE_EXT_FSMONITOR :read_fsmonitor_extension (istate, data, sz);
t/t3011-ls-files-json.sh +2 -1
@@ -30,6 +30,7 @@ test_expect_success 'setup' ' git add sub/two && git commit -m first && + git update-index --untracked-cache && echo intent-to-add >ita && git add -N ita &&
@@ -39,7 +40,7 @@ test_expect_success 'setup' ' strip_string oid ident ' -test_expect_success 'ls-files --json, main entries and TREE' ' +test_expect_success 'ls-files --json, main entries, UNTR and TREE' ' compare_json basic '
t/t3011/basic +39 -0
@@ -80,6 +80,45 @@ } ] } + }, + "UNTR": { + "file_offset": <number>, + "ext_size": <number>, + "ident": <string>, + "info_exclude": { + "valid": true, + "oid": <string>, + "stat": { + "ctime_sec": <number>, + "ctime_nsec": <number>, + "mtime_sec": <number>, + "mtime_nsec": <number>, + "device": <number>, + "inode": <number>, + "uid": <number>, + "gid": <number>, + "size": 0 + } + }, + "excludes_file": { + "valid": true, + "oid": <string>, + "stat": { + "ctime_sec": <number>, + "ctime_nsec": <number>, + "mtime_sec": <number>, + "mtime_nsec": <number>, + "device": <number>, + "inode": <number>, + "uid": <number>, + "gid": <number>, + "size": 0 + } + }, + "flags": 6, + "show_other_directories": true, + "hide_empty_directories": true, + "excludes_per_dir": ".gitignore" } } }
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
PATCH v2 5/10 split-index.c: dump "link" extension as json
In reply to Nguyễn Thái Ngọc Duy--- json-writer.c | 14 ++++++++++++++ json-writer.h | 3 +++ split-index.c | 9 ++++++++- t/t3011-ls-files-json.sh | 14 ++++++++++++++ t/t3011/split-index (new) | 39 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 78 insertions(+), 1 deletion(-)
json-writer.c +14 -0
@@ -1,4 +1,5 @@#include "cache.h" +#include "ewah/ewok.h" #include "json-writer.h" void jw_init (struct json_writer *jw)
@@ -224,6 +225,19 @@ void jw_object_stat_data(struct json_writer *jw, const char *name,jw_end (jw); } +static void dump_ewah_one (size_t pos,void *jw) +{ +jw_array_intmax (jw, pos); +} + +void jw_object_ewah (struct json_writer *jw,const char *key, +struct ewah_bitmap *ewah) +{ +jw_object_inline_begin_array (jw, key); +ewah_each_bit (ewah, dump_ewah_one, jw); +jw_end (jw); +} +static void increase_indent (struct strbuf *sb,const struct json_writer *jw,int indent)
json-writer.h +3 -0
@@ -45,6 +45,7 @@#include "git-compat-util.h" #include "strbuf.h" +struct ewah_bitmap ;struct stat_data ;struct json_writer
@@ -87,6 +88,8 @@ void jw_object_null(struct json_writer *jw, const char *key);void jw_object_filemode (struct json_writer *jw,const char *key,mode_t value);void jw_object_stat_data (struct json_writer *jw,const char *key,const struct stat_data *sd); +void jw_object_ewah (struct json_writer *jw,const char *key, +struct ewah_bitmap *ewah);void jw_object_sub_jw (struct json_writer *jw,const char *key,const struct json_writer *value);
split-index.c +8 -1
@@ -1,4 +1,5 @@#include "cache.h" +#include "json-writer.h" #include "split-index.h" #include "ewah/ewok.h"
@@ -25,7 +26,7 @@ int read_link_extension(struct index_state *istate, data += the_hash_algo->rawsz; sz -= the_hash_algo->rawsz;if (!sz) -return 0 ; + goto done; si->delete_bitmap =ewah_new (); ret =ewah_read_mmap (si->delete_bitmap, data, sz);if (ret <0 )
@@ -38,6 +39,12 @@ int read_link_extension(struct index_state *istate,return error ("corrupt replace bitmap in link extension" );if (ret != sz)return error ("garbage at the end of link extension" ); +done: +if (istate->jw) { +jw_object_string (istate->jw,"oid" ,oid_to_hex (&si->base_oid)); +jw_object_ewah (istate->jw,"delete_bitmap" , si->delete_bitmap); +jw_object_ewah (istate->jw,"replace_bitmap" , si->replace_bitmap); + }return 0 ; }
t/t3011-ls-files-json.sh +14 -0
@@ -44,4 +44,18 @@ test_expect_success 'ls-files --json, main entries, UNTR and TREE' ' compare_json basic ' +test_expect_success 'ls-files --json, split index' ' + git init split && + ( + cd split && + echo one >one && + git add one && + git update-index --split-index && + echo updated >>one && + test_must_fail git -c splitIndex.maxPercentChange=100 update-index --refresh && + cp ../filter.sed . && + compare_json split-index + ) +' + test_done
t/t3011/split-index new file +39 -0
@@ -0,0 +1,39 @@ +{ + "version": 2, + "oid": <string>, + "mtime_sec": <number>, + "mtime_nsec": <number>, + "entries": [ + { + "id": 0, + "name": "", + "mode": "100644", + "flags": 0, + "oid": <string>, + "stat": { + "ctime_sec": <number>, + "ctime_nsec": <number>, + "mtime_sec": <number>, + "mtime_nsec": <number>, + "device": <number>, + "inode": <number>, + "uid": <number>, + "gid": <number>, + "size": 4 + }, + "file_offset": <number> + } + ], + "extensions": { + "link": { + "file_offset": <number>, + "ext_size": <number>, + "oid": <string>, + "delete_bitmap": [ + ], + "replace_bitmap": [ + 0 + ] + } + } +}
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
PATCH v2 6/10 fsmonitor.c: dump "FSMN" extension as json
In reply to Nguyễn Thái Ngọc Duy--- fsmonitor.c | 6 ++++++ t/t3011-ls-files-json.sh | 14 +++++++++++++- t/t3011/fsmonitor (new) | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-)
fsmonitor.c +6 -0
@@ -3,6 +3,7 @@#include "dir.h" #include "ewah/ewok.h" #include "fsmonitor.h" +#include "json-writer.h" #include "run-command.h" #include "strbuf.h"
@@ -50,6 +51,11 @@ int read_fsmonitor_extension(struct index_state *istate, const void *data, } istate->fsmonitor_dirty = fsmonitor_dirty; +if (istate->jw) { +jw_object_intmax (istate->jw,"version" , hdr_version); +jw_object_intmax (istate->jw,"last_update" , istate->fsmonitor_last_update); +jw_object_ewah (istate->jw,"dirty" , fsmonitor_dirty); + }trace_printf_key (&trace_fsmonitor,"read fsmonitor extension successful" );return 0 ; }
t/t3011-ls-files-json.sh +13 -1
@@ -36,7 +36,7 @@ test_expect_success 'setup' ' git add -N ita && strip_number ctime_sec ctime_nsec mtime_sec mtime_nsec && - strip_number device inode uid gid file_offset ext_size && + strip_number device inode uid gid file_offset ext_size last_update && strip_string oid ident '
@@ -58,4 +58,16 @@ test_expect_success 'ls-files --json, split index' ' ) ' +test_expect_success 'ls-files --json, fsmonitor extension ' ' + git init fsmonitor && + ( + cd fsmonitor && + echo one >one && + git add one && + git update-index --fsmonitor && + cp ../filter.sed . && + compare_json fsmonitor + ) +' + test_done
t/t3011/fsmonitor new file +38 -0
@@ -0,0 +1,38 @@ +{ + "version": 2, + "oid": <string>, + "mtime_sec": <number>, + "mtime_nsec": <number>, + "entries": [ + { + "id": 0, + "name": "one", + "mode": "100644", + "flags": 0, + "oid": <string>, + "stat": { + "ctime_sec": <number>, + "ctime_nsec": <number>, + "mtime_sec": <number>, + "mtime_nsec": <number>, + "device": <number>, + "inode": <number>, + "uid": <number>, + "gid": <number>, + "size": 4 + }, + "file_offset": <number> + } + ], + "extensions": { + "FSMN": { + "file_offset": <number>, + "ext_size": <number>, + "version": 1, + "last_update": <number>, + "dirty": [ + 0 + ] + } + } +}
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
PATCH v2 7/10 resolve-undo.c: dump "REUC" extension as json
In reply to Nguyễn Thái Ngọc Duy--- read-cache.c | 2 +- resolve-undo.c | 30 +++++++++++++++++- resolve-undo.h | 4 ++- t/t3011-ls-files-json.sh | 20 ++++++++++++ t/t3011/rerere (new) | 66 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 119 insertions(+), 3 deletions(-)
read-cache.c +1 -1
@@ -1719,7 +1719,7 @@ static int read_index_extension(struct index_state *istate, istate->cache_tree =cache_tree_read (data, sz, istate->jw);break ;case CACHE_EXT_RESOLVE_UNDO : - istate->resolve_undo =resolve_undo_read (data, sz); + istate->resolve_undo =resolve_undo_read (data, sz, istate->jw);break ;case CACHE_EXT_LINK : ret =read_link_extension (istate, data, sz);
resolve-undo.c +29 -1
@@ -1,5 +1,6 @@#include "cache.h" #include "dir.h" +#include "json-writer.h" #include "resolve-undo.h" #include "string-list.h"
@@ -49,7 +50,30 @@ void resolve_undo_write(struct strbuf *sb, struct string_list *resolve_undo) } } -struct string_list *resolve_undo_read (const char *data,unsigned long size) +static void dump_resolve_undo (struct json_writer *jw, +const char *path, +const struct resolve_undo_info *ui) +{ +int i; + +if (!jw) +return ; + +jw_array_inline_begin_object (jw); +jw_object_string (jw,"path" , path); + +jw_object_inline_begin_array (jw,"stages" ); +for (i =0 ; i <3 ; i++) { +jw_array_inline_begin_object (jw); +jw_object_filemode (jw,"mode" , ui->mode[i]); +jw_object_string (jw,"oid" ,oid_to_hex (&ui->oid[i])); +jw_end (jw); + } +jw_end (jw); +} + +struct string_list *resolve_undo_read (const char *data,unsigned long size, +struct json_writer *jw) {struct string_list *resolve_undo;size_t len;
@@ -59,6 +83,7 @@ struct string_list *resolve_undo_read(const char *data, unsigned long size) resolve_undo =xcalloc (1 ,sizeof (*resolve_undo)); resolve_undo->strdup_strings =1 ; +jw_object_inline_begin_array_gently (jw,"entries" );while (size) {struct string_list_item *lost;
@@ -94,7 +119,10 @@ struct string_list *resolve_undo_read(const char *data, unsigned long size) size -= rawsz; data += rawsz; } + +dump_resolve_undo (jw, lost->string, ui); } +jw_end_gently (jw);return resolve_undo; error:
resolve-undo.h +3 -1
@@ -3,6 +3,8 @@#include "cache.h" +struct json_writer ; +struct resolve_undo_info {unsigned int mode[3 ];struct object_id oid[3 ];
@@ -10,7 +12,7 @@ struct resolve_undo_info {void record_resolve_undo (struct index_state *,struct cache_entry *);void resolve_undo_write (struct strbuf *,struct string_list *); -struct string_list *resolve_undo_read (const char *,unsigned long ); +struct string_list *resolve_undo_read (const char *,unsigned long ,struct json_writer *);void resolve_undo_clear_index (struct index_state *);int unmerge_index_entry_at (struct index_state *,int );void unmerge_index (struct index_state *,const struct pathspec *);
t/t3011-ls-files-json.sh +20 -0
@@ -70,4 +70,24 @@ test_expect_success 'ls-files --json, fsmonitor extension ' ' ) ' +test_expect_success 'ls-files --json, rerere extension' ' + git init rerere && + ( + cd rerere && + mkdir fi && + test_commit initial fi/le first && + git branch side && + test_commit second fi/le second && + git checkout side && + test_commit third fi/le third && + git checkout master && + git config rerere.enabled true && + test_must_fail git merge side && + echo resolved >fi/le && + git add fi/le && + cp ../filter.sed . && + compare_json rerere + ) +' + test_done
t/t3011/rerere new file +66 -0
66 lines not on this page · raw message
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
PATCH v2 9/10 read-cache.c: dump "IEOT" extension as json
In reply to Nguyễn Thái Ngọc Duy--- read-cache.c | 43 ++++++++++++++++++++++++++++++++++++------- t/t3011/eoie | 13 ++++++++++++- 2 files changed, 48 insertions(+), 8 deletions(-)
read-cache.c +36 -7
@@ -1693,6 +1693,7 @@ static int verify_hdr(const struct cache_header *hdr, unsigned long size)return 0 ; } +static struct index_entry_offset_table *do_read_ieot_extension (struct index_state *,const char *,uint32_t );static int read_index_extension (struct index_state *istate,const char *map,unsigned long *offset)
@@ -1740,7 +1741,11 @@ static int read_index_extension(struct index_state *istate,/* already handled in do_read_index() */ break ;case CACHE_EXT_INDEXENTRYOFFSETTABLE : -/* already handled in do_read_index() */ +if (istate->jw) { +free (do_read_ieot_extension (istate, data, sz)); + }else { +/* already handled in do_read_index() */ + }break ;default :if (*ext <'A' ||'Z' < *ext)
@@ -1938,7 +1943,7 @@ struct index_entry_offset_tablestruct index_entry_offset entries[FLEX_ARRAY ]; }; -static struct index_entry_offset_table *read_ieot_extension (const char *mmap,size_t mmap_size,size_t offset); +static struct index_entry_offset_table *read_ieot_extension (struct index_state *istate,const char *mmap,size_t mmap_size,size_t offset);static void write_ieot_extension (struct strbuf *sb,struct index_entry_offset_table *ieot);static size_t read_eoie_extension (const char *mmap,size_t mmap_size);
@@ -2292,7 +2297,7 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist)* to multi-thread the reading of the cache entries. */ if (extension_offset && nr_threads >1 ) - ieot =read_ieot_extension (mmap, mmap_size, extension_offset); + ieot =read_ieot_extension (istate, mmap, mmap_size, extension_offset);if (ieot) { src_offset +=load_cache_entries_threaded (istate, mmap, mmap_size, nr_threads, ieot);
@@ -3634,12 +3639,13 @@ static void write_eoie_extension(struct strbuf *sb, git_hash_ctx *eoie_context,#define IEOT_VERSION (1) -static struct index_entry_offset_table *read_ieot_extension (const char *mmap,size_t mmap_size,size_t offset) +static struct index_entry_offset_table *read_ieot_extension ( +struct index_state *istate, +const char *mmap,size_t mmap_size, +size_t offset) {const char *index =NULL ; -uint32_t extsize, ext_version; -struct index_entry_offset_table *ieot; -int i, nr; +uint32_t extsize;/* find the IEOT extension */ if (!offset)
@@ -3655,6 +3661,17 @@ static struct index_entry_offset_table *read_ieot_extension(const char *mmap, si }if (!index)return NULL ; +return do_read_ieot_extension (istate, index, extsize); +} + +static struct index_entry_offset_table *do_read_ieot_extension ( +struct index_state *istate, +const char *index, +uint32_t extsize) +{ +uint32_t ext_version; +struct index_entry_offset_table *ieot; +int i, nr;/* validate the version is IEOT_VERSION */ ext_version =get_be32 (index);
@@ -3670,6 +3687,10 @@ static struct index_entry_offset_table *read_ieot_extension(const char *mmap, sierror ("invalid number of IEOT entries %d" , nr);return NULL ; } +if (istate->jw) { +jw_object_intmax (istate->jw,"version" , ext_version); +jw_object_inline_begin_array (istate->jw,"entries" ); + } ieot =xmalloc (sizeof (struct index_entry_offset_table ) + (nr *sizeof (struct index_entry_offset ))); ieot->nr = nr;
@@ -3678,7 +3699,15 @@ static struct index_entry_offset_table *read_ieot_extension(const char *mmap, si index +=sizeof (uint32_t ); ieot->entries[i].nr =get_be32 (index); index +=sizeof (uint32_t ); + +if (istate->jw) { +jw_array_inline_begin_object (istate->jw); +jw_object_intmax (istate->jw,"offset" , ieot->entries[i].offset); +jw_object_intmax (istate->jw,"count" , ieot->entries[i].nr); +jw_end (istate->jw); + } } +jw_end_gently (istate->jw);return ieot; }
t/t3011/eoie +12 -1
@@ -84,7 +84,18 @@ "extensions": { "IEOT": { "file_offset": <number>, - "ext_size": <number> + "ext_size": <number>, + "version": 1, + "entries": [ + { + "offset": <number>, + "count": 2 + }, + { + "offset": <number>, + "count": 2 + } + ] }, "EOIE": { "file_offset": <number>,
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
PATCH v2 8/10 read-cache.c: dump "EOIE" extension as json
In reply to Nguyễn Thái Ngọc Duy--- read-cache.c | 8 ++++ t/t3011-ls-files-json.sh | 13 ++++++ t/t3011/eoie (new) | 96 ++++++++++++++++++++++++++++++++++++++++ t/test-lib.sh | 4 ++ 4 files changed, 121 insertions(+)
read-cache.c +8 -0
@@ -1731,6 +1731,14 @@ static int read_index_extension(struct index_state *istate,read_fsmonitor_extension (istate, data, sz);break ;case CACHE_EXT_ENDOFINDEXENTRIES : +if (istate->jw) { +/* must be synchronized with read_eoie_extension() */ +jw_object_intmax (istate->jw,"offset" ,get_be32 (data)); +jw_object_string (istate->jw,"oid" , +hash_to_hex ((const unsigned char *)data +sizeof (uint32_t ))); + } +/* already handled in do_read_index() */ +break ;case CACHE_EXT_INDEXENTRYOFFSETTABLE :/* already handled in do_read_index() */ break ;
t/t3011-ls-files-json.sh +13 -0
@@ -90,4 +90,17 @@ test_expect_success 'ls-files --json, rerere extension' ' ) ' +test_expect_success !SINGLE_CPU 'ls-files --json and multicore extensions' ' + git init eoie && + ( + cd eoie && + git config index.threads 2 && + touch one two three four && + git add . && + cp ../filter.sed . && + strip_number offset && + compare_json eoie + ) +' + test_done
t/t3011/eoie new file +96 -0
96 lines not on this page · raw message
t/test-lib.sh +4 -0
@@ -1611,3 +1611,7 @@ test_lazy_prereq REBASE_P ' test_lazy_prereq FAIL_PREREQS ' test -n "$GIT_TEST_FAIL_PREREQS" ' + +test_lazy_prereq SINGLE_CPU ' + test "$(test-tool online-cpus)" -eq 1 +'
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
PATCH v2 10/10 t3008: use the new SINGLE_CPU prereq
In reply to Nguyễn Thái Ngọc Duy--- t/t3008-ls-files-lazy-init-name-hash.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-)
t/t3008-ls-files-lazy-init-name-hash.sh +1 -7
@@ -4,15 +4,9 @@ test_description='Test the lazy init name hash with various folder structures' . ./test-lib.sh -if test 1 -eq $($GIT_BUILD_DIR/t/helper/test-tool online-cpus) -then - skip_all='skipping lazy-init tests, single cpu' - test_done -fi - LAZY_THREAD_COST=2000 -test_expect_success 'no buffer overflow in lazy_init_name_hash' ' +test_expect_success !SINGLE_CPU 'no buffer overflow in lazy_init_name_hash' ' ( test_seq $LAZY_THREAD_COST | sed "s/^/a_/" && echo b/b/b &&
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
Hi Duy,
Quoted 3 lines · Nguyễn Thái Ngọc Duy
- json field names now use '_' instead of '.' to be friendlier to some languages. I stick to
underscore_nameinstead of camelCase because the former is closer to what we use
This is not a good reason. People who are used to read JSON will stumble over this all the time because it is so uncommon.
Quoted 7 lines
- extension location is printed, in case you need to decode the extension by yourself (previously only the size is printed)
- all extensions are printed in the same order they appear in the file (previously eoie and ieot are printed first because that's how we parse)
- resolve undo extension is reorganized a bit to be easier to read
- tests added. Example json files are in t/t3011
It might actually make sense to optionally disable showing extensions.
You also forgot to mention that you explicitly disable handling <pathspec>, which I find a bit odd, personally, as that would probably come in real handy at times, especially when we offer this as a better way for 3rd-party applications to interact with Git (which I think will be the use case for this feature that will be _far_ more common than using it for debugging).
Ciao,
Dscho
Quoted 10 lines · Johannes Schindelin
Hi Duy,
Quoted 3 lines · Nguyễn Thái Ngọc Duy
- json field names now use '_' instead of '.' to be friendlier to some languages. I stick to
underscore_nameinstead of camelCase because the former is closer to what we useThis is not a good reason. People who are used to read JSON will stumble over this all the time because it is so uncommon.
Getting rid of "." and "-" in field names is the important part. These confuse some languages and make us drop into object["<field>"] syntax in my experience.
As for "_" or camelCase (or PascalCase), I'm not sure it matters one way or the other. Personally, I'd vote for underscores.
Jeff
Re: [PATCH v2 01/10] ls-files: add --json to dump the index
In reply to Nguyễn Thái Ngọc DuyQuoted 15 lines · Nguyễn Thái Ngọc Duy
So far we don't have a command to basically dump the index file out, with all its glory details. Checking some info, for example, stat time, usually involves either writing new code or firing up "xxd" and decoding values by yourself.
This --json is supposed to help that. It dumps the index in a human readable format but also easy to be processed with tools. And it will print almost enough info to reconstruct the index later.
In this patch we only dump the main part, not extensions. But at the end of the series, the entire index is dumped. The end result could be very verbose even on a small repository such as git.git.
Signed-off-by Nguyễn Thái Ngọc Duy <[email protected]>---
...
Quoted 6 lines
json-writer.c +0
-0@@ -202,6 +202,28 @@ void jw_object_null(struct json_writer *jw, const char *key)strbuf_addstr (&jw->json,"null" );
json-writer.c void jw_object_null(struct json_writer *jw, const char *key) · line 212
Quoted 10 lines
json-writer.c +8
-0} +void jw_object_filemode (struct json_writer *jw,const char *key,mode_t mode) +{ +object_common (jw, key); +strbuf_addf (&jw->json,"\"%06o\"" , mode); +} + +void jw_object_stat_data (struct json_writer *jw,const char *name, +const struct stat_data *sd)
Should this be in json_writer.c or in read-cache.c ? Currently, json_writer.c is concerned with formatting JSON on basic/scalar types. Do we want to start extending it to handle arbitrary structures? Or would it be better for the code that defines/manipulates the structure to define a "stat_data_dump_json()" function.
I'm torn on the jw_object_filemode() function, JSON format limits us to decimal integers and there are places where I'd like to have hex, or in this case octal values.
I'm thinking it'd be better to have a helper function in read-cache.c that formats a local strbuf and calls js_object_string(&jw, key, buf);
json-writer.c void jw_object_null(struct json_writer *jw, const char *key) · line 218
Quoted 6 lines
json-writer.c +6
-0+{ +jw_object_inline_begin_object (jw, name); +jw_object_intmax (jw,"ctime_sec" , sd->sd_ctime.sec); +jw_object_intmax (jw,"ctime_nsec" , sd->sd_ctime.nsec); +jw_object_intmax (jw,"mtime_sec" , sd->sd_mtime.sec); +jw_object_intmax (jw,"mtime_nsec" , sd->sd_mtime.nsec);
It'd be nice if we could also have a formatted date for the mtime and ctime in addition to the integer values. (I'm not sure whether you'd always want them or make it a verbose option.)
Quoted 85 lines
json-writer.c +8
-0+jw_object_intmax (jw,"device" , sd->sd_dev); +jw_object_intmax (jw,"inode" , sd->sd_ino); +jw_object_intmax (jw,"uid" , sd->sd_uid); +jw_object_intmax (jw,"gid" , sd->sd_gid); +jw_object_intmax (jw,"size" , sd->sd_size); +jw_end (jw); +} +static void increase_indent (struct strbuf *sb,const struct json_writer *jw,int indent)json-writer.h +23
-0@@ -42,8 +42,11 @@* of the given strings. */ +#include "git-compat-util.h" #include "strbuf.h" +struct stat_data ; +struct json_writer {/* @@ -81,6 +84,9 @@ void jw_object_true(struct json_writer *jw, const char *key);void jw_object_false (struct json_writer *jw,const char *key);void jw_object_bool (struct json_writer *jw,const char *key,int value);void jw_object_null (struct json_writer *jw,const char *key); +void jw_object_filemode (struct json_writer *jw,const char *key,mode_t value); +void jw_object_stat_data (struct json_writer *jw,const char *key, +const struct stat_data *sd);void jw_object_sub_jw (struct json_writer *jw,const char *key,const struct json_writer *value);@@ -104,4 +110,21 @@ void jw_array_inline_begin_array(struct json_writer *jw);int jw_is_terminated (const struct json_writer *jw);void jw_end (struct json_writer *jw); +/* +* These _gently versions accept NULL json_writer to reduce too much +* branching at the call site. +*/ +static inline void jw_object_inline_begin_array_gently (struct json_writer *jw, +const char *name) +{ +if (jw) +jw_object_inline_begin_array (jw, name); +} + +static inline void jw_end_gently (struct json_writer *jw) +{ +if (jw) +jw_end (jw); +} +#endif /* JSON_WRITER_H */ read-cache.c +11
-0@@ -25,6 +25,7 @@#include "fsmonitor.h" #include "thread-utils.h" #include "progress.h" +#include "json-writer.h" /* Mask for the name length in ce_flags in the on-disk index */ @@ -1952,6 +1953,49 @@ static void *load_index_extensions(void *_data)return NULL ; } +static void dump_cache_entry (struct index_state *istate, +int index, +unsigned long offset, +const struct cache_entry *ce) +{ +struct json_writer *jw = istate->jw; + +jw_array_inline_begin_object (jw); + +/*
read-cache.c static void *load_index_extensions(void *_data) · line 1975
Quoted 10 lines
read-cache.c +10
-0+* this is technically redundant, but it's for easier +* navigation when there hundreds of entries +*/ +jw_object_intmax (jw,"id" , index); + +jw_object_string (jw,"name" , ce->name); + +jw_object_filemode (jw,"mode" , ce->ce_mode); + +jw_object_intmax (jw,"flags" , ce->ce_flags);
It would be nice to have the flags as a hex-formatted string in addition to (or instead of) the decimal integer value.
Quoted 72 lines
read-cache.c +33
-0+/* +* again redundant info, just so you don't have to decode +* flags values manually +*/ +if (ce->ce_flags &CE_EXTENDED ) +jw_object_true (jw,"extended_flags" ); +if (ce->ce_flags &CE_VALID ) +jw_object_true (jw,"assume_unchanged" ); +if (ce->ce_flags &CE_INTENT_TO_ADD ) +jw_object_true (jw,"intent_to_add" ); +if (ce->ce_flags &CE_SKIP_WORKTREE ) +jw_object_true (jw,"skip_worktree" ); +if (ce_stage (ce)) +jw_object_intmax (jw,"stage" ,ce_stage (ce)); + +jw_object_string (jw,"oid" ,oid_to_hex (&ce->oid)); + +jw_object_stat_data (jw,"stat" , &ce->ce_stat_data); +jw_object_intmax (jw,"file_offset" , offset); + +jw_end (jw); +} +/* * A helper function that will load the specified range of cache entries * from the memory mapped file and add them to the given index. @@ -1972,6 +2016,9 @@ static unsigned long load_cache_entry_block(struct index_state *istate, ce =create_from_disk (ce_mem_pool, istate->version, disk_ce, &consumed, previous_ce);set_index_entry (istate, i, ce); +if (istate->jw) +dump_cache_entry (istate, i, src_offset, ce); + src_offset += consumed; previous_ce = ce; }@@ -1983,6 +2030,8 @@ static unsigned long load_all_cache_entries(struct index_state *istate, {unsigned long consumed; +jw_object_inline_begin_array_gently (istate->jw,"entries" ); +if (istate->version ==4 ) {mem_pool_init (&istate->ce_mem_pool,estimate_cache_size_from_compressed (istate->cache_nr));@@ -1993,6 +2042,8 @@ static unsigned long load_all_cache_entries(struct index_state *istate, consumed =load_cache_entry_block (istate, istate->ce_mem_pool,0 , istate->cache_nr, mmap, src_offset,NULL ); + +jw_end_gently (istate->jw);return consumed; }@@ -2120,6 +2171,7 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist)size_t extension_offset =0 ;int nr_threads, cpus;struct index_entry_offset_table *ieot =NULL ; +int jw_pretty =1 ;if (istate->initialized)return istate->cache_nr;@@ -2154,6 +2206,8 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) istate->cache_nr =ntohl (hdr->hdr_entries); istate->cache_alloc =alloc_nr (istate->cache_nr); istate->cache =xcalloc (istate->cache_alloc,sizeof (*istate->cache)); + istate->timestamp.sec = st.st_mtime; + istate->timestamp.nsec =ST_MTIME_NSEC (st); istate->initialized =1 ; p.istate = istate;
read-cache.c int do_read_index(struct index_state *istate, const char *path, int must_exist) · line 2238
Quoted 9 lines
read-cache.c +6
-0@@ -2176,6 +2230,20 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist)if (!HAVE_THREADS ) nr_threads =1 ; +if (istate->jw) { +jw_object_begin (istate->jw, jw_pretty); +jw_object_intmax (istate->jw,"version" , istate->version); +jw_object_string (istate->jw,"oid" ,oid_to_hex (&istate->oid)); +jw_object_intmax (istate->jw,"mtime_sec" , istate->timestamp.sec); +jw_object_intmax (istate->jw,"mtime_nsec" , istate->timestamp.nsec);
again, it would be nice to also have a formated version of the mtime.
read-cache.c int do_read_index(struct index_state *istate, const char *path, int must_exist) · line 2244
Quoted 6 lines
read-cache.c +6
-0+ +/* +* Threading may mess up json writing. This is for +* debugging only, so performance is not a concern. +*/ + nr_threads =1 ;
yes. we should turn off threading when dumping to json.
Quoted 16 lines
read-cache.c +2
-2+ } +if (nr_threads >1 ) { extension_offset =read_eoie_extension (mmap, mmap_size);if (extension_offset) {@@ -2204,8 +2272,6 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) src_offset +=load_all_cache_entries (istate, mmap, mmap_size, src_offset); } - istate->timestamp.sec = st.st_mtime; - istate->timestamp.nsec =ST_MTIME_NSEC (st);/* if we created a thread, join it otherwise load the extensions on the primary thread */ if (extension_offset) {
read-cache.c int do_read_index(struct index_state *istate, const char *path, int must_exist) · line 2289
Quoted 8 lines
read-cache.c +2
-0@@ -2216,6 +2282,8 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist) p.src_offset = src_offset;load_index_extensions (&p); } +jw_end_gently (istate->jw); +munmap ((void *)mmap, mmap_size);/*
...
Thanks
Jeff
Re: [PATCH v2 04/10] dir.c: dump "UNTR" extension as json
In reply to Nguyễn Thái Ngọc DuyQuoted 27 lines · Nguyễn Thái Ngọc Duy
The big part of UNTR extension is dumped at the end instead of dumping as soon as we read it, because we actually "patch" some fields in
untracked_cache_dirwith EWAH bitmaps at the end.Signed-off-by Nguyễn Thái Ngọc Duy <[email protected]>--- dir.c | 57 +++++++++++++++++++++++++++++++++++++++- dir.h | 4 ++- json-writer.h | 6 +++++ read-cache.c | 2 +- t/t3011-ls-files-json.sh | 3 ++- t/t3011/basic | 39 +++++++++++++++++++++++++++ 6 files changed, 107 insertions(+), 4 deletions(-)dir.c +1
-0@@ -19,6 +19,7 @@#include "varint.h" #include "ewah/ewok.h" #include "fsmonitor.h" +#include "json-writer.h" #include "submodule-config.h" /*
dir.c static void load_oid_stat(struct oid_stat *oid_stat, const unsigned char *data, · line 2831
Quoted 6 lines
dir.c +2
-1@@ -2826,7 +2827,42 @@ static void load_oid_stat(struct oid_stat *oid_stat, const unsigned char *data, oid_stat->valid =1 ; } -struct untracked_cache *read_untracked_extension (const void *data,unsigned long sz) +static void jw_object_oid_stat (struct json_writer *jw,const char *key, +const struct oid_stat *oid_stat)
I was hoping to reserve the jw_, jw_object_, and jw_array_ prefixes to refer to public functions in the json-writer.[ch] files, rather than private functions elsewhere in the code. I think it is less confusing that way. IMHO.
It's fine to have such helper functions for local structure types, but I think it'd be better for them to not be named like other public functions.
Quoted 99 lines
dir.c +53
-0+{ +jw_object_inline_begin_object (jw, key); +jw_object_bool (jw,"valid" , oid_stat->valid); +jw_object_string (jw,"oid" ,oid_to_hex (&oid_stat->oid)); +jw_object_stat_data (jw,"stat" , &oid_stat->stat); +jw_end (jw); +} + +static void jw_object_untracked_cache_dir (struct json_writer *jw, +const struct untracked_cache_dir *ucd) +{ +int i; + +jw_object_bool (jw,"valid" , ucd->valid); +jw_object_bool (jw,"check-only" , ucd->check_only); +jw_object_stat_data (jw,"stat" , &ucd->stat_data); +jw_object_string (jw,"exclude-oid" ,oid_to_hex (&ucd->exclude_oid)); +jw_object_inline_begin_array (jw,"untracked" ); +for (i =0 ; i < ucd->untracked_nr; i++) +jw_array_string (jw, ucd->untracked[i]); +jw_end (jw); + +jw_object_inline_begin_object (jw,"dirs" ); +for (i =0 ; i < ucd->dirs_nr; i++) { +jw_object_inline_begin_object (jw, ucd->dirs[i]->name); +jw_object_untracked_cache_dir (jw, ucd->dirs[i]); +jw_end (jw); + } +jw_end (jw); +} + +struct untracked_cache *read_untracked_extension (const void *data, +unsigned long sz, +struct json_writer *jw) {struct untracked_cache *uc;struct read_data rd;@@ -2864,6 +2900,19 @@ struct untracked_cache *read_untracked_extension(const void *data, unsigned long uc->dir_flags =get_be32 (next +ouc_offset (dir_flags)); exclude_per_dir = (const char *)next + exclude_per_dir_offset; uc->exclude_per_dir =xstrdup (exclude_per_dir); + +if (jw) { +jw_object_string (jw,"ident" , ident); +jw_object_oid_stat (jw,"info_exclude" , &uc->ss_info_exclude); +jw_object_oid_stat (jw,"excludes_file" , &uc->ss_excludes_file); +jw_object_intmax (jw,"flags" , uc->dir_flags); +if (uc->dir_flags &DIR_SHOW_OTHER_DIRECTORIES ) +jw_object_bool (jw,"show_other_directories" ,1 ); +if (uc->dir_flags &DIR_HIDE_EMPTY_DIRECTORIES ) +jw_object_bool (jw,"hide_empty_directories" ,1 ); +jw_object_string (jw,"excludes_per_dir" , uc->exclude_per_dir); + } +/* NUL after exclude_per_dir is covered by sizeof(*ouc) */ next += exclude_per_dir_offset +strlen (exclude_per_dir) +1 ;if (next >= end)@@ -2905,6 +2954,12 @@ struct untracked_cache *read_untracked_extension(const void *data, unsigned longewah_each_bit (rd.sha1_valid, read_oid, &rd); next = rd.data; +if (jw) { +jw_object_inline_begin_object (jw,"root" ); +jw_object_untracked_cache_dir (jw, uc->root); +jw_end (jw); + } + done:free (rd.ucd);ewah_free (rd.valid);dir.h +3
-1@@ -6,6 +6,8 @@#include "cache.h" #include "strbuf.h" +struct json_writer ; +struct dir_entry {unsigned int len;char name[FLEX_ARRAY ];/* more */ @@ -362,7 +364,7 @@ void untracked_cache_remove_from_index(struct index_state *, const char *);void untracked_cache_add_to_index (struct index_state *,const char *);void free_untracked_cache (struct untracked_cache *); -struct untracked_cache *read_untracked_extension (const void *data,unsigned long sz); +struct untracked_cache *read_untracked_extension (const void *data,unsigned long sz,struct json_writer *jw);void write_untracked_extension (struct strbuf *out,struct untracked_cache *untracked);void add_untracked_cache (struct index_state *istate);void remove_untracked_cache (struct index_state *istate);json-writer.h +0
-0@@ -121,6 +121,12 @@ static inline void jw_object_inline_begin_array_gently(struct json_writer *jw,jw_object_inline_begin_array (jw, name); }
json-writer.h static inline void jw_object_inline_begin_array_gently(struct json_writer *jw, · line 132
Quoted 10 lines
json-writer.h +6
-0+static inline void jw_array_inline_begin_object_gently (struct json_writer *jw) +{ +if (jw) +jw_array_inline_begin_object (jw); +} +static inline void jw_end_gently (struct json_writer *jw) {if (jw)
I'm not sure about the need for these _gently versions, but maybe make them macros in json-writer.h
Jeff
Re: [PATCH v2 01/10] ls-files: add --json to dump the index
In reply to Jeff HostetlerQuoted 8 lines · Jeff Hostetler
On 6/24/2019 9:02 AM, Nguyễn Thái Ngọc Duy wrote:
...json-writer.c void jw_object_null(struct json_writer *jw, const char *key) · line 212
Quoted 2 lines
json-writer.c +2
-0+void jw_object_stat_data (struct json_writer *jw,const char *name, +const struct stat_data *sd)Should this be in
json_writer.cor inread-cache.c? Currently,json_writer.cis concerned with formatting JSON on basic/scalar types.
That's an interesting question.
In the longer run, is it reasonable to assume that the current callers of these functions will stay to be the only places that would want to say "For this path, various attributes like timestamps and other stuff are these values"? The answer is probably no.
Would it be reasonable to assume that the future callers are all likely to be closely related to what functions that are currently in read-cache.c do and would appear only in that file? I think the answer would also be no. Functions in entry.c may want to report "I've created/updated these filesystem entities and their stat info now look like so", for example. So (assuming that we would want to represent "struct stat" the same way no matter who wants to write it out), I think it is reasonable to have this function on the json side, not on Git side.
I do not think it is a bad idea to have a layer that sits above json-writer.c and below read-cache.c that give us standardized mapping from in-core objects (like "struct stat") to objects in json (they are set of key-value pairs after all). Call it json-schema.c or something?
What other "higher than scaler" types do we foresee that we'd need to serialize in a standardised way? If we do not have that many yet, perhaps a split like that might be a bit premature and we can start by having the function in json-writer.c side, not in the API consumer side.
Thanks.
Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json
In reply to Nguyễn Thái Ngọc DuyQuoted 26 lines · Nguyễn Thái Ngọc Duy
Signed-off-by Nguyễn Thái Ngọc Duy <[email protected]>--- json-writer.c | 14 ++++++++++++++ json-writer.h | 3 +++ split-index.c | 9 ++++++++- t/t3011-ls-files-json.sh | 14 ++++++++++++++ t/t3011/split-index (new) | 39 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 78 insertions(+), 1 deletion(-)json-writer.c +4
-0@@ -1,4 +1,5 @@#include "cache.h" +#include "ewah/ewok.h" #include "json-writer.h" void jw_init (struct json_writer *jw)@@ -224,6 +225,19 @@ void jw_object_stat_data(struct json_writer *jw, const char *name,jw_end (jw); } +static void dump_ewah_one (size_t pos,void *jw) +{ +jw_array_intmax (jw, pos);
json-writer.c void jw_object_stat_data(struct json_writer *jw, const char *name, · line 240
Quoted 10 lines
json-writer.c +10
-0+} + +void jw_object_ewah (struct json_writer *jw,const char *key, +struct ewah_bitmap *ewah) +{ +jw_object_inline_begin_array (jw, key); +ewah_each_bit (ewah, dump_ewah_one, jw); +jw_end (jw); +} +
As I said in an earlier commit in this series, I'd prefer that we keep such data-structure-specific helper functions in the source file that defines the data structure rather than adding them to json-writer.[ch]
I'm curious how big these EWAHs will be in practice and how useful an array of integers will be (especially as the pretty format will be one integer per line). Perhaps it would helpful to have an extended example in one of the tests.
Would it be better to have the caller of ewah_each_bit() build a hex or bit string in a strbuf and then write it as a single string?
Jeff
Re: [PATCH v2 01/10] ls-files: add --json to dump the index
In reply to Nguyễn Thái Ngọc DuyQuoted 83 lines · Nguyễn Thái Ngọc Duy
So far we don't have a command to basically dump the index file out, with all its glory details. Checking some info, for example, stat time, usually involves either writing new code or firing up "xxd" and decoding values by yourself.
This --json is supposed to help that. It dumps the index in a human readable format but also easy to be processed with tools. And it will print almost enough info to reconstruct the index later.
In this patch we only dump the main part, not extensions. But at the end of the series, the entire index is dumped. The end result could be very verbose even on a small repository such as git.git.
Signed-off-by Nguyễn Thái Ngọc Duy <[email protected]>--- Documentation/git-ls-files.txt | 5 +++ builtin/ls-files.c | 38 +++++++++++++--- cache.h | 2 + json-writer.c | 22 ++++++++++ json-writer.h | 23 ++++++++++ read-cache.c | 72 ++++++++++++++++++++++++++++++- t/t3011-ls-files-json.sh (new +x) | 44 +++++++++++++++++++ t/t3011/basic (new) | 67 ++++++++++++++++++++++++++++ 8 files changed, 265 insertions(+), 8 deletions(-)[...]
t/t3011-ls-files-json.sh new file +44
-0@@ -0,0 +1,44 @@ +#!/bin/sh + +test_description='ls-files dumping json' + +. ./test-lib.sh + +strip_number() { + for name; do + echo 's/\("'$name'":\) [0-9]\+/\1 <number>/' >>filter.sed + done +} + +strip_string() { + for name; do + echo 's/\("'$name'":\) ".*"/\1 <string>/' >>filter.sed + done +} + +compare_json() { + git ls-files --debug-json >json && + sed -f filter.sed json >filtered && + test_cmp "$TEST_DIRECTORY"/t3011/"$1" filtered +} + +test_expect_success 'setup' ' + mkdir sub && + echo one >one && + git add one && + echo 2 >sub/two && + git add sub/two && + + echo intent-to-add >ita && + git add -N ita && + + strip_number ctime_sec ctime_nsec mtime_sec mtime_nsec && + strip_number device inode uid gid file_offset ext_size && + strip_string oid ident +' + +test_expect_success 'ls-files --json, main entries' ' + compare_json basic +' + +test_donet/t3011/basic new file +0
-0
t/t3011/basic line 2
Quoted 2 lines
t/t3011/basic new file +2
-0@@ -0,0 +1,67 @@ +{ + "version": 3,
This will break the test suite when 'GIT_TEST_INDEX_VERSION' is set to 4 for example. I think this applies to a few other tests in later patches as well.
Quoted 68 lines
t/t3011/basic new file +65
-0+ "oid": <string>, + "mtime_sec": <number>, + "mtime_nsec": <number>, + "entries": [ + { + "id": 0, + "name": "ita", + "mode": "100644", + "flags": 536887296, + "extended_flags": true, + "intent_to_add": true, + "oid": <string>, + "stat": { + "ctime_sec": <number>, + "ctime_nsec": <number>, + "mtime_sec": <number>, + "mtime_nsec": <number>, + "device": <number>, + "inode": <number>, + "uid": <number>, + "gid": <number>, + "size": 0 + }, + "file_offset": <number> + }, + { + "id": 1, + "name": "one", + "mode": "100644", + "flags": 0, + "oid": <string>, + "stat": { + "ctime_sec": <number>, + "ctime_nsec": <number>, + "mtime_sec": <number>, + "mtime_nsec": <number>, + "device": <number>, + "inode": <number>, + "uid": <number>, + "gid": <number>, + "size": 4 + }, + "file_offset": <number> + }, + { + "id": 2, + "name": "sub/two", + "mode": "100644", + "flags": 0, + "oid": <string>, + "stat": { + "ctime_sec": <number>, + "ctime_nsec": <number>, + "mtime_sec": <number>, + "mtime_nsec": <number>, + "device": <number>, + "inode": <number>, + "uid": <number>, + "gid": <number>, + "size": 2 + }, + "file_offset": <number> + } + ] +}Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
Quoted 8 lines · Johannes Schindelin
Quoted 7 lines
- extension location is printed, in case you need to decode the extension by yourself (previously only the size is printed)
- all extensions are printed in the same order they appear in the file (previously eoie and ieot are printed first because that's how we parse)
- resolve undo extension is reorganized a bit to be easier to read
- tests added. Example json files are in t/t3011
Quoted 5 lines
It might actually make sense to optionally disable showing extensions.
You also forgot to mention that you explicitly disable handling
<pathspec>, which I find a bit odd, personally, as that would probably come in real handy at times,
No. I mentioned the land of high level languages before. Filtering in any Python, Ruby, Scheme, JavaScript, Java is a piece of cake and much more flexible than pathspec. Even with shell scripts, jq could do a much better job than pathspec. If you filter by pathspec, good luck trying that on extensions.
It's the same reason why I will not provide a flexible way to disable extensions. I'm not starting a JSON API for Git. I provide an index file in JSON format. You do what you want with it. You have a format easy enough to import to native data structures of your favorite language.
Quoted 4 lines
especially when we offer this as a better way
for 3rd-party applications to interact with Git (which I think will be the use case for this feature that will be _far_ more common than using it for debugging).
We may have conflicting goals. For me, first priority is the debug tool for Git developers. 3rd-party support is a stretch. I could move all this back to test-tool, then you can provide a 3rd-party API if you want. Or I'll withdraw this series and go back to my original plan.
Signature · 1 line
Duy
Re: [PATCH v2 01/10] ls-files: add --json to dump the index
In reply to Jeff HostetlerHi Jeff,
Quoted 13 lines · Jeff Hostetler
On 6/24/2019 9:02 AM, Nguyễn Thái Ngọc Duy wrote:
[...]json-writer.c void jw_object_null(struct json_writer *jw, const char *key) · line 218
Quoted 6 lines
json-writer.c +6
-0+{ +jw_object_inline_begin_object (jw, name); +jw_object_intmax (jw,"ctime_sec" , sd->sd_ctime.sec); +jw_object_intmax (jw,"ctime_nsec" , sd->sd_ctime.nsec); +jw_object_intmax (jw,"mtime_sec" , sd->sd_mtime.sec); +jw_object_intmax (jw,"mtime_nsec" , sd->sd_mtime.nsec);It'd be nice if we could also have a formatted date for the mtime and ctime in addition to the integer values. (I'm not sure whether you'd always want them or make it a verbose option.)
It would be more consistent with JSON conventions to have only the formatted date, preferably in ISO-8601 format [*1*], I would think.
And for debugging, it would also make more sense, in my opinion.
Quoted 32 lines
[...]
Quoted 31 lines
read-cache.c +16
-0@@ -25,6 +25,7 @@#include "fsmonitor.h" #include "thread-utils.h" #include "progress.h" +#include "json-writer.h" /* Mask for the name length in ce_flags in the on-disk index */ @@ -1952,6 +1953,49 @@ static void *load_index_extensions(void *_data)return NULL ; } +static void dump_cache_entry (struct index_state *istate, +int index, +unsigned long offset, +const struct cache_entry *ce) +{ +struct json_writer *jw = istate->jw; + +jw_array_inline_begin_object (jw); + +/* +* this is technically redundant, but it's for easier +* navigation when there hundreds of entries +*/ +jw_object_intmax (jw,"id" , index); +
Quoted 8 lines
read-cache.c static void *load_index_extensions(void *_data) · line 1975
Quoted 5 lines
read-cache.c +5
-0+jw_object_string (jw,"name" , ce->name); + +jw_object_filemode (jw,"mode" , ce->ce_mode); + +jw_object_intmax (jw,"flags" , ce->ce_flags);It would be nice to have the flags as a hex-formatted string in addition to (or instead of) the decimal integer value.
Instead of, please, prefixed with "0x" to make it clear that this is hex.
And "mode" in octal, please, prefixed with "0", as that is the convention to display file modes.
Thanks,
Dscho
Footnote *1*: It is too bad that JSON leaves the exact date format unspecified, leading to a lot of inconsistency and putting the burden on parsers:
https://stackoverflow.com/questions/10286204/the-right-json-date-format
Quoted 7 lines · Duy Nguyen
Quoted 4 lines · Johannes Schindelin
especially when we offer this as a better way
for 3rd-party applications to interact with Git (which I think will be the use case for this feature that will be _far_ more common than using it for debugging).
Quoted 5 lines
We may have conflicting goals. For me, first priority is the debug tool for Git developers. 3rd-party support is a stretch. I could move all this back to test-tool, then you can provide a 3rd-party API if you want. Or I'll withdraw this series and go back to my original plan.
FWIW, I am very much in favor of this series, and would have found something like this very useful many times in the past when I was digging into the index code. So I'd be more than happy to just have this as debug tool, rather than as 3rd-party API.
I'd also be fine with this living in test-tool, as long as it's somewhere in git.git where it's easily usable, I'd find it helpful.
Thanks for working on this!
Re: [PATCH v2 01/10] ls-files: add --json to dump the index
In reply to Nguyễn Thái Ngọc DuyHi Duy,
Quoted 6 lines · Nguyễn Thái Ngọc Duy
t/t3011-ls-files-json.sh new file +0
-0
t/t3011-ls-files-json.sh line 9
Quoted 9 lines
t/t3011-ls-files-json.sh new file +9
-0@@ -0,0 +1,44 @@ +#!/bin/sh + +test_description='ls-files dumping json' + +. ./test-lib.sh + +strip_number() { + for name; do + echo 's/\("'$name'":\) [0-9]\+/\1 <number>/' >>filter.sed
This does not do what you think it does, in Ubuntu Xenial and on macOS:
The \1 is expanded to the ASCII character 001. Therefore your test cases fail on almost all platforms.
Funnily enough, they pass on Windows...
Ciao,
Johannes
Re: [PATCH v2 01/10] ls-files: add --json to dump the index
In reply to Jeff HostetlerQuoted 20 lines · Jeff Hostetler
json-writer.c void jw_object_null(struct json_writer *jw, const char *key) · line 212
Quoted 12 lines
json-writer.c +8
-0@@ -202,6 +202,28 @@ void jw_object_null(struct json_writer *jw, const char *key)strbuf_addstr (&jw->json,"null" ); } +void jw_object_filemode (struct json_writer *jw,const char *key,mode_t mode) +{ +object_common (jw, key); +strbuf_addf (&jw->json,"\"%06o\"" , mode); +} + +void jw_object_stat_data (struct json_writer *jw,const char *name, +const struct stat_data *sd)Should this be in
json_writer.cor inread-cache.c? Currently,json_writer.cis concerned with formatting JSON on basic/scalar types. Do we want to start extending it to handle arbitrary structures? Or would it be better for the code that defines/manipulates the structure to define a "stat_data_dump_json()" function.
Quoted 7 lines
I'm torn on the
jw_object_filemode()function, JSON format limits us to decimal integers and there are places where I'd like to have hex, or in this case octal values.I'm thinking it'd be better to have a helper function in
read-cache.cthat formats a local strbuf and callsjs_object_string(&jw, key, buf);
I can move these back to read-cache.c. Though if we have a lot more jw helpers like this (hard to tell at the moment) then perhaps we can have json-writer-utils.c or something to group them together. That keep the "boring" code out of main logic code in read-cache.c and other call sites.
Quoted 19 lines
Quoted 19 lines
read-cache.c +15
-0@@ -1952,6 +1953,49 @@ static void *load_index_extensions(void *_data)return NULL ; } +static void dump_cache_entry (struct index_state *istate, +int index, +unsigned long offset, +const struct cache_entry *ce) +{ +struct json_writer *jw = istate->jw; + +jw_array_inline_begin_object (jw); + +/* +* this is technically redundant, but it's for easier +* navigation when there hundreds of entries +*/ +jw_object_intmax (jw,"id" , index); +
Quoted 8 lines
read-cache.c static void *load_index_extensions(void *_data) · line 1975
Quoted 5 lines
read-cache.c +5
-0+jw_object_string (jw,"name" , ce->name); + +jw_object_filemode (jw,"mode" , ce->ce_mode); + +jw_object_intmax (jw,"flags" , ce->ce_flags);It would be nice to have the flags as a hex-formatted string in addition to (or instead of) the decimal integer value.
I'm not against reformatting it in hex string, but is there a value in it? ce_flags is expanded in the code below so that you don't have to decode it yourself when you read each entry. The "flags" field here is for further processing in tools. I'm trying to see if looking at hex values helps, but I'm still not seeing it...
Quoted 14 lines
Quoted 14 lines
read-cache.c +14
-0+/* +* again redundant info, just so you don't have to decode +* flags values manually +*/ +if (ce->ce_flags &CE_EXTENDED ) +jw_object_true (jw,"extended_flags" ); +if (ce->ce_flags &CE_VALID ) +jw_object_true (jw,"assume_unchanged" ); +if (ce->ce_flags &CE_INTENT_TO_ADD ) +jw_object_true (jw,"intent_to_add" ); +if (ce->ce_flags &CE_SKIP_WORKTREE ) +jw_object_true (jw,"skip_worktree" ); +if (ce_stage (ce)) +jw_object_intmax (jw,"stage" ,ce_stage (ce));
Signature · 1 line
Duy
Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json
In reply to Jeff HostetlerQuoted 5 lines · Jeff Hostetler
I'm curious how big these EWAHs will be in practice and how useful an array of integers will be (especially as the pretty format will be one integer per line). Perhaps it would helpful to have an extended example in one of the tests.
It's one integer per updated entry. So if you have a giant index and updated every single one of them, the EWAH bitmap contains that many integers.
If it was easy to just merge these bitmaps back to the entry (e.g. in this example, add "replaced": true to entry zero) I would have done it. But we dump as we stream and it's already too late to do it.
Quoted 3 lines
Would it be better to have the caller of
ewah_each_bit()build a hex or bit string in a strbuf and then write it as a single string?
I don't think the current EWAH representation is easy to read in the first place. You'll probably have to run through some script to update the main entries part and will have a much better view, but that's pretty quick. If it's for scripts, then it's probably best to keep as an array of integers, not a string. Less post processing.
Another reason for not merging to one string (might not be a very good argument though) is to help diff between two indexes. One-number-per-line works well with "git diff --no-index" while one long string is a bit harder. I did this kind of comparison when I made changes in read-cache.c and wanted to check if the new index file is completely broken, or just slighly broken.
Signature · 1 line
Duy
Hi Duy,
Quoted 12 lines · Duy Nguyen
Quoted 10 lines · Johannes Schindelin
Quoted 7 lines
- extension location is printed, in case you need to decode the extension by yourself (previously only the size is printed)
- all extensions are printed in the same order they appear in the file (previously eoie and ieot are printed first because that's how we parse)
- resolve undo extension is reorganized a bit to be easier to read
- tests added. Example json files are in t/t3011It might actually make sense to optionally disable showing extensions.
Quoted 7 lines
Quoted 3 lines
You also forgot to mention that you explicitly disable handling
<pathspec>, which I find a bit odd, personally, as that would probably come in real handy at times,No. I mentioned the land of high level languages before. Filtering in any Python, Ruby, Scheme, JavaScript, Java is a piece of cake and much more flexible than pathspec.
I heard that type of argument before. I was working on the initial Windows port of Git, uh, of course I was working on a big refactoring of a big C++ application backed by a database. A colleague suggested that filtering could be done much better in C++, on the desktop, than in SQL. And so they changed the paradigm to "simplify" the SQL query, and instead dropped the unwanted data after it had hit the RAM of the client machine.
Turns out it was a bad idea. A _really_ bad idea. Because it required downloading 30MB of data for about several dozens computers in parallel, at the start of every shift.
This change was reverted in one big hurry, and the colleague was tasked to learn them some SQL.
Why am I telling you this story? Because you fall into the exact same trap as my colleague.
In this instance, it may not be so much network bandwidth, but it is still quite a suboptimal idea to render JSON for possibly tens of thousands of files, then parse the same JSON on the receiving side, the spend even more time to drop all but a dozen files.
And this is _even more_ relevant when you want to debug things.
In short: I am quite puzzled why this is even debated here. There is a reason, a good reason, why git ls-files accepts pathspecs. I would not want to ignore the lessons of history as willfully here.
Quoted 2 lines
Even with shell scripts, jq could do a much better job than pathspec. If you filter by pathspec, good luck trying that on extensions.
You keep harping on extensions, but the reality of the matter is that they are rarely interesting. I would even wager a bet that we will end up excluding them from the JSON output by default.
Most of the times when I had to decode the index file manually in the past, it was about the regular file entries.
There was *one* week in which I had to decode the untracked cache a bit, to the point where I patched the test helper locally to help me with that.
If my experience in debugging these things is any indicator, extensions do not matter even 10% of the non-extension data.
And that's not even taking into account the third-party software that could definitely benefit from having this JSON format as query result.
In my work as Git for Windows maintainer, I do hear about the needs of third-party software developers quite a bit, so I would claim that I know a bit about what they need, why the NUL-terminated format is not a good match, and how much a JSON-based API would help.
So while that is not your pet, it will be the most useful part of the outcome of your work.
Quoted 5 lines
It's the same reason why I will not provide a flexible way to disable extensions. I'm not starting a JSON API for Git. I provide an index file in JSON format. You do what you want with it. You have a format easy enough to import to native data structures of your favorite language.
I understand that you don't care.
Your patch series is just too good a start on something truly useful to pass up on the opportunity.
Quoted 10 lines
Quoted 4 lines
especially when we offer this as a better way for 3rd-party applications to interact with Git (which I think will be the use case for this feature that will be _far_ more common than using it for debugging).
We may have conflicting goals. For me, first priority is the debug tool for Git developers. 3rd-party support is a stretch. I could move all this back to test-tool, then you can provide a 3rd-party API if you want. Or I'll withdraw this series and go back to my original plan.
You don't need JSON if you want to debug things. That would be a lot of love lost, if debugging was your goal.
I guess I'll wait until your patch series hits next, and then try to find some time to work on that feature.
Ciao,
Johannes
Re: [PATCH v2 01/10] ls-files: add --json to dump the index
In reply to Johannes SchindelinHi Duy,
Quoted 20 lines · Johannes Schindelin
Quoted 6 lines · Nguyễn Thái Ngọc Duy
t/t3011-ls-files-json.sh new file +0
-0t/t3011-ls-files-json.sh line 9
Quoted 9 lines
t/t3011-ls-files-json.sh new file +9
-0@@ -0,0 +1,44 @@ +#!/bin/sh + +test_description='ls-files dumping json' + +. ./test-lib.sh + +strip_number() { + for name; do + echo 's/\("'$name'":\) [0-9]\+/\1 <number>/' >>filter.sedThis does not do what you think it does, in Ubuntu Xenial and on macOS:
Quoted 4 lines
The
\1is expanded to the ASCII character 001. Therefore your test cases fail on almost all platforms.
The strip_number()/strip_string() approach might look elegant from a design perspective, but from a readability perspective (and obviously, when one wants to make those tests more robust and cross-platform), it would be a lot better to do it more explicitly.
This patch on top of your patch series makes the test run correctly in my Linux and Windows setup, and much easier to understand:
-- snipsnap --
t/t3011-ls-files-json.sh +18 -16
@@ -4,18 +4,6 @@ test_description='ls-files dumping json' . ./test-lib.sh -strip_number() { - for name; do - echo 's/\("'$name'":\) [0-9]\+/\1 <number>/' >>filter.sed - done -} - -strip_string() { - for name; do - echo 's/\("'$name'":\) ".*"/\1 <string>/' >>filter.sed - done -} - compare_json() { git ls-files --debug-json >json && sed -f filter.sed json >filtered &&
@@ -35,9 +23,21 @@ test_expect_success 'setup' ' echo intent-to-add >ita && git add -N ita && - strip_number ctime_sec ctime_nsec mtime_sec mtime_nsec && - strip_number device inode uid gid file_offset ext_size last_update && - strip_string oid ident + cat >filter.sed <<-\EOF + s/\("ctime_sec":\) [0-9]\+/\1 <number>/ + s/\("ctime_nsec":\) [0-9]\+/\1 <number>/ + s/\("mtime_sec":\) [0-9]\+/\1 <number>/ + s/\("mtime_nsec":\) [0-9]\+/\1 <number>/ + s/\("device":\) [0-9]\+/\1 <number>/ + s/\("inode":\) [0-9]\+/\1 <number>/ + s/\("uid":\) [0-9]\+/\1 <number>/ + s/\("gid":\) [0-9]\+/\1 <number>/ + s/\("file_offset":\) [0-9]\+/\1 <number>/ + s/\("ext_size":\) [0-9]\+/\1 <number>/ + s/\("last_update":\) [0-9]\+/\1 <number>/ + s/\("oid":\) ".*"/\1 <string>/ + s/\("ident":\) ".*"/\1 <string>/ + EOF ' test_expect_success 'ls-files --json, main entries, UNTR and TREE' '
@@ -98,7 +98,9 @@ test_expect_success !SINGLE_CPU 'ls-files --json and multicore extensions' ' touch one two three four && git add . && cp ../filter.sed . && - strip_number offset && + cat >>filter.sed <<-\EOF && + s/\("offset":\) [0-9]\+/\1 <number>/ + EOF compare_json eoie ) '
Quoted 39 lines · Johannes Schindelin
Hi Duy,
Quoted 12 lines · Duy Nguyen
Quoted 10 lines · Johannes Schindelin
Quoted 7 lines
- extension location is printed, in case you need to decode the extension by yourself (previously only the size is printed)
- all extensions are printed in the same order they appear in the file (previously eoie and ieot are printed first because that's how we parse)
- resolve undo extension is reorganized a bit to be easier to read
- tests added. Example json files are in t/t3011It might actually make sense to optionally disable showing extensions.
Quoted 7 lines
Quoted 3 lines
You also forgot to mention that you explicitly disable handling
<pathspec>, which I find a bit odd, personally, as that would probably come in real handy at times,No. I mentioned the land of high level languages before. Filtering in any Python, Ruby, Scheme, JavaScript, Java is a piece of cake and much more flexible than pathspec.
I heard that type of argument before. I was working on the initial Windows port of Git, uh, of course I was working on a big refactoring of a big C++ application backed by a database. A colleague suggested that filtering could be done much better in C++, on the desktop, than in SQL. And so they changed the paradigm to "simplify" the SQL query, and instead dropped the unwanted data after it had hit the RAM of the client machine.
Turns out it was a bad idea. A _really_ bad idea. Because it required downloading 30MB of data for about several dozens computers in parallel, at the start of every shift.
This change was reverted in one big hurry, and the colleague was tasked to learn them some SQL.
Quoted 7 lines
Why am I telling you this story? Because you fall into the exact same trap as my colleague.
In this instance, it may not be so much network bandwidth, but it is still quite a suboptimal idea to render JSON for possibly tens of thousands of files, then parse the same JSON on the receiving side, the spend even more time to drop all but a dozen files.
This was mentioned before [1]. Of course I don't work on giant index files, but I would assume the cost of parsing JSON (at least with a stream-based one, not loading the whole thing in core) is still cheaper. And you could still do it in iteration, saving every step until you have the reasonable small dataset to work on. The other side of the story is, are we sure parsing and executing pathspec is cheap? I'm not so sure, especially when pathspec code is not exactly optimized.
Consider the amount of code to support something like that. I'd rather wait until a real example come up and no good solution found without modify git.git, before actually supporting it.
[1] https://public-inbox.org/git/[email protected]/
Quoted 5 lines
And this is
_evenmore_relevant when you want to debug things.In short: I am quite puzzled why this is even debated here. There is a reason, a good reason, why
git ls-filesaccepts pathspecs. I would not want to ignore the lessons of history as willfully here.
I guess you and I have different ways of debugging things.
Quoted 7 lines
Quoted 2 lines
Even with shell scripts, jq could do a much better job than pathspec. If you filter by pathspec, good luck trying that on extensions.
You keep harping on extensions, but the reality of the matter is that they are rarely interesting. I would even wager a bet that we will end up excluding them from the JSON output by default.
Quoted 8 lines
Most of the times when I had to decode the index file manually in the past, it was about the regular file entries.
There was *one* week in which I had to decode the untracked cache a bit, to the point where I patched the test helper locally to help me with that.
If my experience in debugging these things is any indicator, extensions do not matter even 10% of the non-extension data.
Again our experiences differ. Mine is mostly about extensions, probably because I had to work on them more often. For normal entries "ls-files --debug" gives you 99% what's in the index file already.
Quoted 5 lines
Quoted 5 lines
Quoted 4 lines
especially when we offer this as a better way for 3rd-party applications to interact with Git (which I think will be the use case for this feature that will be _far_ more common than using it for debugging).
Quoted 8 lines
Quoted 5 lines
We may have conflicting goals. For me, first priority is the debug tool for Git developers. 3rd-party support is a stretch. I could move all this back to test-tool, then you can provide a 3rd-party API if you want. Or I'll withdraw this series and go back to my original plan.
You don't need JSON if you want to debug things. That would be a lot of love lost, if debugging was your goal.
No, I did think of some other line-based format before I ended up with JSON. I did not want to use it in the beginning.
The thing is, a giant table to cover all fields and entries in the main index is not as easy to navigate, or search even in 'less'. And the hierarchical structure of some extensions is hard to represent in good way (at least without writing lots of code). On top of that I still need some easy way to parse and post-process, like how much saving I could gain if I compressed stat data. And the final nail is json-writer.c is already there, much less work.
So JSON was the best option I found to meet all those points.
Signature · 1 line
Duy
Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json
In reply to Duy NguyenQuoted 19 lines · Duy Nguyen
Quoted 5 lines · Jeff Hostetler
I'm curious how big these EWAHs will be in practice and how useful an array of integers will be (especially as the pretty format will be one integer per line). Perhaps it would helpful to have an extended example in one of the tests.
It's one integer per updated entry. So if you have a giant index and updated every single one of them, the EWAH bitmap contains that many integers.
If it was easy to just merge these bitmaps back to the entry (e.g. in this example, add "replaced": true to entry zero) I would have done it. But we dump as we stream and it's already too late to do it.
Quoted 3 lines
Would it be better to have the caller of
ewah_each_bit()build a hex or bit string in a strbuf and then write it as a single string?
Quoted 5 lines
I don't think the current EWAH representation is easy to read in the first place. You'll probably have to run through some script to update the main entries part and will have a much better view, but that's pretty quick. If it's for scripts, then it's probably best to keep as an array of integers, not a string. Less post processing.
I don't think the intent is to dump the EWAH directly, but instead to dump a string of the uncompressed bitmap. Something like:
"delete_bitmap" : "01101101101"
instead of
"delete_bitmap" : [ 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1 ]
Quoted 6 lines
Another reason for not merging to one string (might not be a very good argument though) is to help diff between two indexes. One-number-per-line works well with "git diff --no-index" while one long string is a bit harder. I did this kind of comparison when I made changes in
read-cache.cand wanted to check if the new index file is completely broken, or just slighly broken.
You're right that the diff of the json output is an interesting use, and the "single string" output is not helpful. What about batches of 64-bit strings? For example:
"delete_bitmap" : [ "0101010101010101010101010101010101010101010101010101010101010101", "0101010101010101010101010101010101010101010101010101010101010101", "0101010101010101010101010101010101010101010101010101010101010101", "01010101010101" ]
This could be a happy medium between the two options, but does require some extra work in the formatter.
Thanks,
-Stolee
Re: [PATCH v2 01/10] ls-files: add --json to dump the index
In reply to Johannes SchindelinHi Duy,
Quoted 24 lines · Johannes Schindelin
t/t3011-ls-files-json.sh +0
-12@@ -4,18 +4,6 @@ test_description='ls-files dumping json' . ./test-lib.sh -strip_number() { - for name; do - echo 's/\("'$name'":\) [0-9]\+/\1 <number>/' >>filter.sed - done -} - -strip_string() { - for name; do - echo 's/\("'$name'":\) ".*"/\1 <string>/' >>filter.sed - done -} - compare_json() { git ls-files --debug-json >json && sed -f filter.sed json >filtered &&
t/t3011-ls-files-json.sh test_expect_success 'setup' ' · line 27
Quoted 8 lines
t/t3011-ls-files-json.sh +2
-3@@ -35,9 +23,21 @@ test_expect_success 'setup' ' echo intent-to-add >ita && git add -N ita && - strip_number ctime_sec ctime_nsec mtime_sec mtime_nsec && - strip_number device inode uid gid file_offset ext_size last_update && - strip_string oid ident + cat >filter.sed <<-\EOF + s/\("ctime_sec":\) [0-9]\+/\1 <number>/
And of course, \+ still isn't POSIX, so you have to write [0-9][1-9]* instead.
Ciao,
Johannes
Quoted 27 lines
t/t3011-ls-files-json.sh +16
-1+ s/\("ctime_nsec":\) [0-9]\+/\1 <number>/ + s/\("mtime_sec":\) [0-9]\+/\1 <number>/ + s/\("mtime_nsec":\) [0-9]\+/\1 <number>/ + s/\("device":\) [0-9]\+/\1 <number>/ + s/\("inode":\) [0-9]\+/\1 <number>/ + s/\("uid":\) [0-9]\+/\1 <number>/ + s/\("gid":\) [0-9]\+/\1 <number>/ + s/\("file_offset":\) [0-9]\+/\1 <number>/ + s/\("ext_size":\) [0-9]\+/\1 <number>/ + s/\("last_update":\) [0-9]\+/\1 <number>/ + s/\("oid":\) ".*"/\1 <string>/ + s/\("ident":\) ".*"/\1 <string>/ + EOF ' test_expect_success 'ls-files --json, main entries, UNTR and TREE' '@@ -98,7 +98,9 @@ test_expect_success !SINGLE_CPU 'ls-files --json and multicore extensions' ' touch one two three four && git add . && cp ../filter.sed . && - strip_number offset && + cat >>filter.sed <<-\EOF && + s/\("offset":\) [0-9]\+/\1 <number>/ + EOF compare_json eoie ) '
Hi Duy,
Quoted 47 lines · Duy Nguyen
Quoted 37 lines · Johannes Schindelin
Quoted 12 lines · Duy Nguyen
Quoted 10 lines · Johannes Schindelin
Quoted 7 lines
- extension location is printed, in case you need to decode the extension by yourself (previously only the size is printed)
- all extensions are printed in the same order they appear in the file (previously eoie and ieot are printed first because that's how we parse)
- resolve undo extension is reorganized a bit to be easier to read
- tests added. Example json files are in t/t3011It might actually make sense to optionally disable showing extensions.
Quoted 7 lines
Quoted 3 lines
You also forgot to mention that you explicitly disable handling
<pathspec>, which I find a bit odd, personally, as that would probably come in real handy at times,No. I mentioned the land of high level languages before. Filtering in any Python, Ruby, Scheme, JavaScript, Java is a piece of cake and much more flexible than pathspec.
I heard that type of argument before. I was working on the initial Windows port of Git, uh, of course I was working on a big refactoring of a big C++ application backed by a database. A colleague suggested that filtering could be done much better in C++, on the desktop, than in SQL. And so they changed the paradigm to "simplify" the SQL query, and instead dropped the unwanted data after it had hit the RAM of the client machine.
Turns out it was a bad idea. A _really_ bad idea. Because it required downloading 30MB of data for about several dozens computers in parallel, at the start of every shift.
This change was reverted in one big hurry, and the colleague was tasked to learn them some SQL.
Quoted 7 lines
Why am I telling you this story? Because you fall into the exact same trap as my colleague.
In this instance, it may not be so much network bandwidth, but it is still quite a suboptimal idea to render JSON for possibly tens of thousands of files, then parse the same JSON on the receiving side, the spend even more time to drop all but a dozen files.
Quoted 4 lines
This was mentioned before [1]. Of course I don't work on giant index files, but I would assume the cost of parsing JSON (at least with a stream-based one, not loading the whole thing in core) is still cheaper.
You may have heard that a few thousand of my colleagues are working on what they call the largest repository on this planet.
No, the cost of parsing JSON only to throw away the majority of the parsed information is not cheap. It is a clear sign of a design in want of being improved.
Quoted 4 lines
And you could still do it in iteration, saving every step until you have the reasonable small dataset to work on. The other side of the story is, are we sure parsing and executing pathspec is cheap? I'm not so sure, especially when pathspec code is not exactly optimized.
Let's not try to slap on workaround over workaround. Let's fix the root cause. (Being: don't filter at the wrong end.)
Quoted 1 line
Consider the amount of code to support something like that.
Given that I am pretty familiar with the pathspec machinery due to working with it in the git stash and git add -p built-ins, I have a very easy time considering the amount of code. It makes me smile how little code will be needed.
Quoted 2 lines
I'd rather wait until a real example come up and no good solution found without modify git.git, before actually supporting it.
Oh hey, there you go: Team Explorer. Visual Studio Code. Literally every single 3rd-party application that needs to deal with real-world loads. Every single one.
Quoted 7 lines
Quoted 5 lines
And this is
_evenmore_relevant when you want to debug things.In short: I am quite puzzled why this is even debated here. There is a reason, a good reason, why
git ls-filesaccepts pathspecs. I would not want to ignore the lessons of history as willfully here.I guess you and I have different ways of debugging things.
Yep, I'm with Lincoln here: Give me six hours to debug a problem and I will spend the first four optimizing the edit-build-test cycle.
Quoted 13 lines
Quoted 13 lines
Quoted 2 lines
Even with shell scripts, jq could do a much better job than pathspec. If you filter by pathspec, good luck trying that on extensions.
You keep harping on extensions, but the reality of the matter is that they are rarely interesting. I would even wager a bet that we will end up excluding them from the JSON output by default.
Most of the times when I had to decode the index file manually in the past, it was about the regular file entries.
There was *one* week in which I had to decode the untracked cache a bit, to the point where I patched the test helper locally to help me with that.
Quoted 6 lines
Quoted 2 lines
If my experience in debugging these things is any indicator, extensions do not matter even 10% of the non-extension data.
Again our experiences differ. Mine is mostly about extensions, probably because I had to work on them more often. For normal entries "ls-files --debug" gives you 99% what's in the index file already.
Like the device. And the ctime. And the file size. And the uid/gid. Is that what you mean?
I don't know whether I missed a joke or not.
Quoted 5 lines
Quoted 2 lines
You don't need JSON if you want to debug things. That would be a lot of love lost, if debugging was your goal.
No, I did think of some other line-based format before I ended up with JSON. I did not want to use it in the beginning.
Then why bother.
Quoted 9 lines
The thing is, a giant table to cover all fields and entries in the main index is not as easy to navigate, or search even in 'less'. And the hierarchical structure of some extensions is hard to represent in good way (at least without writing lots of code). On top of that I still need some easy way to parse and post-process, like how much saving I could gain if I compressed stat data. And the final nail is
json-writer.cis already there, much less work.So JSON was the best option I found to meet all those points.
Well, as I said: you're obviously dead-set to optimize this for debugging your own problems. The beauty of open source is that it can be turned into something of wider use.
Ciao,
Johannes
Re: [PATCH v2 01/10] ls-files: add --json to dump the index
In reply to Duy NguyenQuoted 29 lines · Duy Nguyen
Quoted 20 lines · Jeff Hostetler
json-writer.c void jw_object_null(struct json_writer *jw, const char *key) · line 212
Quoted 12 lines
json-writer.c +8
-0@@ -202,6 +202,28 @@ void jw_object_null(struct json_writer *jw, const char *key)strbuf_addstr (&jw->json,"null" ); } +void jw_object_filemode (struct json_writer *jw,const char *key,mode_t mode) +{ +object_common (jw, key); +strbuf_addf (&jw->json,"\"%06o\"" , mode); +} + +void jw_object_stat_data (struct json_writer *jw,const char *name, +const struct stat_data *sd)Should this be in
json_writer.cor inread-cache.c? Currently,json_writer.cis concerned with formatting JSON on basic/scalar types. Do we want to start extending it to handle arbitrary structures? Or would it be better for the code that defines/manipulates the structure to define a "stat_data_dump_json()" function.Quoted 7 lines
I'm torn on the
jw_object_filemode()function, JSON format limits us to decimal integers and there are places where I'd like to have hex, or in this case octal values.I'm thinking it'd be better to have a helper function in
read-cache.cthat formats a local strbuf and callsjs_object_string(&jw, key, buf);
Quoted 5 lines
I can move these back to
read-cache.c. Though if we have a lot more jw helpers like this (hard to tell at the moment) then perhaps we can havejson-writer-utils.cor something to group them together. That keep the "boring" code out of main logic code inread-cache.cand other call sites.
yeah, in an utils file or close to the "constructors" of the structure types. either one works.
Quoted 29 lines
Quoted 20 lines
Quoted 20 lines
read-cache.c +15
-0@@ -1952,6 +1953,49 @@ static void *load_index_extensions(void *_data)return NULL ; } +static void dump_cache_entry (struct index_state *istate, +int index, +unsigned long offset, +const struct cache_entry *ce) +{ +struct json_writer *jw = istate->jw; + +jw_array_inline_begin_object (jw); + +/* +* this is technically redundant, but it's for easier +* navigation when there hundreds of entries +*/ +jw_object_intmax (jw,"id" , index); +Quoted 8 lines
read-cache.c static void *load_index_extensions(void *_data) · line 1975
Quoted 5 lines
read-cache.c +5
-0+jw_object_string (jw,"name" , ce->name); + +jw_object_filemode (jw,"mode" , ce->ce_mode); + +jw_object_intmax (jw,"flags" , ce->ce_flags);It would be nice to have the flags as a hex-formatted string in addition to (or instead of) the decimal integer value.
Quoted 6 lines
I'm not against reformatting it in hex string, but is there a value in it?
ce_flagsis expanded in the code below so that you don't have to decode it yourself when you read each entry. The "flags" field here is for further processing in tools. I'm trying to see if looking at hex values helps, but I'm still not seeing it...
I guess I was thinking of the in-memory bits and thinking it'd be useful to be able to dump the index immediately after reading it and then later after some operation or traversal and see the intermediate states. But I realize now that that's not what you're building. This is a dump it while you're reading it feature (and that's fine).
So, as long as you have all of the on-disk bits, we should be fine as you suggest.
Jeff
Quoted 14 lines
Quoted 14 lines
Quoted 14 lines
read-cache.c +14
-0+/* +* again redundant info, just so you don't have to decode +* flags values manually +*/ +if (ce->ce_flags &CE_EXTENDED ) +jw_object_true (jw,"extended_flags" ); +if (ce->ce_flags &CE_VALID ) +jw_object_true (jw,"assume_unchanged" ); +if (ce->ce_flags &CE_INTENT_TO_ADD ) +jw_object_true (jw,"intent_to_add" ); +if (ce->ce_flags &CE_SKIP_WORKTREE ) +jw_object_true (jw,"skip_worktree" ); +if (ce_stage (ce)) +jw_object_intmax (jw,"stage" ,ce_stage (ce));
Quoted 1 line · Johannes Schindelin
Hi Duy,
[snip]
Quoted 6 lines
Quoted 3 lines
Again our experiences differ. Mine is mostly about extensions, probably because I had to work on them more often. For normal entries "ls-files --debug" gives you 99% what's in the index file already.
Like the device. And the ctime. And the file size. And the uid/gid. Is that what you mean?
Hmm, well I think so:
$ git ls-files --debug git.c git-compat-util.h
git-compat-util.h
ctime: 1561457278:502638001
mtime: 1561457278:502638001
dev: 2049 ino: 262663
uid: 1000 gid: 1000
size: 35440 flags: 0
git.c
ctime: 1561457278:518646000
mtime: 1561457278:518646000
dev: 2049 ino: 263083
uid: 1000 gid: 1000
size: 26837 flags: 0
$ I have occasionally added stuff to the '--debug' output while debugging something, but the above is usually sufficient for my uses. (Having said that, I have not had the need to debug extensions [yet!]).
ATB,
Ramsay Jones
Re: [PATCH v2 01/10] ls-files: add --json to dump the index
In reply to Johannes SchindelinQuoted 10 lines · Johannes Schindelin
t/t3011-ls-files-json.sh line 9
Quoted 1 line
t/t3011-ls-files-json.sh new file +1
-0+ echo 's/\("'$name'":\) [0-9]\+/\1 <number>/' >>filter.sedThis does not do what you think it does, in Ubuntu Xenial and on macOS:
The
\1is expanded to the ASCII character 001. Therefore your test cases fail on almost all platforms.Funnily enough, they pass on Windows...
bash, dash and /bin/echo behave differently given
$ echo 'foo \1 bar'
some 'echo' suffer from the "\<n>" interpolation. Some don't.
I think your spelled-out version downthread (except for stepping out of BRE which would break your sed script, as you realized) would be a much readable alternative.
Thanks.
Hi Ramsay,
Quoted 28 lines · Ramsay Jones
Quoted 1 line · Johannes Schindelin
Hi Duy,
[snip]
Quoted 6 lines
Quoted 3 lines
Again our experiences differ. Mine is mostly about extensions, probably because I had to work on them more often. For normal entries "ls-files --debug" gives you 99% what's in the index file already.
Like the device. And the ctime. And the file size. And the uid/gid. Is that what you mean?
Hmm, well I think so:
$ git ls-files --debug git.c git-compat-util.h git-compat-util.h ctime: 1561457278:502638001 mtime: 1561457278:502638001 dev: 2049 ino: 262663 uid: 1000 gid: 1000 size: 35440 flags: 0 git.c ctime: 1561457278:518646000 mtime: 1561457278:518646000 dev: 2049 ino: 263083 uid: 1000 gid: 1000 size: 26837 flags: 0 $
Quoted 4 lines
I have occasionally added stuff to the '--debug' output while debugging something, but the above is usually sufficient for my uses. (Having said that, I have not had the need to debug extensions [yet!]).
Well, live and learn. So for debugging purposes, we already have a good facility and would not need JSON at all.
Good to know,
Dscho
Re: [PATCH v2 01/10] ls-files: add --json to dump the index
In reply to Nguyễn Thái Ngọc DuyQuoted 1 line · Nguyễn Thái Ngọc Duy
This --json is supposed to help that. It dumps the index in a human
This and another one on the title need to become "--debug-json".
Are we expecting a reroll to introduce another layer above the most primitive json writer that knows the schema used to represent both system standard and our application-specific structures, or is the current arrangement to have them in json-writer.c until there are enough of them to warrant such a split good enough?
Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json
In reply to Derrick StoleeQuoted 27 lines · Derrick Stolee
Quoted 19 lines · Duy Nguyen
Quoted 5 lines · Jeff Hostetler
I'm curious how big these EWAHs will be in practice and how useful an array of integers will be (especially as the pretty format will be one integer per line). Perhaps it would helpful to have an extended example in one of the tests.
It's one integer per updated entry. So if you have a giant index and updated every single one of them, the EWAH bitmap contains that many integers.
If it was easy to just merge these bitmaps back to the entry (e.g. in this example, add "replaced": true to entry zero) I would have done it. But we dump as we stream and it's already too late to do it.
Quoted 3 lines
Would it be better to have the caller of
ewah_each_bit()build a hex or bit string in a strbuf and then write it as a single string?Quoted 5 lines
I don't think the current EWAH representation is easy to read in the first place. You'll probably have to run through some script to update the main entries part and will have a much better view, but that's pretty quick. If it's for scripts, then it's probably best to keep as an array of integers, not a string. Less post processing.
Quoted 8 lines
I don't think the intent is to dump the EWAH directly, but instead to dump a string of the uncompressed bitmap. Something like:
"delete_bitmap" : "01101101101"instead of
"delete_bitmap" : [ 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1 ]
I get this part. But the numbers in the array were the position of the set bits. It's not showing just the actual bit map.
The same bitmap would be currently displayed as
"delete_bitmap": [ 1, 2, 4, 5, 7, 8, 9, 11 ]
And that maps back to the entry[1], entry[2], entry[4]... in the index being deleted from the base index. So displaying as a real bit map actually adds more work for both the reader and the tool because you have to calculate the position either way. And it gets harder if the bit you're intereted in is on the far right.
Quoted 18 lines
Quoted 6 lines
Another reason for not merging to one string (might not be a very good argument though) is to help diff between two indexes. One-number-per-line works well with "git diff --no-index" while one long string is a bit harder. I did this kind of comparison when I made changes in
read-cache.cand wanted to check if the new index file is completely broken, or just slighly broken.You're right that the diff of the json output is an interesting use, and the "single string" output is not helpful. What about batches of 64-bit strings? For example:
"delete_bitmap" : [ "0101010101010101010101010101010101010101010101010101010101010101", "0101010101010101010101010101010101010101010101010101010101010101", "0101010101010101010101010101010101010101010101010101010101010101", "01010101010101" ]
Quoted 2 lines
This could be a happy medium between the two options, but does require some extra work in the formatter.
And the reader/parser too since you have to join that array back in one string first.
--
Duy
Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json
Quoted 42 lines · Duy Nguyen
Quoted 27 lines · Derrick Stolee
Quoted 19 lines · Duy Nguyen
Quoted 5 lines · Jeff Hostetler
I'm curious how big these EWAHs will be in practice and how useful an array of integers will be (especially as the pretty format will be one integer per line). Perhaps it would helpful to have an extended example in one of the tests.
It's one integer per updated entry. So if you have a giant index and updated every single one of them, the EWAH bitmap contains that many integers.
If it was easy to just merge these bitmaps back to the entry (e.g. in this example, add "replaced": true to entry zero) I would have done it. But we dump as we stream and it's already too late to do it.
Quoted 3 lines
Would it be better to have the caller of
ewah_each_bit()build a hex or bit string in a strbuf and then write it as a single string?Quoted 5 lines
I don't think the current EWAH representation is easy to read in the first place. You'll probably have to run through some script to update the main entries part and will have a much better view, but that's pretty quick. If it's for scripts, then it's probably best to keep as an array of integers, not a string. Less post processing.
Quoted 8 lines
I don't think the intent is to dump the EWAH directly, but instead to dump a string of the uncompressed bitmap. Something like:
"delete_bitmap" : "01101101101"instead of
"delete_bitmap" : [ 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1 ]I get this part. But the numbers in the array were the position of the set bits. It's not showing just the actual bit map.
The same bitmap would be currently displayed as
Quoted 7 lines
"
delete_bitmap": [ 1, 2, 4, 5, 7, 8, 9, 11 ]And that maps back to the entry[1], entry[2], entry[4]... in the index being deleted from the base index. So displaying as a real bit map actually adds more work for both the reader and the tool because you have to calculate the position either way. And it gets harder if the bit you're intereted in is on the far right.
Thanks for the clarification. That helps.
Jeff
Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json
Quoted 45 lines · Jeff Hostetler
Quoted 44 lines · Duy Nguyen
Quoted 27 lines · Derrick Stolee
Quoted 19 lines · Duy Nguyen
Quoted 5 lines · Jeff Hostetler
I'm curious how big these EWAHs will be in practice and how useful an array of integers will be (especially as the pretty format will be one integer per line). Perhaps it would helpful to have an extended example in one of the tests.
It's one integer per updated entry. So if you have a giant index and updated every single one of them, the EWAH bitmap contains that many integers.
If it was easy to just merge these bitmaps back to the entry (e.g. in this example, add "replaced": true to entry zero) I would have done it. But we dump as we stream and it's already too late to do it.
Quoted 3 lines
Would it be better to have the caller of
ewah_each_bit()build a hex or bit string in a strbuf and then write it as a single string?Quoted 5 lines
I don't think the current EWAH representation is easy to read in the first place. You'll probably have to run through some script to update the main entries part and will have a much better view, but that's pretty quick. If it's for scripts, then it's probably best to keep as an array of integers, not a string. Less post processing.
Quoted 8 lines
I don't think the intent is to dump the EWAH directly, but instead to dump a string of the uncompressed bitmap. Something like:
"
delete_bitmap" : "01101101101"instead of
"
delete_bitmap" : [ 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1 ]I get this part. But the numbers in the array were the position of the set bits. It's not showing just the actual bit map.
The same bitmap would be currently displayed as
"
delete_bitmap": [ 1, 2, 4, 5, 7, 8, 9, 11 ]
Quoted 8 lines
Quoted 5 lines
And that maps back to the entry[1], entry[2], entry[4]... in the index being deleted from the base index. So displaying as a real bit map actually adds more work for both the reader and the tool because you have to calculate the position either way. And it gets harder if the bit you're intereted in is on the far right.
Thanks for the clarification. That helps.
Same here! We expect these to be much smaller than the full set, correct?
Thanks,
-Stolee
Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json
Quoted 54 lines · Derrick Stolee
Quoted 52 lines · Jeff Hostetler
Quoted 49 lines · Duy Nguyen
Quoted 27 lines · Derrick Stolee
Quoted 19 lines · Duy Nguyen
Quoted 5 lines · Jeff Hostetler
I'm curious how big these EWAHs will be in practice and how useful an array of integers will be (especially as the pretty format will be one integer per line). Perhaps it would helpful to have an extended example in one of the tests.
It's one integer per updated entry. So if you have a giant index and updated every single one of them, the EWAH bitmap contains that many integers.
If it was easy to just merge these bitmaps back to the entry (e.g. in this example, add "replaced": true to entry zero) I would have done it. But we dump as we stream and it's already too late to do it.
Quoted 3 lines
Would it be better to have the caller of
ewah_each_bit()build a hex or bit string in a strbuf and then write it as a single string?Quoted 5 lines
I don't think the current EWAH representation is easy to read in the first place. You'll probably have to run through some script to update the main entries part and will have a much better view, but that's pretty quick. If it's for scripts, then it's probably best to keep as an array of integers, not a string. Less post processing.
Quoted 8 lines
I don't think the intent is to dump the EWAH directly, but instead to dump a string of the uncompressed bitmap. Something like:
"delete_bitmap" : "01101101101"instead of
"delete_bitmap" : [ 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1 ]I get this part. But the numbers in the array were the position of the set bits. It's not showing just the actual bit map.
The same bitmap would be currently displayed as
"
delete_bitmap": [ 1, 2, 4, 5, 7, 8, 9, 11 ]And that maps back to the entry[1], entry[2], entry[4]... in the index being deleted from the base index. So displaying as a real bit map actually adds more work for both the reader and the tool because you have to calculate the position either way. And it gets harder if the bit you're intereted in is on the far right.
Quoted 3 lines
Quoted 1 line
Thanks for the clarification. That helps.
Same here! We expect these to be much smaller than the full set, correct?
For split-index, the number of 1 bits should be about the size of your working set, not the index size. In the normal case, then yes it should be much smaller. After a big merge or branch switch, it could get as big as the index. But I would hope the logic to re-split the index kicks in, which essentially empties these bitmaps.
EWAH bitmap is also used in UNTR extension if I remember correctly. Those bitmaps may have as many bits as the directories you have in the index.
Quoted 3 lines
Thanks,
-Stolee
Signature · 1 line
Duy
Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json
In reply to Nguyễn Thái Ngọc DuyQuoted 11 lines · Nguyễn Thái Ngọc Duy
t/t3011-ls-files-json.sh +3
-0@@ -44,4 +44,18 @@ test_expect_success 'ls-files --json, main entries, UNTR and TREE' ' compare_json basic ' +test_expect_success 'ls-files --json, split index' ' + git init split && + (
t/t3011-ls-files-json.sh test_expect_success 'ls-files --json, main entries, UNTR and TREE' ' · line 59
Quoted 10 lines
t/t3011-ls-files-json.sh +10
-0+ cd split && + echo one >one && + git add one && + git update-index --split-index && + echo updated >>one && + test_must_fail git -c splitIndex.maxPercentChange=100 update-index --refresh && + cp ../filter.sed . && + compare_json split-index + ) +'
I think this test should 'sane_unset GIT_TEST_SPLIT_INDEX'. Maybe it's not absolutely necessary, because the explicit '--split-index' and '-c splitIndex.maxPercentChange=100' would already fully control when index splitting is performed, eliminating any indeterminism inherent to GIT_TEST_SPLIT_INDEX... but unsetting it would reduce the cognitive load on future readers.
The same might apply to GIT_TEST_FSMONITOR in the following patch, and perhaps even to GIT_TEST_INDEX_THREADS.
Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json
In reply to Nguyễn Thái Ngọc DuyQuoted 58 lines · Nguyễn Thái Ngọc Duy
t/t3011-ls-files-json.sh +14
-0@@ -44,4 +44,18 @@ test_expect_success 'ls-files --json, main entries, UNTR and TREE' ' compare_json basic ' +test_expect_success 'ls-files --json, split index' ' + git init split && + ( + cd split && + echo one >one && + git add one && + git update-index --split-index && + echo updated >>one && + test_must_fail git -c splitIndex.maxPercentChange=100 update-index --refresh && + cp ../filter.sed . && + compare_json split-index + ) +' + test_donet/t3011/split-index new file +29
-0@@ -0,0 +1,39 @@ +{ + "version": 2, + "oid": <string>, + "mtime_sec": <number>, + "mtime_nsec": <number>, + "entries": [ + { + "id": 0, + "name": "", + "mode": "100644", + "flags": 0, + "oid": <string>, + "stat": { + "ctime_sec": <number>, + "ctime_nsec": <number>, + "mtime_sec": <number>, + "mtime_nsec": <number>, + "device": <number>, + "inode": <number>, + "uid": <number>, + "gid": <number>, + "size": 4 + }, + "file_offset": <number> + } + ], + "extensions": { + "link": { + "file_offset": <number>,
t/t3011/split-index line 39
Quoted 10 lines
t/t3011/split-index new file +10
-0+ "ext_size": <number>, + "oid": <string>, + "delete_bitmap": [ + ], + "replace_bitmap": [ + 0 + ] + } + } +}
This test is flaky, as reported in:
https://public-inbox.org/git/[email protected]/
This is because it relies on racy behaviour, namely that the following three commands
echo one >one &&
git add one &&
git update-index --split-index &&are executed within the same second, leaving 'one' racily clean. To deal with the racily clean file, 5581a019ba (split-index: smudge and add racily clean cache entries to split index, 2018-10-11) kicks in, and 'one's smudged index entry is stored both in the shared index and in the split index. That's why this test expects the offset 0 in the "replace_bitmap" array.
However, it's possible that a second boundary is crossed between writing to 'one' and splitting the index, and then 'one' is not racily clean, and its index entry is only stored in the shared index. Consequently, there are no index entries in the split index, so the "replace_bitmap" array ends up being empty, ultimately failing the test.
A 'test-tool chmtime' invocation or two could make the test deterministic (i.e it could make sure that 'one' is either always racily clean or it never is, whichever is preferred).
What I still don't understand, however, is that when the test fails this way, then the "entries" array ends up being empty as well. It looks as if the JSON dump included only index entries that were actually stored in '.git/index', but omitted entries that were only present in the shared index. I think this is wrong, and it should dump the unified view of the split and shared indexes. Or include all entries from the shared index as well. Or perhaps I'm completely missing something...
Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json
Quoted 89 lines · SZEDER Gábor
Quoted 58 lines · Nguyễn Thái Ngọc Duy
t/t3011-ls-files-json.sh +14
-0@@ -44,4 +44,18 @@ test_expect_success 'ls-files --json, main entries, UNTR and TREE' ' compare_json basic ' +test_expect_success 'ls-files --json, split index' ' + git init split && + ( + cd split && + echo one >one && + git add one && + git update-index --split-index && + echo updated >>one && + test_must_fail git -c splitIndex.maxPercentChange=100 update-index --refresh && + cp ../filter.sed . && + compare_json split-index + ) +' + test_donet/t3011/split-index new file +29
-0@@ -0,0 +1,39 @@ +{ + "version": 2, + "oid": <string>, + "mtime_sec": <number>, + "mtime_nsec": <number>, + "entries": [ + { + "id": 0, + "name": "", + "mode": "100644", + "flags": 0, + "oid": <string>, + "stat": { + "ctime_sec": <number>, + "ctime_nsec": <number>, + "mtime_sec": <number>, + "mtime_nsec": <number>, + "device": <number>, + "inode": <number>, + "uid": <number>, + "gid": <number>, + "size": 4 + }, + "file_offset": <number> + } + ], + "extensions": { + "link": { + "file_offset": <number>,t/t3011/split-index line 39
Quoted 10 lines
t/t3011/split-index new file +10
-0+ "ext_size": <number>, + "oid": <string>, + "delete_bitmap": [ + ], + "replace_bitmap": [ + 0 + ] + } + } +}This test is flaky, as reported in:
https://public-inbox.org/git/[email protected]/
This is because it relies on racy behaviour, namely that the following three commands
echo one >one && git add one && git update-index --split-index &&are executed within the same second, leaving 'one' racily clean. To deal with the racily clean file,
5581a019ba(split-index: smudge and add racily clean cache entries to split index, 2018-10-11) kicks in, and 'one's smudged index entry is stored both in the shared index and in the split index. That's why this test expects the offset 0 in the "replace_bitmap" array.
Quoted 6 lines
However, it's possible that a second boundary is crossed between writing to 'one' and splitting the index, and then 'one' is not racily clean, and its index entry is only stored in the shared index. Consequently, there are no index entries in the split index, so the "
replace_bitmap" array ends up being empty, ultimately failing the test.
Yep. I came up with the same conclusion. But I still have a couple other things to update before resending.
Quoted 4 lines
A 'test-tool chmtime' invocation or two could make the test deterministic (i.e it could make sure that 'one' is either always racily clean or it never is, whichever is preferred).
Quoted 8 lines
What I still don't understand, however, is that when the test fails this way, then the "entries" array ends up being empty as well. It looks as if the JSON dump included only index entries that were actually stored in '.git/index', but omitted entries that were only present in the shared index. I think this is wrong, and it should dump the unified view of the split and shared indexes. Or include all entries from the shared index as well. Or perhaps I'm completely missing something...
The command is to dump .git/index, not the shared one. And since this is not a split index test, rather a (quite low-level) json dump test, I did not bother to also dump the shared index, which should look like a regular one. Producing a unified view in json might not be easy with the current code because it's tied to file reading code, nearly stream out json as we read the file, and split-index requires a post processing step. I could contribute a python script or something to combine shared/main index together. That way you can still see the combined one, but we don't have to add/maintain more C code.
Signature · 1 line
Duy
Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json
Quoted 9 lines · Duy Nguyen
The command is to dump .git/index, not the shared one. And since this is not a split index test, rather a (quite low-level) json dump test, I did not bother to also dump the shared index, which should look like a regular one. Producing a unified view in json might not be easy with the current code because it's tied to file reading code, nearly stream out json as we read the file, and split-index requires a post processing step. I could contribute a python script or something to combine shared/main index together. That way you can still see the combined one, but we don't have to add/maintain more C code.
Well, such a post-processing is something external scripts shine at and exporting the internal data in json format is exactly to support these scripts, so it may make a good first test case ;-)