Skip to contentSkip to conversation

Add 'ls-files --debug-json' to dump the index in json

PATCH In review Nguyễn Thái Ngọc Duy 8 participants 43 messages Jun 24, 2019 – Jul 8, 2019 Documentation

lore
Share & tools
lore
#0 lore raw

v2 cover Add 'ls-files --debug-json' to dump the index in json

v2 main changes:

- --json is renamed --debug-json
- json field names now use '_' instead of '.' to be friendlier to some languages. I stick to underscore_name instead of camelCase because the former is closer to what we use
- extension location is printed, in case you need to decode the extension by yourself (previously only the size is printed)
- all extensions are printed in the same order they appear in the file (previously eoie and ieot are printed first because that's how we parse)
- resolve undo extension is reorganized a bit to be easier to read
- tests added. Example json files are in t/t3011

Interdiff

Documentation/git-ls-files.txt +5 -5
index 54011c8f65..fec5cb7170 100644
@@ -60,11 +60,6 @@ OPTIONS
 --stage::
 	Show staged contents' mode bits, object name and stage number in the output.
 
---json::
-	Dump the entire index content in JSON format. This is for
-	debugging purposes and the JSON structure may change from time
-	to time.
-
 --directory::
 	If a whole directory is classified as "other", show just its
 	name (with a trailing slash) and not its whole contents.
@@ -167,6 +162,11 @@ a space) at the start of each line:
 	possible for manual inspection; the exact format may change at
 	any time.
 
+--debug-json::
+	Dump the entire index content in JSON format. This is for
+	debugging purposes. The JSON structure is subject to change.
+	Note that the strings are not always valid UTF-8.
+
 --eol::
 	Show <eolinfo> and <eolattr> of files.
 	<eolinfo> is the file content identification used by Git when
builtin/ls-files.c +11 -5
index d00f6d3074..b60cd9ab28 100644
@@ -545,8 +545,6 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)
 			N_("show staged contents' object name in the output")),
 		OPT_BOOL('k', "killed", &show_killed,
 			N_("show files on the filesystem that need to be removed")),
-		OPT_BOOL(0, "json", &show_json,
-			N_("dump index content in json format")),
 		OPT_BIT(0, "directory", &dir.flags,
 			N_("show 'other' directories' names only"),
 			DIR_SHOW_OTHER_DIRECTORIES),
@@ -581,6 +579,8 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)
 			N_("pretend that paths removed since <tree-ish> are still present")),
 		OPT__ABBREV(&abbrev),
 		OPT_BOOL(0, "debug", &debug_mode, N_("show debugging data")),
+		OPT_BOOL(0, "debug-json", &show_json,
+			N_("dump index content in JSON format")),
 		OPT_END()
 	};
 
@@ -636,7 +636,7 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)
 		    "--error-unmatch");
 
 	parse_pathspec(&pathspec, 0,
-		       PATHSPEC_PREFER_CWD,
+		       show_json ? PATHSPEC_PREFER_FULL : PATHSPEC_PREFER_CWD,
 		       prefix, argv);
 
 	/*
@@ -668,8 +668,14 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)
 		show_cached = 1;
 	if (show_json && (show_stage || show_deleted || show_others ||
 			  show_unmerged || show_killed || show_modified ||
-			  show_cached || show_resolve_undo || with_tree))
-		die(_("--show-json cannot be used with other --show- options, or --with-tree"));
+			  show_cached || pathspec.nr))
+		die(_("--debug-json cannot be used with other file selection options"));
+	if (show_json && show_resolve_undo)
+		die(_("--debug-json cannot be used with %s"), "--resolve-undo");
+	if (show_json && with_tree)
+		die(_("--debug-json cannot be used with %s"), "--with-tree");
+	if (show_json && debug_mode)
+		die(_("--debug-json cannot be used with %s"), "--debug");
 
 	if (with_tree) {
 		/*
cache-tree.c +1 -6
index fc44016fe8..b6a233307e 100644
@@ -599,18 +599,13 @@ struct cache_tree *cache_tree_read(const char *buffer, unsigned long size,
 	struct cache_tree *ret;
 
 	if (jw) {
-		jw_object_inline_begin_object(jw, "cache-tree");
-		jw_object_intmax(jw, "ext-size", size);
 		jw_object_inline_begin_object(jw, "root");
 	}
 	if (buffer[0])
 		ret = NULL; /* not the whole tree */
 	else
 		ret = read_one(&buffer, &size, jw);
-	if (jw) {
-		jw_end(jw);	/* root */
-		jw_end(jw);	/* cache-tree */
-	}
+	jw_end_gently(jw);
 	return ret;
 }
 
dir.c +7 -6
index f389eee24a..8808577ea3 100644
@@ -2902,13 +2902,15 @@ struct untracked_cache *read_untracked_extension(const void *data,
 	uc->exclude_per_dir = xstrdup(exclude_per_dir);
 
 	if (jw) {
-		jw_object_inline_begin_object(jw, "untracked-cache");
-		jw_object_intmax(jw, "ext-size", sz);
 		jw_object_string(jw, "ident", ident);
-		jw_object_oid_stat(jw, "info/exclude", &uc->ss_info_exclude);
-		jw_object_oid_stat(jw, "excludes-file", &uc->ss_excludes_file);
+		jw_object_oid_stat(jw, "info_exclude", &uc->ss_info_exclude);
+		jw_object_oid_stat(jw, "excludes_file", &uc->ss_excludes_file);
 		jw_object_intmax(jw, "flags", uc->dir_flags);
-		jw_object_string(jw, "excludes-per-dir", uc->exclude_per_dir);
+		if (uc->dir_flags & DIR_SHOW_OTHER_DIRECTORIES)
+			jw_object_bool(jw, "show_other_directories", 1);
+		if (uc->dir_flags & DIR_HIDE_EMPTY_DIRECTORIES)
+			jw_object_bool(jw, "hide_empty_directories", 1);
+		jw_object_string(jw, "excludes_per_dir", uc->exclude_per_dir);
 	}
 
 	/* NUL after exclude_per_dir is covered by sizeof(*ouc) */
@@ -2968,7 +2970,6 @@ struct untracked_cache *read_untracked_extension(const void *data,
 		free_untracked_cache(uc);
 		uc = NULL;
 	}
-	jw_end_gently(jw);
 	return uc;
 }
 
fsmonitor.c +1 -4
index f6ba437255..5ed55ad176 100644
@@ -52,12 +52,9 @@ int read_fsmonitor_extension(struct index_state *istate, const void *data,
 	istate->fsmonitor_dirty = fsmonitor_dirty;
 
 	if (istate->jw) {
-		jw_object_inline_begin_object(istate->jw, "fsmonitor");
 		jw_object_intmax(istate->jw, "version", hdr_version);
-		jw_object_intmax(istate->jw, "last-update", istate->fsmonitor_last_update);
+		jw_object_intmax(istate->jw, "last_update", istate->fsmonitor_last_update);
 		jw_object_ewah(istate->jw, "dirty", fsmonitor_dirty);
-		jw_object_intmax(istate->jw, "ext-size", sz);
-		jw_end(istate->jw);
 	}
 	trace_printf_key(&trace_fsmonitor, "read fsmonitor extension successful");
 	return 0;
json-writer.c +15 -9
index 70403580ca..c0bd302e4e 100644
@@ -203,19 +203,25 @@ void jw_object_null(struct json_writer *jw, const char *key)
 	strbuf_addstr(&jw->json, "null");
 }
 
+void jw_object_filemode(struct json_writer *jw, const char *key, mode_t mode)
+{
+	object_common(jw, key);
+	strbuf_addf(&jw->json, "\"%06o\"", mode);
+}
+
 void jw_object_stat_data(struct json_writer *jw, const char *name,
 			 const struct stat_data *sd)
 {
 	jw_object_inline_begin_object(jw, name);
-	jw_object_intmax(jw, "st_ctime.sec", sd->sd_ctime.sec);
-	jw_object_intmax(jw, "st_ctime.nsec", sd->sd_ctime.nsec);
-	jw_object_intmax(jw, "st_mtime.sec", sd->sd_mtime.sec);
-	jw_object_intmax(jw, "st_mtime.nsec", sd->sd_mtime.nsec);
-	jw_object_intmax(jw, "st_dev", sd->sd_dev);
-	jw_object_intmax(jw, "st_ino", sd->sd_ino);
-	jw_object_intmax(jw, "st_uid", sd->sd_uid);
-	jw_object_intmax(jw, "st_gid", sd->sd_gid);
-	jw_object_intmax(jw, "st_size", sd->sd_size);
+	jw_object_intmax(jw, "ctime_sec", sd->sd_ctime.sec);
+	jw_object_intmax(jw, "ctime_nsec", sd->sd_ctime.nsec);
+	jw_object_intmax(jw, "mtime_sec", sd->sd_mtime.sec);
+	jw_object_intmax(jw, "mtime_nsec", sd->sd_mtime.nsec);
+	jw_object_intmax(jw, "device", sd->sd_dev);
+	jw_object_intmax(jw, "inode", sd->sd_ino);
+	jw_object_intmax(jw, "uid", sd->sd_uid);
+	jw_object_intmax(jw, "gid", sd->sd_gid);
+	jw_object_intmax(jw, "size", sd->sd_size);
 	jw_end(jw);
 }
 
json-writer.h +3 -0
index f778e019a2..07d841d52a 100644
@@ -42,8 +42,10 @@
  * of the given strings.
  */
 
+#include "git-compat-util.h"
 #include "strbuf.h"
 
+struct ewah_bitmap;
 struct stat_data;
 
 struct json_writer
@@ -83,6 +85,7 @@ void jw_object_true(struct json_writer *jw, const char *key);
 void jw_object_false(struct json_writer *jw, const char *key);
 void jw_object_bool(struct json_writer *jw, const char *key, int value);
 void jw_object_null(struct json_writer *jw, const char *key);
+void jw_object_filemode(struct json_writer *jw, const char *key, mode_t value);
 void jw_object_stat_data(struct json_writer *jw, const char *key,
 			 const struct stat_data *sd);
 void jw_object_ewah(struct json_writer *jw, const char *key,
read-cache.c +87 -78

317 lines not on this page · raw message

resolve-undo.c +25 -31

75 lines not on this page · raw message

split-index.c +2 -6
index d7b4420c92..41552bf771 100644
@@ -17,7 +17,6 @@ int read_link_extension(struct index_state *istate,
 {
 	const unsigned char *data = data_;
 	struct split_index *si;
-	unsigned long original_sz = sz;
 	int ret;
 
 	if (sz < the_hash_algo->rawsz)
@@ -42,12 +41,9 @@ int read_link_extension(struct index_state *istate,
 		return error("garbage at the end of link extension");
 done:
 	if (istate->jw) {
-		jw_object_inline_begin_object(istate->jw, "split-index");
 		jw_object_string(istate->jw, "oid", oid_to_hex(&si->base_oid));
-		jw_object_ewah(istate->jw, "delete-bitmap", si->delete_bitmap);
-		jw_object_ewah(istate->jw, "replace-bitmap", si->replace_bitmap);
-		jw_object_intmax(istate->jw, "ext-size", original_sz);
-		jw_end(istate->jw);
+		jw_object_ewah(istate->jw, "delete_bitmap", si->delete_bitmap);
+		jw_object_ewah(istate->jw, "replace_bitmap", si->replace_bitmap);
 	}
 	return 0;
 }
t/t3008-ls-files-lazy-init-name-hash.sh +1 -7
index 64f047332b..7f918c05f6 100755
@@ -4,15 +4,9 @@ test_description='Test the lazy init name hash with various folder structures'
 
 . ./test-lib.sh
 
-if test 1 -eq $($GIT_BUILD_DIR/t/helper/test-tool online-cpus)
-then
-	skip_all='skipping lazy-init tests, single cpu'
-	test_done
-fi
-
 LAZY_THREAD_COST=2000
 
-test_expect_success 'no buffer overflow in lazy_init_name_hash' '
+test_expect_success !SINGLE_CPU 'no buffer overflow in lazy_init_name_hash' '
 	(
 	    test_seq $LAZY_THREAD_COST | sed "s/^/a_/" &&
 	    echo b/b/b &&
t/t3011-ls-files-json.sh new file +106 -0

106 lines not on this page · raw message

t/t3011/basic new file +124 -0

124 lines not on this page · raw message

t/t3011/eoie new file +107 -0

107 lines not on this page · raw message

t/t3011/fsmonitor new file +38 -0
new file mode 100644
index 0000000000..17f2d4a664
@@ -0,0 +1,38 @@
+{
+  "version": 2,
+  "oid": <string>,
+  "mtime_sec": <number>,
+  "mtime_nsec": <number>,
+  "entries": [
+    {
+      "id": 0,
+      "name": "one",
+      "mode": "100644",
+      "flags": 0,
+      "oid": <string>,
+      "stat": {
+        "ctime_sec": <number>,
+        "ctime_nsec": <number>,
+        "mtime_sec": <number>,
+        "mtime_nsec": <number>,
+        "device": <number>,
+        "inode": <number>,
+        "uid": <number>,
+        "gid": <number>,
+        "size": 4
+      },
+      "file_offset": <number>
+    }
+  ],
+  "extensions": {
+    "FSMN": {
+      "file_offset": <number>,
+      "ext_size": <number>,
+      "version": 1,
+      "last_update": <number>,
+      "dirty": [
+        0
+      ]
+    }
+  }
+}
t/t3011/rerere new file +66 -0

66 lines not on this page · raw message

t/t3011/split-index new file +39 -0
new file mode 100644
index 0000000000..cdcc4ddded
@@ -0,0 +1,39 @@
+{
+  "version": 2,
+  "oid": <string>,
+  "mtime_sec": <number>,
+  "mtime_nsec": <number>,
+  "entries": [
+    {
+      "id": 0,
+      "name": "",
+      "mode": "100644",
+      "flags": 0,
+      "oid": <string>,
+      "stat": {
+        "ctime_sec": <number>,
+        "ctime_nsec": <number>,
+        "mtime_sec": <number>,
+        "mtime_nsec": <number>,
+        "device": <number>,
+        "inode": <number>,
+        "uid": <number>,
+        "gid": <number>,
+        "size": 4
+      },
+      "file_offset": <number>
+    }
+  ],
+  "extensions": {
+    "link": {
+      "file_offset": <number>,
+      "ext_size": <number>,
+      "oid": <string>,
+      "delete_bitmap": [
+      ],
+      "replace_bitmap": [
+        0
+      ]
+    }
+  }
+}
t/test-lib.sh +4 -0
index 4b346467df..9d5b273b40 100644
@@ -1611,3 +1611,7 @@ test_lazy_prereq REBASE_P '
 test_lazy_prereq FAIL_PREREQS '
 	test -n "$GIT_TEST_FAIL_PREREQS"
 '
+
+test_lazy_prereq SINGLE_CPU '
+	test "$(test-tool online-cpus)" -eq 1
+'

Nguyễn Thái Ngọc Duy (10):
ls-files: add --json to dump the index read-cache.c: dump common extension info in json cache-tree.c: dump "TREE" extension as json dir.c: dump "UNTR" extension as json split-index.c: dump "link" extension as json fsmonitor.c: dump "FSMN" extension as json resolve-undo.c: dump "REUC" extension as json read-cache.c: dump "EOIE" extension as json read-cache.c: dump "IEOT" extension as json
t3008: use the new SINGLE_CPU prereq

 Documentation/git-ls-files.txt          |   5 +
 builtin/ls-files.c                      |  38 ++++-
 cache-tree.c                            |  36 ++++-
 cache-tree.h                            |   5 +-
 cache.h                                 |   2 +
 dir.c                                   |  57 +++++++-
 dir.h                                   |   4 +-
 fsmonitor.c                             |   6 +
 json-writer.c                           |  36 +++++
 json-writer.h                           |  32 +++++
 read-cache.c                            | 178 ++++++++++++++++++++----
 resolve-undo.c                          |  30 +++-
 resolve-undo.h                          |   4 +-
 split-index.c                           |   9 +-
 t/t3008-ls-files-lazy-init-name-hash.sh |   8 +-
 t/t3011-ls-files-json.sh (new +x)       | 106 ++++++++++++++
 t/t3011/basic (new)                     | 124 +++++++++++++++++
 t/t3011/eoie (new)                      | 107 ++++++++++++++
 t/t3011/fsmonitor (new)                 |  38 +++++
 t/t3011/rerere (new)                    |  66 +++++++++
 t/t3011/split-index (new)               |  39 ++++++
 t/test-lib.sh                           |   4 +
 22 files changed, 884 insertions(+), 50 deletions(-)
 create mode 100755 t/t3011-ls-files-json.sh
 create mode 100644 t/t3011/basic
 create mode 100644 t/t3011/eoie
 create mode 100644 t/t3011/fsmonitor
 create mode 100644 t/t3011/rerere
 create mode 100644 t/t3011/split-index
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
#1 lore raw

PATCH v2 1/10 ls-files: add --json to dump the index

So far we don't have a command to basically dump the index file out, with all its glory details. Checking some info, for example, stat time, usually involves either writing new code or firing up "xxd" and decoding values by yourself.

This --json is supposed to help that. It dumps the index in a human readable format but also easy to be processed with tools. And it will print almost enough info to reconstruct the index later.

In this patch we only dump the main part, not extensions. But at the end of the series, the entire index is dumped. The end result could be very verbose even on a small repository such as git.git.

---
 Documentation/git-ls-files.txt    |  5 +++
 builtin/ls-files.c                | 38 +++++++++++++---
 cache.h                           |  2 +
 json-writer.c                     | 22 ++++++++++
 json-writer.h                     | 23 ++++++++++
 read-cache.c                      | 72 ++++++++++++++++++++++++++++++-
 t/t3011-ls-files-json.sh (new +x) | 44 +++++++++++++++++++
 t/t3011/basic (new)               | 67 ++++++++++++++++++++++++++++
 8 files changed, 265 insertions(+), 8 deletions(-)
Documentation/git-ls-files.txt +5 -0
index 8461c0e83e..fec5cb7170 100644
@@ -162,6 +162,11 @@ a space) at the start of each line:
 	possible for manual inspection; the exact format may change at
 	any time.
 
+--debug-json::
+	Dump the entire index content in JSON format. This is for
+	debugging purposes. The JSON structure is subject to change.
+	Note that the strings are not always valid UTF-8.
+
 --eol::
 	Show <eolinfo> and <eolattr> of files.
 	<eolinfo> is the file content identification used by Git when
builtin/ls-files.c +32 -6
index 7f83c9a6f2..b60cd9ab28 100644
@@ -8,6 +8,7 @@
 #include "cache.h"
 #include "repository.h"
 #include "config.h"
+#include "json-writer.h"
 #include "quote.h"
 #include "dir.h"
 #include "builtin.h"
@@ -31,6 +32,7 @@ static int show_modified;
 static int show_killed;
 static int show_valid_bit;
 static int show_fsmonitor_bit;
+static int show_json;
 static int line_terminator = '\n';
 static int debug_mode;
 static int show_eol;
@@ -577,6 +579,8 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)
 			N_("pretend that paths removed since <tree-ish> are still present")),
 		OPT__ABBREV(&abbrev),
 		OPT_BOOL(0, "debug", &debug_mode, N_("show debugging data")),
+		OPT_BOOL(0, "debug-json", &show_json,
+			N_("dump index content in JSON format")),
 		OPT_END()
 	};
 
@@ -632,7 +636,7 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)
 		    "--error-unmatch");
 
 	parse_pathspec(&pathspec, 0,
-		       PATHSPEC_PREFER_CWD,
+		       show_json ? PATHSPEC_PREFER_FULL : PATHSPEC_PREFER_CWD,
 		       prefix, argv);
 
 	/*
@@ -660,8 +664,18 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)
 
 	/* With no flags, we default to showing the cached files */
 	if (!(show_stage || show_deleted || show_others || show_unmerged ||
-	      show_killed || show_modified || show_resolve_undo))
+	      show_killed || show_modified || show_resolve_undo || show_json))
 		show_cached = 1;
+	if (show_json && (show_stage || show_deleted || show_others ||
+			  show_unmerged || show_killed || show_modified ||
+			  show_cached || pathspec.nr))
+		die(_("--debug-json cannot be used with other file selection options"));
+	if (show_json && show_resolve_undo)
+		die(_("--debug-json cannot be used with %s"), "--resolve-undo");
+	if (show_json && with_tree)
+		die(_("--debug-json cannot be used with %s"), "--with-tree");
+	if (show_json && debug_mode)
+		die(_("--debug-json cannot be used with %s"), "--debug");
 
 	if (with_tree) {
 		/*
@@ -673,10 +687,22 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)
 		overlay_tree_on_index(the_repository->index, with_tree, max_prefix);
 	}
 
-	show_files(the_repository, &dir);
-
-	if (show_resolve_undo)
-		show_ru_info(the_repository->index);
+	if (!show_json) {
+		show_files(the_repository, &dir);
+
+		if (show_resolve_undo)
+			show_ru_info(the_repository->index);
+	} else {
+		struct json_writer jw = JSON_WRITER_INIT;
+
+		discard_index(the_repository->index);
+		the_repository->index->jw = &jw;
+		if (repo_read_index(the_repository) < 0)
+			die("index file corrupt");
+		puts(jw.json.buf);
+		the_repository->index->jw = NULL;
+		jw_release(&jw);
+	}
 
 	if (ps_matched) {
 		int bad;
cache.h +2 -0
index bf20337ef4..84d0aeed20 100644
@@ -326,6 +326,7 @@ static inline unsigned int canon_mode(unsigned int mode)
 #define UNTRACKED_CHANGED	(1 << 7)
 #define FSMONITOR_CHANGED	(1 << 8)
 
+struct json_writer;
 struct split_index;
 struct untracked_cache;
 
@@ -350,6 +351,7 @@ struct index_state {
 	uint64_t fsmonitor_last_update;
 	struct ewah_bitmap *fsmonitor_dirty;
 	struct mem_pool *ce_mem_pool;
+	struct json_writer *jw;
 };
 
 /* Name hashing */
json-writer.c +22 -0
index aadb9dbddc..0608726512 100644
@@ -202,6 +202,28 @@ void jw_object_null(struct json_writer *jw, const char *key)
 	strbuf_addstr(&jw->json, "null");
 }
 
+void jw_object_filemode(struct json_writer *jw, const char *key, mode_t mode)
+{
+	object_common(jw, key);
+	strbuf_addf(&jw->json, "\"%06o\"", mode);
+}
+
+void jw_object_stat_data(struct json_writer *jw, const char *name,
+			 const struct stat_data *sd)
+{
+	jw_object_inline_begin_object(jw, name);
+	jw_object_intmax(jw, "ctime_sec", sd->sd_ctime.sec);
+	jw_object_intmax(jw, "ctime_nsec", sd->sd_ctime.nsec);
+	jw_object_intmax(jw, "mtime_sec", sd->sd_mtime.sec);
+	jw_object_intmax(jw, "mtime_nsec", sd->sd_mtime.nsec);
+	jw_object_intmax(jw, "device", sd->sd_dev);
+	jw_object_intmax(jw, "inode", sd->sd_ino);
+	jw_object_intmax(jw, "uid", sd->sd_uid);
+	jw_object_intmax(jw, "gid", sd->sd_gid);
+	jw_object_intmax(jw, "size", sd->sd_size);
+	jw_end(jw);
+}
+
 static void increase_indent(struct strbuf *sb,
 			    const struct json_writer *jw,
 			    int indent)
json-writer.h +23 -0
index 83906b09c1..c48c4cbf33 100644
@@ -42,8 +42,11 @@
  * of the given strings.
  */
 
+#include "git-compat-util.h"
 #include "strbuf.h"
 
+struct stat_data;
+
 struct json_writer
 {
 	/*
@@ -81,6 +84,9 @@ void jw_object_true(struct json_writer *jw, const char *key);
 void jw_object_false(struct json_writer *jw, const char *key);
 void jw_object_bool(struct json_writer *jw, const char *key, int value);
 void jw_object_null(struct json_writer *jw, const char *key);
+void jw_object_filemode(struct json_writer *jw, const char *key, mode_t value);
+void jw_object_stat_data(struct json_writer *jw, const char *key,
+			 const struct stat_data *sd);
 void jw_object_sub_jw(struct json_writer *jw, const char *key,
 		      const struct json_writer *value);
 
@@ -104,4 +110,21 @@ void jw_array_inline_begin_array(struct json_writer *jw);
 int jw_is_terminated(const struct json_writer *jw);
 void jw_end(struct json_writer *jw);
 
+/*
+ * These _gently versions accept NULL json_writer to reduce too much
+ * branching at the call site.
+ */
+static inline void jw_object_inline_begin_array_gently(struct json_writer *jw,
+						       const char *name)
+{
+	if (jw)
+		jw_object_inline_begin_array(jw, name);
+}
+
+static inline void jw_end_gently(struct json_writer *jw)
+{
+	if (jw)
+		jw_end(jw);
+}
+
 #endif /* JSON_WRITER_H */
read-cache.c +70 -2

132 lines not on this page · raw message

t/t3011-ls-files-json.sh new file +44 -0

44 lines not on this page · raw message

t/t3011/basic new file +67 -0

67 lines not on this page · raw message

Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
#2 lore raw

Re: [PATCH v2 01/10] ls-files: add --json to dump the index

Quoted 15 lines · Nguyễn Thái Ngọc Duy

So far we don't have a command to basically dump the index file out, with all its glory details. Checking some info, for example, stat time, usually involves either writing new code or firing up "xxd" and decoding values by yourself.

This --json is supposed to help that. It dumps the index in a human readable format but also easy to be processed with tools. And it will print almost enough info to reconstruct the index later.

In this patch we only dump the main part, not extensions. But at the end of the series, the entire index is dumped. The end result could be very verbose even on a small repository such as git.git.

---

...

Quoted 6 lines
json-writer.c +0 -0
index aadb9dbddc..0608726512 100644
@@ -202,6 +202,28 @@ void jw_object_null(struct json_writer *jw, const char *key)
  	strbuf_addstr(&jw->json, "null");

json-writer.c void jw_object_null(struct json_writer *jw, const char *key) · line 212

Quoted 10 lines
json-writer.c +8 -0
  }
 
+void jw_object_filemode(struct json_writer *jw, const char *key, mode_t mode)
+{
+	object_common(jw, key);
+	strbuf_addf(&jw->json, "\"%06o\"", mode);
+}
+
+void jw_object_stat_data(struct json_writer *jw, const char *name,
+			 const struct stat_data *sd)

Should this be in json_writer.c or in read-cache.c ? Currently, json_writer.c is concerned with formatting JSON on basic/scalar types. Do we want to start extending it to handle arbitrary structures? Or would it be better for the code that defines/manipulates the structure to define a "stat_data_dump_json()" function.

I'm torn on the jw_object_filemode() function, JSON format limits us to decimal integers and there are places where I'd like to have hex, or in this case octal values.

I'm thinking it'd be better to have a helper function in read-cache.c that formats a local strbuf and calls js_object_string(&jw, key, buf);

json-writer.c void jw_object_null(struct json_writer *jw, const char *key) · line 218

Quoted 6 lines
json-writer.c +6 -0
+{
+	jw_object_inline_begin_object(jw, name);
+	jw_object_intmax(jw, "ctime_sec", sd->sd_ctime.sec);
+	jw_object_intmax(jw, "ctime_nsec", sd->sd_ctime.nsec);
+	jw_object_intmax(jw, "mtime_sec", sd->sd_mtime.sec);
+	jw_object_intmax(jw, "mtime_nsec", sd->sd_mtime.nsec);

It'd be nice if we could also have a formatted date for the mtime and ctime in addition to the integer values. (I'm not sure whether you'd always want them or make it a verbose option.)

Quoted 85 lines
json-writer.c +8 -0
+	jw_object_intmax(jw, "device", sd->sd_dev);
+	jw_object_intmax(jw, "inode", sd->sd_ino);
+	jw_object_intmax(jw, "uid", sd->sd_uid);
+	jw_object_intmax(jw, "gid", sd->sd_gid);
+	jw_object_intmax(jw, "size", sd->sd_size);
+	jw_end(jw);
+}
+
  static void increase_indent(struct strbuf *sb,
  			    const struct json_writer *jw,
  			    int indent)
json-writer.h +23 -0
index 83906b09c1..c48c4cbf33 100644
@@ -42,8 +42,11 @@
   * of the given strings.
   */
 
+#include "git-compat-util.h"
  #include "strbuf.h"
 
+struct stat_data;
+
  struct json_writer
  {
  	/*
@@ -81,6 +84,9 @@ void jw_object_true(struct json_writer *jw, const char *key);
  void jw_object_false(struct json_writer *jw, const char *key);
  void jw_object_bool(struct json_writer *jw, const char *key, int value);
  void jw_object_null(struct json_writer *jw, const char *key);
+void jw_object_filemode(struct json_writer *jw, const char *key, mode_t value);
+void jw_object_stat_data(struct json_writer *jw, const char *key,
+			 const struct stat_data *sd);
  void jw_object_sub_jw(struct json_writer *jw, const char *key,
  		      const struct json_writer *value);
 
@@ -104,4 +110,21 @@ void jw_array_inline_begin_array(struct json_writer *jw);
  int jw_is_terminated(const struct json_writer *jw);
  void jw_end(struct json_writer *jw);
 
+/*
+ * These _gently versions accept NULL json_writer to reduce too much
+ * branching at the call site.
+ */
+static inline void jw_object_inline_begin_array_gently(struct json_writer *jw,
+						       const char *name)
+{
+	if (jw)
+		jw_object_inline_begin_array(jw, name);
+}
+
+static inline void jw_end_gently(struct json_writer *jw)
+{
+	if (jw)
+		jw_end(jw);
+}
+
  #endif /* JSON_WRITER_H */
read-cache.c +11 -0
index 4dd22f4f6e..db5147d088 100644
@@ -25,6 +25,7 @@
  #include "fsmonitor.h"
  #include "thread-utils.h"
  #include "progress.h"
+#include "json-writer.h"
 
  /* Mask for the name length in ce_flags in the on-disk index */
 
@@ -1952,6 +1953,49 @@ static void *load_index_extensions(void *_data)
  	return NULL;
  }
 
+static void dump_cache_entry(struct index_state *istate,
+			     int index,
+			     unsigned long offset,
+			     const struct cache_entry *ce)
+{
+	struct json_writer *jw = istate->jw;
+
+	jw_array_inline_begin_object(jw);
+
+	/*

read-cache.c static void *load_index_extensions(void *_data) · line 1975

Quoted 10 lines
read-cache.c +10 -0
+	 * this is technically redundant, but it's for easier
+	 * navigation when there hundreds of entries
+	 */
+	jw_object_intmax(jw, "id", index);
+
+	jw_object_string(jw, "name", ce->name);
+
+	jw_object_filemode(jw, "mode", ce->ce_mode);
+
+	jw_object_intmax(jw, "flags", ce->ce_flags);

It would be nice to have the flags as a hex-formatted string in addition to (or instead of) the decimal integer value.

Quoted 72 lines
read-cache.c +33 -0
+	/*
+	 * again redundant info, just so you don't have to decode
+	 * flags values manually
+	 */
+	if (ce->ce_flags & CE_EXTENDED)
+		jw_object_true(jw, "extended_flags");
+	if (ce->ce_flags & CE_VALID)
+		jw_object_true(jw, "assume_unchanged");
+	if (ce->ce_flags & CE_INTENT_TO_ADD)
+		jw_object_true(jw, "intent_to_add");
+	if (ce->ce_flags & CE_SKIP_WORKTREE)
+		jw_object_true(jw, "skip_worktree");
+	if (ce_stage(ce))
+		jw_object_intmax(jw, "stage", ce_stage(ce));
+
+	jw_object_string(jw, "oid", oid_to_hex(&ce->oid));
+
+	jw_object_stat_data(jw, "stat", &ce->ce_stat_data);
+	jw_object_intmax(jw, "file_offset", offset);
+
+	jw_end(jw);
+}
+
  /*
   * A helper function that will load the specified range of cache entries
   * from the memory mapped file and add them to the given index.
@@ -1972,6 +2016,9 @@ static unsigned long load_cache_entry_block(struct index_state *istate,
  		ce = create_from_disk(ce_mem_pool, istate->version, disk_ce, &consumed, previous_ce);
  		set_index_entry(istate, i, ce);
 
+		if (istate->jw)
+			dump_cache_entry(istate, i, src_offset, ce);
+
  		src_offset += consumed;
  		previous_ce = ce;
  	}
@@ -1983,6 +2030,8 @@ static unsigned long load_all_cache_entries(struct index_state *istate,
  {
  	unsigned long consumed;
 
+	jw_object_inline_begin_array_gently(istate->jw, "entries");
+
  	if (istate->version == 4) {
  		mem_pool_init(&istate->ce_mem_pool,
  				estimate_cache_size_from_compressed(istate->cache_nr));
@@ -1993,6 +2042,8 @@ static unsigned long load_all_cache_entries(struct index_state *istate,
 
  	consumed = load_cache_entry_block(istate, istate->ce_mem_pool,
  					0, istate->cache_nr, mmap, src_offset, NULL);
+
+	jw_end_gently(istate->jw);
  	return consumed;
  }
 
@@ -2120,6 +2171,7 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist)
  	size_t extension_offset = 0;
  	int nr_threads, cpus;
  	struct index_entry_offset_table *ieot = NULL;
+	int jw_pretty = 1;
 
  	if (istate->initialized)
  		return istate->cache_nr;
@@ -2154,6 +2206,8 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist)
  	istate->cache_nr = ntohl(hdr->hdr_entries);
  	istate->cache_alloc = alloc_nr(istate->cache_nr);
  	istate->cache = xcalloc(istate->cache_alloc, sizeof(*istate->cache));
+	istate->timestamp.sec = st.st_mtime;
+	istate->timestamp.nsec = ST_MTIME_NSEC(st);
  	istate->initialized = 1;
 
  	p.istate = istate;

read-cache.c int do_read_index(struct index_state *istate, const char *path, int must_exist) · line 2238

Quoted 9 lines
read-cache.c +6 -0
@@ -2176,6 +2230,20 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist)
  	if (!HAVE_THREADS)
  		nr_threads = 1;
 
+	if (istate->jw) {
+		jw_object_begin(istate->jw, jw_pretty);
+		jw_object_intmax(istate->jw, "version", istate->version);
+		jw_object_string(istate->jw, "oid", oid_to_hex(&istate->oid));
+		jw_object_intmax(istate->jw, "mtime_sec", istate->timestamp.sec);
+		jw_object_intmax(istate->jw, "mtime_nsec", istate->timestamp.nsec);

again, it would be nice to also have a formated version of the mtime.

read-cache.c int do_read_index(struct index_state *istate, const char *path, int must_exist) · line 2244

Quoted 6 lines
read-cache.c +6 -0
+
+		/*
+		 * Threading may mess up json writing. This is for
+		 * debugging only, so performance is not a concern.
+		 */
+		nr_threads = 1;

yes. we should turn off threading when dumping to json.

Quoted 16 lines
read-cache.c +2 -2
+	}
+
  	if (nr_threads > 1) {
  		extension_offset = read_eoie_extension(mmap, mmap_size);
  		if (extension_offset) {
@@ -2204,8 +2272,6 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist)
  		src_offset += load_all_cache_entries(istate, mmap, mmap_size, src_offset);
  	}
 
-	istate->timestamp.sec = st.st_mtime;
-	istate->timestamp.nsec = ST_MTIME_NSEC(st);
 
  	/* if we created a thread, join it otherwise load the extensions on the primary thread */
  	if (extension_offset) {

read-cache.c int do_read_index(struct index_state *istate, const char *path, int must_exist) · line 2289

Quoted 8 lines
read-cache.c +2 -0
@@ -2216,6 +2282,8 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist)
  		p.src_offset = src_offset;
  		load_index_extensions(&p);
  	}
+	jw_end_gently(istate->jw);
+
  	munmap((void *)mmap, mmap_size);
 
  	/*

...

Thanks
Jeff

#3 lore raw

Re: [PATCH v2 01/10] ls-files: add --json to dump the index

Quoted 8 lines · Jeff Hostetler

On 6/24/2019 9:02 AM, Nguyễn Thái Ngọc Duy wrote:
...

json-writer.c void jw_object_null(struct json_writer *jw, const char *key) · line 212

Quoted 2 lines
json-writer.c +2 -0
+void jw_object_stat_data(struct json_writer *jw, const char *name,
+			 const struct stat_data *sd)

Should this be in json_writer.c or in read-cache.c ? Currently, json_writer.c is concerned with formatting JSON on basic/scalar types.

That's an interesting question.

In the longer run, is it reasonable to assume that the current callers of these functions will stay to be the only places that would want to say "For this path, various attributes like timestamps and other stuff are these values"? The answer is probably no.

Would it be reasonable to assume that the future callers are all likely to be closely related to what functions that are currently in read-cache.c do and would appear only in that file? I think the answer would also be no. Functions in entry.c may want to report "I've created/updated these filesystem entities and their stat info now look like so", for example. So (assuming that we would want to represent "struct stat" the same way no matter who wants to write it out), I think it is reasonable to have this function on the json side, not on Git side.

I do not think it is a bad idea to have a layer that sits above json-writer.c and below read-cache.c that give us standardized mapping from in-core objects (like "struct stat") to objects in json (they are set of key-value pairs after all). Call it json-schema.c or something?

What other "higher than scaler" types do we foresee that we'd need to serialize in a standardised way? If we do not have that many yet, perhaps a split like that might be a bit premature and we can start by having the function in json-writer.c side, not in the API consumer side.

Thanks.

#4 lore raw

Re: [PATCH v2 01/10] ls-files: add --json to dump the index

In reply to Jeff Hostetler

Hi Jeff,

Quoted 13 lines · Jeff Hostetler

On 6/24/2019 9:02 AM, Nguyễn Thái Ngọc Duy wrote:
[...]

json-writer.c void jw_object_null(struct json_writer *jw, const char *key) · line 218

Quoted 6 lines
json-writer.c +6 -0
+{
+	jw_object_inline_begin_object(jw, name);
+	jw_object_intmax(jw, "ctime_sec", sd->sd_ctime.sec);
+	jw_object_intmax(jw, "ctime_nsec", sd->sd_ctime.nsec);
+	jw_object_intmax(jw, "mtime_sec", sd->sd_mtime.sec);
+	jw_object_intmax(jw, "mtime_nsec", sd->sd_mtime.nsec);

It'd be nice if we could also have a formatted date for the mtime and ctime in addition to the integer values. (I'm not sure whether you'd always want them or make it a verbose option.)

It would be more consistent with JSON conventions to have only the formatted date, preferably in ISO-8601 format [*1*], I would think.

And for debugging, it would also make more sense, in my opinion.

Quoted 32 lines

[...]

Quoted 31 lines
read-cache.c +16 -0
index 4dd22f4f6e..db5147d088 100644
@@ -25,6 +25,7 @@
  #include "fsmonitor.h"
  #include "thread-utils.h"
  #include "progress.h"
+#include "json-writer.h"
 
  /* Mask for the name length in ce_flags in the on-disk index */
 
@@ -1952,6 +1953,49 @@ static void *load_index_extensions(void *_data)
  	return NULL;
  }
 
+static void dump_cache_entry(struct index_state *istate,
+			     int index,
+			     unsigned long offset,
+			     const struct cache_entry *ce)
+{
+	struct json_writer *jw = istate->jw;
+
+	jw_array_inline_begin_object(jw);
+
+	/*
+	 * this is technically redundant, but it's for easier
+	 * navigation when there hundreds of entries
+	 */
+	jw_object_intmax(jw, "id", index);
+
Quoted 8 lines

read-cache.c static void *load_index_extensions(void *_data) · line 1975

Quoted 5 lines
read-cache.c +5 -0
+	jw_object_string(jw, "name", ce->name);
+
+	jw_object_filemode(jw, "mode", ce->ce_mode);
+
+	jw_object_intmax(jw, "flags", ce->ce_flags);

It would be nice to have the flags as a hex-formatted string in addition to (or instead of) the decimal integer value.

Instead of, please, prefixed with "0x" to make it clear that this is hex.

And "mode" in octal, please, prefixed with "0", as that is the convention to display file modes.

Thanks,
Dscho

Footnote *1*: It is too bad that JSON leaves the exact date format unspecified, leading to a lot of inconsistency and putting the burden on parsers:
https://stackoverflow.com/questions/10286204/the-right-json-date-format

author
#5 lore raw

Re: [PATCH v2 01/10] ls-files: add --json to dump the index

In reply to Jeff Hostetler
Quoted 20 lines · Jeff Hostetler

json-writer.c void jw_object_null(struct json_writer *jw, const char *key) · line 212

Quoted 12 lines
json-writer.c +8 -0
@@ -202,6 +202,28 @@ void jw_object_null(struct json_writer *jw, const char *key)
      strbuf_addstr(&jw->json, "null");
  }
 
+void jw_object_filemode(struct json_writer *jw, const char *key, mode_t mode)
+{
+     object_common(jw, key);
+     strbuf_addf(&jw->json, "\"%06o\"", mode);
+}
+
+void jw_object_stat_data(struct json_writer *jw, const char *name,
+                      const struct stat_data *sd)

Should this be in json_writer.c or in read-cache.c ? Currently, json_writer.c is concerned with formatting JSON on basic/scalar types. Do we want to start extending it to handle arbitrary structures? Or would it be better for the code that defines/manipulates the structure to define a "stat_data_dump_json()" function.

Quoted 7 lines

I'm torn on the jw_object_filemode() function, JSON format limits us to decimal integers and there are places where I'd like to have hex, or in this case octal values.

I'm thinking it'd be better to have a helper function in read-cache.c that formats a local strbuf and calls js_object_string(&jw, key, buf);

I can move these back to read-cache.c. Though if we have a lot more jw helpers like this (hard to tell at the moment) then perhaps we can have json-writer-utils.c or something to group them together. That keep the "boring" code out of main logic code in read-cache.c and other call sites.

Quoted 19 lines
Quoted 19 lines
read-cache.c +15 -0
@@ -1952,6 +1953,49 @@ static void *load_index_extensions(void *_data)
      return NULL;
  }
 
+static void dump_cache_entry(struct index_state *istate,
+                          int index,
+                          unsigned long offset,
+                          const struct cache_entry *ce)
+{
+     struct json_writer *jw = istate->jw;
+
+     jw_array_inline_begin_object(jw);
+
+     /*
+      * this is technically redundant, but it's for easier
+      * navigation when there hundreds of entries
+      */
+     jw_object_intmax(jw, "id", index);
+
Quoted 8 lines

read-cache.c static void *load_index_extensions(void *_data) · line 1975

Quoted 5 lines
read-cache.c +5 -0
+     jw_object_string(jw, "name", ce->name);
+
+     jw_object_filemode(jw, "mode", ce->ce_mode);
+
+     jw_object_intmax(jw, "flags", ce->ce_flags);

It would be nice to have the flags as a hex-formatted string in addition to (or instead of) the decimal integer value.

I'm not against reformatting it in hex string, but is there a value in it? ce_flags is expanded in the code below so that you don't have to decode it yourself when you read each entry. The "flags" field here is for further processing in tools. I'm trying to see if looking at hex values helps, but I'm still not seeing it...

Quoted 14 lines
Quoted 14 lines
read-cache.c +14 -0
+     /*
+      * again redundant info, just so you don't have to decode
+      * flags values manually
+      */
+     if (ce->ce_flags & CE_EXTENDED)
+             jw_object_true(jw, "extended_flags");
+     if (ce->ce_flags & CE_VALID)
+             jw_object_true(jw, "assume_unchanged");
+     if (ce->ce_flags & CE_INTENT_TO_ADD)
+             jw_object_true(jw, "intent_to_add");
+     if (ce->ce_flags & CE_SKIP_WORKTREE)
+             jw_object_true(jw, "skip_worktree");
+     if (ce_stage(ce))
+             jw_object_intmax(jw, "stage", ce_stage(ce));
Signature · 1 line
Duy
#6 lore raw

Re: [PATCH v2 01/10] ls-files: add --json to dump the index

Quoted 29 lines · Duy Nguyen
Quoted 20 lines · Jeff Hostetler

json-writer.c void jw_object_null(struct json_writer *jw, const char *key) · line 212

Quoted 12 lines
json-writer.c +8 -0
@@ -202,6 +202,28 @@ void jw_object_null(struct json_writer *jw, const char *key)
       strbuf_addstr(&jw->json, "null");
   }
 
+void jw_object_filemode(struct json_writer *jw, const char *key, mode_t mode)
+{
+     object_common(jw, key);
+     strbuf_addf(&jw->json, "\"%06o\"", mode);
+}
+
+void jw_object_stat_data(struct json_writer *jw, const char *name,
+                      const struct stat_data *sd)

Should this be in json_writer.c or in read-cache.c ? Currently, json_writer.c is concerned with formatting JSON on basic/scalar types. Do we want to start extending it to handle arbitrary structures? Or would it be better for the code that defines/manipulates the structure to define a "stat_data_dump_json()" function.

Quoted 7 lines

I'm torn on the jw_object_filemode() function, JSON format limits us to decimal integers and there are places where I'd like to have hex, or in this case octal values.

I'm thinking it'd be better to have a helper function in read-cache.c that formats a local strbuf and calls js_object_string(&jw, key, buf);

Quoted 5 lines

I can move these back to read-cache.c. Though if we have a lot more jw helpers like this (hard to tell at the moment) then perhaps we can have json-writer-utils.c or something to group them together. That keep the "boring" code out of main logic code in read-cache.c and other call sites.

yeah, in an utils file or close to the "constructors" of the structure types. either one works.

Quoted 29 lines
Quoted 20 lines
Quoted 20 lines
read-cache.c +15 -0
@@ -1952,6 +1953,49 @@ static void *load_index_extensions(void *_data)
       return NULL;
   }
 
+static void dump_cache_entry(struct index_state *istate,
+                          int index,
+                          unsigned long offset,
+                          const struct cache_entry *ce)
+{
+     struct json_writer *jw = istate->jw;
+
+     jw_array_inline_begin_object(jw);
+
+     /*
+      * this is technically redundant, but it's for easier
+      * navigation when there hundreds of entries
+      */
+     jw_object_intmax(jw, "id", index);
+
Quoted 8 lines

read-cache.c static void *load_index_extensions(void *_data) · line 1975

Quoted 5 lines
read-cache.c +5 -0
+     jw_object_string(jw, "name", ce->name);
+
+     jw_object_filemode(jw, "mode", ce->ce_mode);
+
+     jw_object_intmax(jw, "flags", ce->ce_flags);

It would be nice to have the flags as a hex-formatted string in addition to (or instead of) the decimal integer value.

Quoted 6 lines

I'm not against reformatting it in hex string, but is there a value in it? ce_flags is expanded in the code below so that you don't have to decode it yourself when you read each entry. The "flags" field here is for further processing in tools. I'm trying to see if looking at hex values helps, but I'm still not seeing it...

I guess I was thinking of the in-memory bits and thinking it'd be useful to be able to dump the index immediately after reading it and then later after some operation or traversal and see the intermediate states. But I realize now that that's not what you're building. This is a dump it while you're reading it feature (and that's fine).

So, as long as you have all of the on-disk bits, we should be fine as you suggest.

Jeff

Quoted 14 lines
Quoted 14 lines
Quoted 14 lines
read-cache.c +14 -0
+     /*
+      * again redundant info, just so you don't have to decode
+      * flags values manually
+      */
+     if (ce->ce_flags & CE_EXTENDED)
+             jw_object_true(jw, "extended_flags");
+     if (ce->ce_flags & CE_VALID)
+             jw_object_true(jw, "assume_unchanged");
+     if (ce->ce_flags & CE_INTENT_TO_ADD)
+             jw_object_true(jw, "intent_to_add");
+     if (ce->ce_flags & CE_SKIP_WORKTREE)
+             jw_object_true(jw, "skip_worktree");
+     if (ce_stage(ce))
+             jw_object_intmax(jw, "stage", ce_stage(ce));
#7 lore raw

Re: [PATCH v2 01/10] ls-files: add --json to dump the index

In reply to Nguyễn Thái Ngọc Duy
Quoted 83 lines · Nguyễn Thái Ngọc Duy

So far we don't have a command to basically dump the index file out, with all its glory details. Checking some info, for example, stat time, usually involves either writing new code or firing up "xxd" and decoding values by yourself.

This --json is supposed to help that. It dumps the index in a human readable format but also easy to be processed with tools. And it will print almost enough info to reconstruct the index later.

In this patch we only dump the main part, not extensions. But at the end of the series, the entire index is dumped. The end result could be very verbose even on a small repository such as git.git.

---
 Documentation/git-ls-files.txt    |  5 +++
 builtin/ls-files.c                | 38 +++++++++++++---
 cache.h                           |  2 +
 json-writer.c                     | 22 ++++++++++
 json-writer.h                     | 23 ++++++++++
 read-cache.c                      | 72 ++++++++++++++++++++++++++++++-
 t/t3011-ls-files-json.sh (new +x) | 44 +++++++++++++++++++
 t/t3011/basic (new)               | 67 ++++++++++++++++++++++++++++
 8 files changed, 265 insertions(+), 8 deletions(-)

[...]

t/t3011-ls-files-json.sh new file +44 -0
new file mode 100755
index 0000000000..97bcd814be
@@ -0,0 +1,44 @@
+#!/bin/sh
+
+test_description='ls-files dumping json'
+
+. ./test-lib.sh
+
+strip_number() {
+	for name; do
+		echo 's/\("'$name'":\) [0-9]\+/\1 <number>/' >>filter.sed
+	done
+}
+
+strip_string() {
+	for name; do
+		echo 's/\("'$name'":\) ".*"/\1 <string>/' >>filter.sed
+	done
+}
+
+compare_json() {
+	git ls-files --debug-json >json &&
+	sed -f filter.sed json >filtered &&
+	test_cmp "$TEST_DIRECTORY"/t3011/"$1" filtered
+}
+
+test_expect_success 'setup' '
+	mkdir sub &&
+	echo one >one &&
+	git add one &&
+	echo 2 >sub/two &&
+	git add sub/two &&
+
+	echo intent-to-add >ita &&
+	git add -N ita &&
+
+	strip_number ctime_sec ctime_nsec mtime_sec mtime_nsec &&
+	strip_number device inode uid gid file_offset ext_size &&
+	strip_string oid ident
+'
+
+test_expect_success 'ls-files --json, main entries' '
+	compare_json basic
+'
+
+test_done
t/t3011/basic new file +0 -0
new file mode 100644
index 0000000000..9436445d90

t/t3011/basic line 2

Quoted 2 lines
t/t3011/basic new file +2 -0
@@ -0,0 +1,67 @@
+{
+  "version": 3,

This will break the test suite when 'GIT_TEST_INDEX_VERSION' is set to 4 for example. I think this applies to a few other tests in later patches as well.

Quoted 68 lines
t/t3011/basic new file +65 -0
+  "oid": <string>,
+  "mtime_sec": <number>,
+  "mtime_nsec": <number>,
+  "entries": [
+    {
+      "id": 0,
+      "name": "ita",
+      "mode": "100644",
+      "flags": 536887296,
+      "extended_flags": true,
+      "intent_to_add": true,
+      "oid": <string>,
+      "stat": {
+        "ctime_sec": <number>,
+        "ctime_nsec": <number>,
+        "mtime_sec": <number>,
+        "mtime_nsec": <number>,
+        "device": <number>,
+        "inode": <number>,
+        "uid": <number>,
+        "gid": <number>,
+        "size": 0
+      },
+      "file_offset": <number>
+    },
+    {
+      "id": 1,
+      "name": "one",
+      "mode": "100644",
+      "flags": 0,
+      "oid": <string>,
+      "stat": {
+        "ctime_sec": <number>,
+        "ctime_nsec": <number>,
+        "mtime_sec": <number>,
+        "mtime_nsec": <number>,
+        "device": <number>,
+        "inode": <number>,
+        "uid": <number>,
+        "gid": <number>,
+        "size": 4
+      },
+      "file_offset": <number>
+    },
+    {
+      "id": 2,
+      "name": "sub/two",
+      "mode": "100644",
+      "flags": 0,
+      "oid": <string>,
+      "stat": {
+        "ctime_sec": <number>,
+        "ctime_nsec": <number>,
+        "mtime_sec": <number>,
+        "mtime_nsec": <number>,
+        "device": <number>,
+        "inode": <number>,
+        "uid": <number>,
+        "gid": <number>,
+        "size": 2
+      },
+      "file_offset": <number>
+    }
+  ]
+}
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
#8 lore raw

Re: [PATCH v2 01/10] ls-files: add --json to dump the index

In reply to Nguyễn Thái Ngọc Duy

Hi Duy,

Quoted 6 lines · Nguyễn Thái Ngọc Duy
t/t3011-ls-files-json.sh new file +0 -0
new file mode 100755
index 0000000000..97bcd814be

t/t3011-ls-files-json.sh line 9

Quoted 9 lines
t/t3011-ls-files-json.sh new file +9 -0
@@ -0,0 +1,44 @@
+#!/bin/sh
+
+test_description='ls-files dumping json'
+
+. ./test-lib.sh
+
+strip_number() {
+	for name; do
+		echo 's/\("'$name'":\) [0-9]\+/\1 <number>/' >>filter.sed

This does not do what you think it does, in Ubuntu Xenial and on macOS:

https://dev.azure.com/gitgitgadget/git/_build/results?buildId=11408&view=ms.vss-test-web.build-test-results-tab&runId=27736&paneView=debug&resultId=105613

The \1 is expanded to the ASCII character 001. Therefore your test cases fail on almost all platforms.

Funnily enough, they pass on Windows...

Ciao,
Johannes

#9 lore raw

Re: [PATCH v2 01/10] ls-files: add --json to dump the index

Hi Duy,

Quoted 20 lines · Johannes Schindelin
Quoted 6 lines · Nguyễn Thái Ngọc Duy
t/t3011-ls-files-json.sh new file +0 -0
new file mode 100755
index 0000000000..97bcd814be

t/t3011-ls-files-json.sh line 9

Quoted 9 lines
t/t3011-ls-files-json.sh new file +9 -0
@@ -0,0 +1,44 @@
+#!/bin/sh
+
+test_description='ls-files dumping json'
+
+. ./test-lib.sh
+
+strip_number() {
+	for name; do
+		echo 's/\("'$name'":\) [0-9]\+/\1 <number>/' >>filter.sed

This does not do what you think it does, in Ubuntu Xenial and on macOS:

Quoted 4 lines

https://dev.azure.com/gitgitgadget/git/_build/results?buildId=11408&view=ms.vss-test-web.build-test-results-tab&runId=27736&paneView=debug&resultId=105613

The \1 is expanded to the ASCII character 001. Therefore your test cases fail on almost all platforms.

The strip_number()/strip_string() approach might look elegant from a design perspective, but from a readability perspective (and obviously, when one wants to make those tests more robust and cross-platform), it would be a lot better to do it more explicitly.

This patch on top of your patch series makes the test run correctly in my Linux and Windows setup, and much easier to understand:

-- snipsnap --

t/t3011-ls-files-json.sh +18 -16
index 9f4ad4c9cf..8b782c48e0 100755
@@ -4,18 +4,6 @@ test_description='ls-files dumping json'
 
 . ./test-lib.sh
 
-strip_number() {
-	for name; do
-		echo 's/\("'$name'":\) [0-9]\+/\1 <number>/' >>filter.sed
-	done
-}
-
-strip_string() {
-	for name; do
-		echo 's/\("'$name'":\) ".*"/\1 <string>/' >>filter.sed
-	done
-}
-
 compare_json() {
 	git ls-files --debug-json >json &&
 	sed -f filter.sed json >filtered &&
@@ -35,9 +23,21 @@ test_expect_success 'setup' '
 	echo intent-to-add >ita &&
 	git add -N ita &&
 
-	strip_number ctime_sec ctime_nsec mtime_sec mtime_nsec &&
-	strip_number device inode uid gid file_offset ext_size last_update &&
-	strip_string oid ident
+	cat >filter.sed <<-\EOF
+	s/\("ctime_sec":\) [0-9]\+/\1 <number>/
+	s/\("ctime_nsec":\) [0-9]\+/\1 <number>/
+	s/\("mtime_sec":\) [0-9]\+/\1 <number>/
+	s/\("mtime_nsec":\) [0-9]\+/\1 <number>/
+	s/\("device":\) [0-9]\+/\1 <number>/
+	s/\("inode":\) [0-9]\+/\1 <number>/
+	s/\("uid":\) [0-9]\+/\1 <number>/
+	s/\("gid":\) [0-9]\+/\1 <number>/
+	s/\("file_offset":\) [0-9]\+/\1 <number>/
+	s/\("ext_size":\) [0-9]\+/\1 <number>/
+	s/\("last_update":\) [0-9]\+/\1 <number>/
+	s/\("oid":\) ".*"/\1 <string>/
+	s/\("ident":\) ".*"/\1 <string>/
+	EOF
 '
 
 test_expect_success 'ls-files --json, main entries, UNTR and TREE' '
@@ -98,7 +98,9 @@ test_expect_success !SINGLE_CPU 'ls-files --json and multicore extensions' '
 		touch one two three four &&
 		git add . &&
 		cp ../filter.sed . &&
-		strip_number offset &&
+		cat >>filter.sed <<-\EOF &&
+		s/\("offset":\) [0-9]\+/\1 <number>/
+		EOF
 		compare_json eoie
 	)
 '
#10 lore raw

Re: [PATCH v2 01/10] ls-files: add --json to dump the index

Hi Duy,

Quoted 24 lines · Johannes Schindelin
t/t3011-ls-files-json.sh +0 -12
index 9f4ad4c9cf..8b782c48e0 100755
@@ -4,18 +4,6 @@ test_description='ls-files dumping json'
 
 . ./test-lib.sh
 
-strip_number() {
-	for name; do
-		echo 's/\("'$name'":\) [0-9]\+/\1 <number>/' >>filter.sed
-	done
-}
-
-strip_string() {
-	for name; do
-		echo 's/\("'$name'":\) ".*"/\1 <string>/' >>filter.sed
-	done
-}
-
 compare_json() {
 	git ls-files --debug-json >json &&
 	sed -f filter.sed json >filtered &&

t/t3011-ls-files-json.sh test_expect_success 'setup' ' · line 27

Quoted 8 lines
t/t3011-ls-files-json.sh +2 -3
@@ -35,9 +23,21 @@ test_expect_success 'setup' '
 	echo intent-to-add >ita &&
 	git add -N ita &&
 
-	strip_number ctime_sec ctime_nsec mtime_sec mtime_nsec &&
-	strip_number device inode uid gid file_offset ext_size last_update &&
-	strip_string oid ident
+	cat >filter.sed <<-\EOF
+	s/\("ctime_sec":\) [0-9]\+/\1 <number>/

And of course, \+ still isn't POSIX, so you have to write [0-9][1-9]* instead.

Ciao,
Johannes

Quoted 27 lines
t/t3011-ls-files-json.sh +16 -1
+	s/\("ctime_nsec":\) [0-9]\+/\1 <number>/
+	s/\("mtime_sec":\) [0-9]\+/\1 <number>/
+	s/\("mtime_nsec":\) [0-9]\+/\1 <number>/
+	s/\("device":\) [0-9]\+/\1 <number>/
+	s/\("inode":\) [0-9]\+/\1 <number>/
+	s/\("uid":\) [0-9]\+/\1 <number>/
+	s/\("gid":\) [0-9]\+/\1 <number>/
+	s/\("file_offset":\) [0-9]\+/\1 <number>/
+	s/\("ext_size":\) [0-9]\+/\1 <number>/
+	s/\("last_update":\) [0-9]\+/\1 <number>/
+	s/\("oid":\) ".*"/\1 <string>/
+	s/\("ident":\) ".*"/\1 <string>/
+	EOF
 '
 
 test_expect_success 'ls-files --json, main entries, UNTR and TREE' '
@@ -98,7 +98,9 @@ test_expect_success !SINGLE_CPU 'ls-files --json and multicore extensions' '
 		touch one two three four &&
 		git add . &&
 		cp ../filter.sed . &&
-		strip_number offset &&
+		cat >>filter.sed <<-\EOF &&
+		s/\("offset":\) [0-9]\+/\1 <number>/
+		EOF
 		compare_json eoie
 	)
 '
#11 lore raw

Re: [PATCH v2 01/10] ls-files: add --json to dump the index

In reply to Johannes Schindelin
Quoted 10 lines · Johannes Schindelin

t/t3011-ls-files-json.sh line 9

Quoted 1 line
t/t3011-ls-files-json.sh new file +1 -0
+		echo 's/\("'$name'":\) [0-9]\+/\1 <number>/' >>filter.sed

This does not do what you think it does, in Ubuntu Xenial and on macOS:

https://dev.azure.com/gitgitgadget/git/_build/results?buildId=11408&view=ms.vss-test-web.build-test-results-tab&runId=27736&paneView=debug&resultId=105613

The \1 is expanded to the ASCII character 001. Therefore your test cases fail on almost all platforms.

Funnily enough, they pass on Windows...

bash, dash and /bin/echo behave differently given

    $ echo 'foo \1 bar'

some 'echo' suffer from the "\<n>" interpolation. Some don't.

I think your spelled-out version downthread (except for stepping out of BRE which would break your sed script, as you realized) would be a much readable alternative.

Thanks.

#12 lore raw

Re: [PATCH v2 01/10] ls-files: add --json to dump the index

In reply to Nguyễn Thái Ngọc Duy
Quoted 1 line · Nguyễn Thái Ngọc Duy

This --json is supposed to help that. It dumps the index in a human

This and another one on the title need to become "--debug-json".

Are we expecting a reroll to introduce another layer above the most primitive json writer that knows the schema used to represent both system standard and our application-specific structures, or is the current arrangement to have them in json-writer.c until there are enough of them to warrant such a split good enough?

#13 lore raw

PATCH v2 2/10 read-cache.c: dump common extension info in json

In reply to Nguyễn Thái Ngọc Duy
---
 read-cache.c | 49 +++++++++++++++++++++++++++++++++++--------------
 1 file changed, 35 insertions(+), 14 deletions(-)
read-cache.c +35 -14
index db5147d088..4accd8bb08 100644
@@ -1694,8 +1694,26 @@ static int verify_hdr(const struct cache_header *hdr, unsigned long size)
 }
 
 static int read_index_extension(struct index_state *istate,
-				const char *ext, const char *data, unsigned long sz)
+				const char *map,
+				unsigned long *offset)
 {
+	int ret = 0;
+	const char *ext = map + *offset;
+	uint32_t sz = get_be32(ext + 4);
+	const char *data = ext + 8;
+
+	if (istate->jw) {
+		char buf[5];
+
+		memcpy(buf, ext, 4);
+		buf[4] = '\0';
+		jw_object_inline_begin_object(istate->jw, buf);
+
+		jw_object_intmax(istate->jw, "file_offset", *offset);
+		jw_object_intmax(istate->jw, "ext_size", sz);
+	}
+	*offset += sz + 8;
+
 	switch (CACHE_EXT(ext)) {
 	case CACHE_EXT_TREE:
 		istate->cache_tree = cache_tree_read(data, sz);
@@ -1704,8 +1722,7 @@ static int read_index_extension(struct index_state *istate,
 		istate->resolve_undo = resolve_undo_read(data, sz);
 		break;
 	case CACHE_EXT_LINK:
-		if (read_link_extension(istate, data, sz))
-			return -1;
+		ret = read_link_extension(istate, data, sz);
 		break;
 	case CACHE_EXT_UNTRACKED:
 		istate->untracked = read_untracked_extension(data, sz);
@@ -1719,12 +1736,14 @@ static int read_index_extension(struct index_state *istate,
 		break;
 	default:
 		if (*ext < 'A' || 'Z' < *ext)
-			return error(_("index uses %.4s extension, which we do not understand"),
+			ret = error(_("index uses %.4s extension, which we do not understand"),
 				     ext);
-		fprintf_ln(stderr, _("ignoring %.4s extension"), ext);
+		else
+			  fprintf_ln(stderr, _("ignoring %.4s extension"), ext);
 		break;
 	}
-	return 0;
+	jw_end_gently(istate->jw);
+	return ret;
 }
 
 static struct cache_entry *create_from_disk(struct mem_pool *ce_mem_pool,
@@ -1930,25 +1949,27 @@ static void *load_index_extensions(void *_data)
 {
 	struct load_index_extensions *p = _data;
 	unsigned long src_offset = p->src_offset;
+	int dump_json = 0;
 
 	while (src_offset <= p->mmap_size - the_hash_algo->rawsz - 8) {
-		/* After an array of active_nr index entries,
+		if (p->istate->jw && !dump_json) {
+			jw_object_inline_begin_object(p->istate->jw, "extensions");
+			dump_json = 1;
+		}
+		/*
+		 * After an array of active_nr index entries,
 		 * there can be arbitrary number of extended
 		 * sections, each of which is prefixed with
 		 * extension name (4-byte) and section length
 		 * in 4-byte network byte order.
 		 */
-		uint32_t extsize = get_be32(p->mmap + src_offset + 4);
-		if (read_index_extension(p->istate,
-					 p->mmap + src_offset,
-					 p->mmap + src_offset + 8,
-					 extsize) < 0) {
+		if (read_index_extension(p->istate, p->mmap, &src_offset) < 0) {
 			munmap((void *)p->mmap, p->mmap_size);
 			die(_("index file corrupt"));
 		}
-		src_offset += 8;
-		src_offset += extsize;
 	}
+	if (dump_json)
+		jw_end(p->istate->jw);
 
 	return NULL;
 }
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
#14 lore raw

PATCH v2 3/10 cache-tree.c: dump "TREE" extension as json

In reply to Nguyễn Thái Ngọc Duy
---
 cache-tree.c             | 36 +++++++++++++++++++++++++++++++-----
 cache-tree.h             |  5 ++++-
 read-cache.c             |  2 +-
 t/t3011-ls-files-json.sh |  4 +++-
 t/t3011/basic            | 20 +++++++++++++++++++-
 5 files changed, 58 insertions(+), 9 deletions(-)
cache-tree.c +31 -5
index b13bfaf71e..b6a233307e 100644
@@ -3,6 +3,7 @@
 #include "tree.h"
 #include "tree-walk.h"
 #include "cache-tree.h"
+#include "json-writer.h"
 #include "object-store.h"
 #include "replace-object.h"
 
@@ -492,7 +493,8 @@ void cache_tree_write(struct strbuf *sb, struct cache_tree *root)
 	write_one(sb, root, "", 0);
 }
 
-static struct cache_tree *read_one(const char **buffer, unsigned long *size_p)
+static struct cache_tree *read_one(const char **buffer, unsigned long *size_p,
+				   struct json_writer *jw)
 {
 	const char *buf = *buffer;
 	unsigned long size = *size_p;
@@ -546,6 +548,15 @@ static struct cache_tree *read_one(const char **buffer, unsigned long *size_p)
 			*buffer, subtree_nr);
 #endif
 
+	if (jw) {
+		if (it->entry_count >= 0) {
+			jw_object_string(jw, "oid", oid_to_hex(&it->oid));
+			jw_object_intmax(jw, "entry_count", it->entry_count);
+		} else {
+			jw_object_null(jw, "oid");
+		}
+		jw_object_inline_begin_array(jw, "subdirs");
+	}
 	/*
 	 * Just a heuristic -- we do not add directories that often but
 	 * we do not want to have to extend it immediately when we do,
@@ -559,12 +570,18 @@ static struct cache_tree *read_one(const char **buffer, unsigned long *size_p)
 		struct cache_tree_sub *subtree;
 		const char *name = buf;
 
-		sub = read_one(&buf, &size);
+		if (jw) {
+			jw_array_inline_begin_object(jw);
+			jw_object_string(jw, "name", name);
+		}
+		sub = read_one(&buf, &size, jw);
+		jw_end_gently(jw);
 		if (!sub)
 			goto free_return;
 		subtree = cache_tree_sub(it, name);
 		subtree->cache_tree = sub;
 	}
+	jw_end_gently(jw);
 	if (subtree_nr != it->subtree_nr)
 		die("cache-tree: internal error");
 	*buffer = buf;
@@ -576,11 +593,20 @@ static struct cache_tree *read_one(const char **buffer, unsigned long *size_p)
 	return NULL;
 }
 
-struct cache_tree *cache_tree_read(const char *buffer, unsigned long size)
+struct cache_tree *cache_tree_read(const char *buffer, unsigned long size,
+				   struct json_writer *jw)
 {
+	struct cache_tree *ret;
+
+	if (jw) {
+		jw_object_inline_begin_object(jw, "root");
+	}
 	if (buffer[0])
-		return NULL; /* not the whole tree */
-	return read_one(&buffer, &size);
+		ret = NULL; /* not the whole tree */
+	else
+		ret = read_one(&buffer, &size, jw);
+	jw_end_gently(jw);
+	return ret;
 }
 
 static struct cache_tree *cache_tree_find(struct cache_tree *it, const char *path)
cache-tree.h +4 -1
index 757bbc48bc..fc3c73284b 100644
@@ -6,6 +6,8 @@
 #include "tree-walk.h"
 
 struct cache_tree;
+struct json_writer;
+
 struct cache_tree_sub {
 	struct cache_tree *cache_tree;
 	int count;		/* internally used by update_one() */
@@ -28,7 +30,8 @@ void cache_tree_invalidate_path(struct index_state *, const char *);
 struct cache_tree_sub *cache_tree_sub(struct cache_tree *, const char *);
 
 void cache_tree_write(struct strbuf *, struct cache_tree *root);
-struct cache_tree *cache_tree_read(const char *buffer, unsigned long size);
+struct cache_tree *cache_tree_read(const char *buffer, unsigned long size,
+				   struct json_writer *jw);
 
 int cache_tree_fully_valid(struct cache_tree *);
 int cache_tree_update(struct index_state *, int);
read-cache.c +1 -1
index 4accd8bb08..d09ce42b9a 100644
@@ -1716,7 +1716,7 @@ static int read_index_extension(struct index_state *istate,
 
 	switch (CACHE_EXT(ext)) {
 	case CACHE_EXT_TREE:
-		istate->cache_tree = cache_tree_read(data, sz);
+		istate->cache_tree = cache_tree_read(data, sz, istate->jw);
 		break;
 	case CACHE_EXT_RESOLVE_UNDO:
 		istate->resolve_undo = resolve_undo_read(data, sz);
t/t3011-ls-files-json.sh +3 -1
index 97bcd814be..fc313f2c9a 100755
@@ -29,6 +29,8 @@ test_expect_success 'setup' '
 	echo 2 >sub/two &&
 	git add sub/two &&
 
+	git commit -m first &&
+
 	echo intent-to-add >ita &&
 	git add -N ita &&
 
@@ -37,7 +39,7 @@ test_expect_success 'setup' '
 	strip_string oid ident
 '
 
-test_expect_success 'ls-files --json, main entries' '
+test_expect_success 'ls-files --json, main entries and TREE' '
 	compare_json basic
 '
 
t/t3011/basic +19 -1
index 9436445d90..e27f5be5ff 100644
@@ -63,5 +63,23 @@
       },
       "file_offset": <number>
     }
-  ]
+  ],
+  "extensions": {
+    "TREE": {
+      "file_offset": <number>,
+      "ext_size": <number>,
+      "root": {
+        "oid": null,
+        "subdirs": [
+          {
+            "name": "sub",
+            "oid": <string>,
+            "entry_count": 1,
+            "subdirs": [
+            ]
+          }
+        ]
+      }
+    }
+  }
 }
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
#15 lore raw

PATCH v2 4/10 dir.c: dump "UNTR" extension as json

In reply to Nguyễn Thái Ngọc Duy

The big part of UNTR extension is dumped at the end instead of dumping as soon as we read it, because we actually "patch" some fields in untracked_cache_dir with EWAH bitmaps at the end.

---
 dir.c                    | 57 +++++++++++++++++++++++++++++++++++++++-
 dir.h                    |  4 ++-
 json-writer.h            |  6 +++++
 read-cache.c             |  2 +-
 t/t3011-ls-files-json.sh |  3 ++-
 t/t3011/basic            | 39 +++++++++++++++++++++++++++
 6 files changed, 107 insertions(+), 4 deletions(-)
dir.c +56 -1
index ba4a51c296..8808577ea3 100644
@@ -19,6 +19,7 @@
 #include "varint.h"
 #include "ewah/ewok.h"
 #include "fsmonitor.h"
+#include "json-writer.h"
 #include "submodule-config.h"
 
 /*
@@ -2826,7 +2827,42 @@ static void load_oid_stat(struct oid_stat *oid_stat, const unsigned char *data,
 	oid_stat->valid = 1;
 }
 
-struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz)
+static void jw_object_oid_stat(struct json_writer *jw, const char *key,
+			       const struct oid_stat *oid_stat)
+{
+	jw_object_inline_begin_object(jw, key);
+	jw_object_bool(jw, "valid", oid_stat->valid);
+	jw_object_string(jw, "oid", oid_to_hex(&oid_stat->oid));
+	jw_object_stat_data(jw, "stat", &oid_stat->stat);
+	jw_end(jw);
+}
+
+static void jw_object_untracked_cache_dir(struct json_writer *jw,
+					  const struct untracked_cache_dir *ucd)
+{
+	int i;
+
+	jw_object_bool(jw, "valid", ucd->valid);
+	jw_object_bool(jw, "check-only", ucd->check_only);
+	jw_object_stat_data(jw, "stat", &ucd->stat_data);
+	jw_object_string(jw, "exclude-oid", oid_to_hex(&ucd->exclude_oid));
+	jw_object_inline_begin_array(jw, "untracked");
+	for (i = 0; i < ucd->untracked_nr; i++)
+		jw_array_string(jw, ucd->untracked[i]);
+	jw_end(jw);
+
+	jw_object_inline_begin_object(jw, "dirs");
+	for (i = 0; i < ucd->dirs_nr; i++) {
+		jw_object_inline_begin_object(jw, ucd->dirs[i]->name);
+		jw_object_untracked_cache_dir(jw, ucd->dirs[i]);
+		jw_end(jw);
+	}
+	jw_end(jw);
+}
+
+struct untracked_cache *read_untracked_extension(const void *data,
+						 unsigned long sz,
+						 struct json_writer *jw)
 {
 	struct untracked_cache *uc;
 	struct read_data rd;
@@ -2864,6 +2900,19 @@ struct untracked_cache *read_untracked_extension(const void *data, unsigned long
 	uc->dir_flags = get_be32(next + ouc_offset(dir_flags));
 	exclude_per_dir = (const char *)next + exclude_per_dir_offset;
 	uc->exclude_per_dir = xstrdup(exclude_per_dir);
+
+	if (jw) {
+		jw_object_string(jw, "ident", ident);
+		jw_object_oid_stat(jw, "info_exclude", &uc->ss_info_exclude);
+		jw_object_oid_stat(jw, "excludes_file", &uc->ss_excludes_file);
+		jw_object_intmax(jw, "flags", uc->dir_flags);
+		if (uc->dir_flags & DIR_SHOW_OTHER_DIRECTORIES)
+			jw_object_bool(jw, "show_other_directories", 1);
+		if (uc->dir_flags & DIR_HIDE_EMPTY_DIRECTORIES)
+			jw_object_bool(jw, "hide_empty_directories", 1);
+		jw_object_string(jw, "excludes_per_dir", uc->exclude_per_dir);
+	}
+
 	/* NUL after exclude_per_dir is covered by sizeof(*ouc) */
 	next += exclude_per_dir_offset + strlen(exclude_per_dir) + 1;
 	if (next >= end)
@@ -2905,6 +2954,12 @@ struct untracked_cache *read_untracked_extension(const void *data, unsigned long
 	ewah_each_bit(rd.sha1_valid, read_oid, &rd);
 	next = rd.data;
 
+	if (jw) {
+		jw_object_inline_begin_object(jw, "root");
+		jw_object_untracked_cache_dir(jw, uc->root);
+		jw_end(jw);
+	}
+
 done:
 	free(rd.ucd);
 	ewah_free(rd.valid);
dir.h +3 -1
index 680079bbe3..80efdd05c4 100644
@@ -6,6 +6,8 @@
 #include "cache.h"
 #include "strbuf.h"
 
+struct json_writer;
+
 struct dir_entry {
 	unsigned int len;
 	char name[FLEX_ARRAY]; /* more */
@@ -362,7 +364,7 @@ void untracked_cache_remove_from_index(struct index_state *, const char *);
 void untracked_cache_add_to_index(struct index_state *, const char *);
 
 void free_untracked_cache(struct untracked_cache *);
-struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz);
+struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz, struct json_writer *jw);
 void write_untracked_extension(struct strbuf *out, struct untracked_cache *untracked);
 void add_untracked_cache(struct index_state *istate);
 void remove_untracked_cache(struct index_state *istate);
json-writer.h +6 -0
index c48c4cbf33..c3d0fbd1ef 100644
@@ -121,6 +121,12 @@ static inline void jw_object_inline_begin_array_gently(struct json_writer *jw,
 		jw_object_inline_begin_array(jw, name);
 }
 
+static inline void jw_array_inline_begin_object_gently(struct json_writer *jw)
+{
+	if (jw)
+		jw_array_inline_begin_object(jw);
+}
+
 static inline void jw_end_gently(struct json_writer *jw)
 {
 	if (jw)
read-cache.c +1 -1
index d09ce42b9a..a70df4b0a5 100644
@@ -1725,7 +1725,7 @@ static int read_index_extension(struct index_state *istate,
 		ret = read_link_extension(istate, data, sz);
 		break;
 	case CACHE_EXT_UNTRACKED:
-		istate->untracked = read_untracked_extension(data, sz);
+		istate->untracked = read_untracked_extension(data, sz, istate->jw);
 		break;
 	case CACHE_EXT_FSMONITOR:
 		read_fsmonitor_extension(istate, data, sz);
t/t3011-ls-files-json.sh +2 -1
index fc313f2c9a..082fe8e966 100755
@@ -30,6 +30,7 @@ test_expect_success 'setup' '
 	git add sub/two &&
 
 	git commit -m first &&
+	git update-index --untracked-cache &&
 
 	echo intent-to-add >ita &&
 	git add -N ita &&
@@ -39,7 +40,7 @@ test_expect_success 'setup' '
 	strip_string oid ident
 '
 
-test_expect_success 'ls-files --json, main entries and TREE' '
+test_expect_success 'ls-files --json, main entries, UNTR and TREE' '
 	compare_json basic
 '
 
t/t3011/basic +39 -0
index e27f5be5ff..8e049f5350 100644
@@ -80,6 +80,45 @@
           }
         ]
       }
+    },
+    "UNTR": {
+      "file_offset": <number>,
+      "ext_size": <number>,
+      "ident": <string>,
+      "info_exclude": {
+        "valid": true,
+        "oid": <string>,
+        "stat": {
+          "ctime_sec": <number>,
+          "ctime_nsec": <number>,
+          "mtime_sec": <number>,
+          "mtime_nsec": <number>,
+          "device": <number>,
+          "inode": <number>,
+          "uid": <number>,
+          "gid": <number>,
+          "size": 0
+        }
+      },
+      "excludes_file": {
+        "valid": true,
+        "oid": <string>,
+        "stat": {
+          "ctime_sec": <number>,
+          "ctime_nsec": <number>,
+          "mtime_sec": <number>,
+          "mtime_nsec": <number>,
+          "device": <number>,
+          "inode": <number>,
+          "uid": <number>,
+          "gid": <number>,
+          "size": 0
+        }
+      },
+      "flags": 6,
+      "show_other_directories": true,
+      "hide_empty_directories": true,
+      "excludes_per_dir": ".gitignore"
     }
   }
 }
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
#16 lore raw

Re: [PATCH v2 04/10] dir.c: dump "UNTR" extension as json

Quoted 27 lines · Nguyễn Thái Ngọc Duy

The big part of UNTR extension is dumped at the end instead of dumping as soon as we read it, because we actually "patch" some fields in untracked_cache_dir with EWAH bitmaps at the end.

---
  dir.c                    | 57 +++++++++++++++++++++++++++++++++++++++-
  dir.h                    |  4 ++-
  json-writer.h            |  6 +++++
  read-cache.c             |  2 +-
  t/t3011-ls-files-json.sh |  3 ++-
  t/t3011/basic            | 39 +++++++++++++++++++++++++++
  6 files changed, 107 insertions(+), 4 deletions(-)
dir.c +1 -0
index ba4a51c296..8808577ea3 100644
@@ -19,6 +19,7 @@
  #include "varint.h"
  #include "ewah/ewok.h"
  #include "fsmonitor.h"
+#include "json-writer.h"
  #include "submodule-config.h"
 
  /*

dir.c static void load_oid_stat(struct oid_stat *oid_stat, const unsigned char *data, · line 2831

Quoted 6 lines
dir.c +2 -1
@@ -2826,7 +2827,42 @@ static void load_oid_stat(struct oid_stat *oid_stat, const unsigned char *data,
  	oid_stat->valid = 1;
  }
 
-struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz)
+static void jw_object_oid_stat(struct json_writer *jw, const char *key,
+			       const struct oid_stat *oid_stat)

I was hoping to reserve the jw_, jw_object_, and jw_array_ prefixes to refer to public functions in the json-writer.[ch] files, rather than private functions elsewhere in the code. I think it is less confusing that way. IMHO.

It's fine to have such helper functions for local structure types, but I think it'd be better for them to not be named like other public functions.

Quoted 99 lines
dir.c +53 -0
+{
+	jw_object_inline_begin_object(jw, key);
+	jw_object_bool(jw, "valid", oid_stat->valid);
+	jw_object_string(jw, "oid", oid_to_hex(&oid_stat->oid));
+	jw_object_stat_data(jw, "stat", &oid_stat->stat);
+	jw_end(jw);
+}
+
+static void jw_object_untracked_cache_dir(struct json_writer *jw,
+					  const struct untracked_cache_dir *ucd)
+{
+	int i;
+
+	jw_object_bool(jw, "valid", ucd->valid);
+	jw_object_bool(jw, "check-only", ucd->check_only);
+	jw_object_stat_data(jw, "stat", &ucd->stat_data);
+	jw_object_string(jw, "exclude-oid", oid_to_hex(&ucd->exclude_oid));
+	jw_object_inline_begin_array(jw, "untracked");
+	for (i = 0; i < ucd->untracked_nr; i++)
+		jw_array_string(jw, ucd->untracked[i]);
+	jw_end(jw);
+
+	jw_object_inline_begin_object(jw, "dirs");
+	for (i = 0; i < ucd->dirs_nr; i++) {
+		jw_object_inline_begin_object(jw, ucd->dirs[i]->name);
+		jw_object_untracked_cache_dir(jw, ucd->dirs[i]);
+		jw_end(jw);
+	}
+	jw_end(jw);
+}
+
+struct untracked_cache *read_untracked_extension(const void *data,
+						 unsigned long sz,
+						 struct json_writer *jw)
  {
  	struct untracked_cache *uc;
  	struct read_data rd;
@@ -2864,6 +2900,19 @@ struct untracked_cache *read_untracked_extension(const void *data, unsigned long
  	uc->dir_flags = get_be32(next + ouc_offset(dir_flags));
  	exclude_per_dir = (const char *)next + exclude_per_dir_offset;
  	uc->exclude_per_dir = xstrdup(exclude_per_dir);
+
+	if (jw) {
+		jw_object_string(jw, "ident", ident);
+		jw_object_oid_stat(jw, "info_exclude", &uc->ss_info_exclude);
+		jw_object_oid_stat(jw, "excludes_file", &uc->ss_excludes_file);
+		jw_object_intmax(jw, "flags", uc->dir_flags);
+		if (uc->dir_flags & DIR_SHOW_OTHER_DIRECTORIES)
+			jw_object_bool(jw, "show_other_directories", 1);
+		if (uc->dir_flags & DIR_HIDE_EMPTY_DIRECTORIES)
+			jw_object_bool(jw, "hide_empty_directories", 1);
+		jw_object_string(jw, "excludes_per_dir", uc->exclude_per_dir);
+	}
+
  	/* NUL after exclude_per_dir is covered by sizeof(*ouc) */
  	next += exclude_per_dir_offset + strlen(exclude_per_dir) + 1;
  	if (next >= end)
@@ -2905,6 +2954,12 @@ struct untracked_cache *read_untracked_extension(const void *data, unsigned long
  	ewah_each_bit(rd.sha1_valid, read_oid, &rd);
  	next = rd.data;
 
+	if (jw) {
+		jw_object_inline_begin_object(jw, "root");
+		jw_object_untracked_cache_dir(jw, uc->root);
+		jw_end(jw);
+	}
+
  done:
  	free(rd.ucd);
  	ewah_free(rd.valid);
dir.h +3 -1
index 680079bbe3..80efdd05c4 100644
@@ -6,6 +6,8 @@
  #include "cache.h"
  #include "strbuf.h"
 
+struct json_writer;
+
  struct dir_entry {
  	unsigned int len;
  	char name[FLEX_ARRAY]; /* more */
@@ -362,7 +364,7 @@ void untracked_cache_remove_from_index(struct index_state *, const char *);
  void untracked_cache_add_to_index(struct index_state *, const char *);
 
  void free_untracked_cache(struct untracked_cache *);
-struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz);
+struct untracked_cache *read_untracked_extension(const void *data, unsigned long sz, struct json_writer *jw);
  void write_untracked_extension(struct strbuf *out, struct untracked_cache *untracked);
  void add_untracked_cache(struct index_state *istate);
  void remove_untracked_cache(struct index_state *istate);
json-writer.h +0 -0
index c48c4cbf33..c3d0fbd1ef 100644
@@ -121,6 +121,12 @@ static inline void jw_object_inline_begin_array_gently(struct json_writer *jw,
  		jw_object_inline_begin_array(jw, name);
  }

json-writer.h static inline void jw_object_inline_begin_array_gently(struct json_writer *jw, · line 132

Quoted 10 lines
json-writer.h +6 -0
 
+static inline void jw_array_inline_begin_object_gently(struct json_writer *jw)
+{
+	if (jw)
+		jw_array_inline_begin_object(jw);
+}
+
  static inline void jw_end_gently(struct json_writer *jw)
  {
  	if (jw)

I'm not sure about the need for these _gently versions, but maybe make them macros in json-writer.h

Jeff

#17 lore raw

PATCH v2 5/10 split-index.c: dump "link" extension as json

In reply to Nguyễn Thái Ngọc Duy
---
 json-writer.c             | 14 ++++++++++++++
 json-writer.h             |  3 +++
 split-index.c             |  9 ++++++++-
 t/t3011-ls-files-json.sh  | 14 ++++++++++++++
 t/t3011/split-index (new) | 39 +++++++++++++++++++++++++++++++++++++++
 5 files changed, 78 insertions(+), 1 deletion(-)
json-writer.c +14 -0
index 0608726512..c0bd302e4e 100644
@@ -1,4 +1,5 @@
 #include "cache.h"
+#include "ewah/ewok.h"
 #include "json-writer.h"
 
 void jw_init(struct json_writer *jw)
@@ -224,6 +225,19 @@ void jw_object_stat_data(struct json_writer *jw, const char *name,
 	jw_end(jw);
 }
 
+static void dump_ewah_one(size_t pos, void *jw)
+{
+	jw_array_intmax(jw, pos);
+}
+
+void jw_object_ewah(struct json_writer *jw, const char *key,
+		    struct ewah_bitmap *ewah)
+{
+	jw_object_inline_begin_array(jw, key);
+	ewah_each_bit(ewah, dump_ewah_one, jw);
+	jw_end(jw);
+}
+
 static void increase_indent(struct strbuf *sb,
 			    const struct json_writer *jw,
 			    int indent)
json-writer.h +3 -0
index c3d0fbd1ef..07d841d52a 100644
@@ -45,6 +45,7 @@
 #include "git-compat-util.h"
 #include "strbuf.h"
 
+struct ewah_bitmap;
 struct stat_data;
 
 struct json_writer
@@ -87,6 +88,8 @@ void jw_object_null(struct json_writer *jw, const char *key);
 void jw_object_filemode(struct json_writer *jw, const char *key, mode_t value);
 void jw_object_stat_data(struct json_writer *jw, const char *key,
 			 const struct stat_data *sd);
+void jw_object_ewah(struct json_writer *jw, const char *key,
+		    struct ewah_bitmap *ewah);
 void jw_object_sub_jw(struct json_writer *jw, const char *key,
 		      const struct json_writer *value);
 
split-index.c +8 -1
index e6154e4ea9..41552bf771 100644
@@ -1,4 +1,5 @@
 #include "cache.h"
+#include "json-writer.h"
 #include "split-index.h"
 #include "ewah/ewok.h"
 
@@ -25,7 +26,7 @@ int read_link_extension(struct index_state *istate,
 	data += the_hash_algo->rawsz;
 	sz -= the_hash_algo->rawsz;
 	if (!sz)
-		return 0;
+		goto done;
 	si->delete_bitmap = ewah_new();
 	ret = ewah_read_mmap(si->delete_bitmap, data, sz);
 	if (ret < 0)
@@ -38,6 +39,12 @@ int read_link_extension(struct index_state *istate,
 		return error("corrupt replace bitmap in link extension");
 	if (ret != sz)
 		return error("garbage at the end of link extension");
+done:
+	if (istate->jw) {
+		jw_object_string(istate->jw, "oid", oid_to_hex(&si->base_oid));
+		jw_object_ewah(istate->jw, "delete_bitmap", si->delete_bitmap);
+		jw_object_ewah(istate->jw, "replace_bitmap", si->replace_bitmap);
+	}
 	return 0;
 }
 
t/t3011-ls-files-json.sh +14 -0
index 082fe8e966..dbb572ce9d 100755
@@ -44,4 +44,18 @@ test_expect_success 'ls-files --json, main entries, UNTR and TREE' '
 	compare_json basic
 '
 
+test_expect_success 'ls-files --json, split index' '
+	git init split &&
+	(
+		cd split &&
+		echo one >one &&
+		git add one &&
+		git update-index --split-index &&
+		echo updated >>one &&
+		test_must_fail git -c splitIndex.maxPercentChange=100 update-index --refresh &&
+		cp ../filter.sed . &&
+		compare_json split-index
+	)
+'
+
 test_done
t/t3011/split-index new file +39 -0
new file mode 100644
index 0000000000..cdcc4ddded
@@ -0,0 +1,39 @@
+{
+  "version": 2,
+  "oid": <string>,
+  "mtime_sec": <number>,
+  "mtime_nsec": <number>,
+  "entries": [
+    {
+      "id": 0,
+      "name": "",
+      "mode": "100644",
+      "flags": 0,
+      "oid": <string>,
+      "stat": {
+        "ctime_sec": <number>,
+        "ctime_nsec": <number>,
+        "mtime_sec": <number>,
+        "mtime_nsec": <number>,
+        "device": <number>,
+        "inode": <number>,
+        "uid": <number>,
+        "gid": <number>,
+        "size": 4
+      },
+      "file_offset": <number>
+    }
+  ],
+  "extensions": {
+    "link": {
+      "file_offset": <number>,
+      "ext_size": <number>,
+      "oid": <string>,
+      "delete_bitmap": [
+      ],
+      "replace_bitmap": [
+        0
+      ]
+    }
+  }
+}
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
#18 lore raw

Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json

Quoted 26 lines · Nguyễn Thái Ngọc Duy
---
  json-writer.c             | 14 ++++++++++++++
  json-writer.h             |  3 +++
  split-index.c             |  9 ++++++++-
  t/t3011-ls-files-json.sh  | 14 ++++++++++++++
  t/t3011/split-index (new) | 39 +++++++++++++++++++++++++++++++++++++++
  5 files changed, 78 insertions(+), 1 deletion(-)
json-writer.c +4 -0
index 0608726512..c0bd302e4e 100644
@@ -1,4 +1,5 @@
  #include "cache.h"
+#include "ewah/ewok.h"
  #include "json-writer.h"
 
  void jw_init(struct json_writer *jw)
@@ -224,6 +225,19 @@ void jw_object_stat_data(struct json_writer *jw, const char *name,
  	jw_end(jw);
  }
 
+static void dump_ewah_one(size_t pos, void *jw)
+{
+	jw_array_intmax(jw, pos);

json-writer.c void jw_object_stat_data(struct json_writer *jw, const char *name, · line 240

Quoted 10 lines
json-writer.c +10 -0
+}
+
+void jw_object_ewah(struct json_writer *jw, const char *key,
+		    struct ewah_bitmap *ewah)
+{
+	jw_object_inline_begin_array(jw, key);
+	ewah_each_bit(ewah, dump_ewah_one, jw);
+	jw_end(jw);
+}
+

As I said in an earlier commit in this series, I'd prefer that we keep such data-structure-specific helper functions in the source file that defines the data structure rather than adding them to json-writer.[ch]

I'm curious how big these EWAHs will be in practice and how useful an array of integers will be (especially as the pretty format will be one integer per line). Perhaps it would helpful to have an extended example in one of the tests.

Would it be better to have the caller of ewah_each_bit() build a hex or bit string in a strbuf and then write it as a single string?

Jeff

author
#19 lore raw

Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json

Quoted 5 lines · Jeff Hostetler

I'm curious how big these EWAHs will be in practice and how useful an array of integers will be (especially as the pretty format will be one integer per line). Perhaps it would helpful to have an extended example in one of the tests.

It's one integer per updated entry. So if you have a giant index and updated every single one of them, the EWAH bitmap contains that many integers.

If it was easy to just merge these bitmaps back to the entry (e.g. in this example, add "replaced": true to entry zero) I would have done it. But we dump as we stream and it's already too late to do it.

Quoted 3 lines

Would it be better to have the caller of ewah_each_bit() build a hex or bit string in a strbuf and then write it as a single string?

I don't think the current EWAH representation is easy to read in the first place. You'll probably have to run through some script to update the main entries part and will have a much better view, but that's pretty quick. If it's for scripts, then it's probably best to keep as an array of integers, not a string. Less post processing.

Another reason for not merging to one string (might not be a very good argument though) is to help diff between two indexes. One-number-per-line works well with "git diff --no-index" while one long string is a bit harder. I did this kind of comparison when I made changes in read-cache.c and wanted to check if the new index file is completely broken, or just slighly broken.

Signature · 1 line
Duy
#20 lore raw

Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json

Quoted 19 lines · Duy Nguyen
Quoted 5 lines · Jeff Hostetler

I'm curious how big these EWAHs will be in practice and how useful an array of integers will be (especially as the pretty format will be one integer per line). Perhaps it would helpful to have an extended example in one of the tests.

It's one integer per updated entry. So if you have a giant index and updated every single one of them, the EWAH bitmap contains that many integers.

If it was easy to just merge these bitmaps back to the entry (e.g. in this example, add "replaced": true to entry zero) I would have done it. But we dump as we stream and it's already too late to do it.

Quoted 3 lines

Would it be better to have the caller of ewah_each_bit() build a hex or bit string in a strbuf and then write it as a single string?

Quoted 5 lines

I don't think the current EWAH representation is easy to read in the first place. You'll probably have to run through some script to update the main entries part and will have a much better view, but that's pretty quick. If it's for scripts, then it's probably best to keep as an array of integers, not a string. Less post processing.

I don't think the intent is to dump the EWAH directly, but instead to dump a string of the uncompressed bitmap. Something like:

	"delete_bitmap" : "01101101101"

instead of

	"delete_bitmap" : [ 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1 ]
Quoted 6 lines

Another reason for not merging to one string (might not be a very good argument though) is to help diff between two indexes. One-number-per-line works well with "git diff --no-index" while one long string is a bit harder. I did this kind of comparison when I made changes in read-cache.c and wanted to check if the new index file is completely broken, or just slighly broken.

You're right that the diff of the json output is an interesting use, and the "single string" output is not helpful. What about batches of 64-bit strings? For example:

	"delete_bitmap" : [
		"0101010101010101010101010101010101010101010101010101010101010101",
		"0101010101010101010101010101010101010101010101010101010101010101",
		"0101010101010101010101010101010101010101010101010101010101010101",
		"01010101010101"
	]

This could be a happy medium between the two options, but does require some extra work in the formatter.

Thanks,
-Stolee

author
#21 lore raw

Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json

Quoted 27 lines · Derrick Stolee
Quoted 19 lines · Duy Nguyen
Quoted 5 lines · Jeff Hostetler

I'm curious how big these EWAHs will be in practice and how useful an array of integers will be (especially as the pretty format will be one integer per line). Perhaps it would helpful to have an extended example in one of the tests.

It's one integer per updated entry. So if you have a giant index and updated every single one of them, the EWAH bitmap contains that many integers.

If it was easy to just merge these bitmaps back to the entry (e.g. in this example, add "replaced": true to entry zero) I would have done it. But we dump as we stream and it's already too late to do it.

Quoted 3 lines

Would it be better to have the caller of ewah_each_bit() build a hex or bit string in a strbuf and then write it as a single string?

Quoted 5 lines

I don't think the current EWAH representation is easy to read in the first place. You'll probably have to run through some script to update the main entries part and will have a much better view, but that's pretty quick. If it's for scripts, then it's probably best to keep as an array of integers, not a string. Less post processing.

Quoted 8 lines

I don't think the intent is to dump the EWAH directly, but instead to dump a string of the uncompressed bitmap. Something like:

        "delete_bitmap" : "01101101101"

instead of

        "delete_bitmap" : [ 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1 ]

I get this part. But the numbers in the array were the position of the set bits. It's not showing just the actual bit map.

The same bitmap would be currently displayed as

"delete_bitmap": [ 1, 2, 4, 5, 7, 8, 9, 11 ]

And that maps back to the entry[1], entry[2], entry[4]... in the index being deleted from the base index. So displaying as a real bit map actually adds more work for both the reader and the tool because you have to calculate the position either way. And it gets harder if the bit you're intereted in is on the far right.

Quoted 18 lines
Quoted 6 lines

Another reason for not merging to one string (might not be a very good argument though) is to help diff between two indexes. One-number-per-line works well with "git diff --no-index" while one long string is a bit harder. I did this kind of comparison when I made changes in read-cache.c and wanted to check if the new index file is completely broken, or just slighly broken.

You're right that the diff of the json output is an interesting use, and the "single string" output is not helpful. What about batches of 64-bit strings? For example:

        "delete_bitmap" : [
                "0101010101010101010101010101010101010101010101010101010101010101",
                "0101010101010101010101010101010101010101010101010101010101010101",
                "0101010101010101010101010101010101010101010101010101010101010101",
                "01010101010101"
        ]
Quoted 2 lines

This could be a happy medium between the two options, but does require some extra work in the formatter.

And the reader/parser too since you have to join that array back in one string first.
--
Duy

#22 lore raw

Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json

Quoted 42 lines · Duy Nguyen
Quoted 27 lines · Derrick Stolee
Quoted 19 lines · Duy Nguyen
Quoted 5 lines · Jeff Hostetler

I'm curious how big these EWAHs will be in practice and how useful an array of integers will be (especially as the pretty format will be one integer per line). Perhaps it would helpful to have an extended example in one of the tests.

It's one integer per updated entry. So if you have a giant index and updated every single one of them, the EWAH bitmap contains that many integers.

If it was easy to just merge these bitmaps back to the entry (e.g. in this example, add "replaced": true to entry zero) I would have done it. But we dump as we stream and it's already too late to do it.

Quoted 3 lines

Would it be better to have the caller of ewah_each_bit() build a hex or bit string in a strbuf and then write it as a single string?

Quoted 5 lines

I don't think the current EWAH representation is easy to read in the first place. You'll probably have to run through some script to update the main entries part and will have a much better view, but that's pretty quick. If it's for scripts, then it's probably best to keep as an array of integers, not a string. Less post processing.

Quoted 8 lines

I don't think the intent is to dump the EWAH directly, but instead to dump a string of the uncompressed bitmap. Something like:

         "delete_bitmap" : "01101101101"

instead of

         "delete_bitmap" : [ 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1 ]

I get this part. But the numbers in the array were the position of the set bits. It's not showing just the actual bit map.

The same bitmap would be currently displayed as

Quoted 7 lines

"delete_bitmap": [ 1, 2, 4, 5, 7, 8, 9, 11 ]

And that maps back to the entry[1], entry[2], entry[4]... in the index being deleted from the base index. So displaying as a real bit map actually adds more work for both the reader and the tool because you have to calculate the position either way. And it gets harder if the bit you're intereted in is on the far right.

Thanks for the clarification. That helps.

Jeff

#23 lore raw

Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json

Quoted 45 lines · Jeff Hostetler
Quoted 44 lines · Duy Nguyen
Quoted 27 lines · Derrick Stolee
Quoted 19 lines · Duy Nguyen
Quoted 5 lines · Jeff Hostetler

I'm curious how big these EWAHs will be in practice and how useful an array of integers will be (especially as the pretty format will be one integer per line).  Perhaps it would helpful to have an extended example in one of the tests.

It's one integer per updated entry. So if you have a giant index and updated every single one of them, the EWAH bitmap contains that many integers.

If it was easy to just merge these bitmaps back to the entry (e.g. in this example, add "replaced": true to entry zero) I would have done it. But we dump as we stream and it's already too late to do it.

Quoted 3 lines

Would it be better to have the caller of ewah_each_bit() build a hex or bit string in a strbuf and then write it as a single string?

Quoted 5 lines

I don't think the current EWAH representation is easy to read in the first place. You'll probably have to run through some script to update the main entries part and will have a much better view, but that's pretty quick. If it's for scripts, then it's probably best to keep as an array of integers, not a string. Less post processing.

Quoted 8 lines

I don't think the intent is to dump the EWAH directly, but instead to dump a string of the uncompressed bitmap. Something like:

         "delete_bitmap" : "01101101101"

instead of

         "delete_bitmap" : [ 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1 ]

I get this part. But the numbers in the array were the position of the set bits. It's not showing just the actual bit map.

The same bitmap would be currently displayed as

  "delete_bitmap": [ 1, 2, 4, 5, 7, 8, 9, 11 ]

Quoted 8 lines
Quoted 5 lines

And that maps back to the entry[1], entry[2], entry[4]... in the index being deleted from the base index. So displaying as a real bit map actually adds more work for both the reader and the tool because you have to calculate the position either way. And it gets harder if the bit you're intereted in is on the far right.

Thanks for the clarification.  That helps.

Same here! We expect these to be much smaller than the full set, correct?

Thanks,
-Stolee

author
#24 lore raw

Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json

Quoted 54 lines · Derrick Stolee
Quoted 52 lines · Jeff Hostetler
Quoted 49 lines · Duy Nguyen
Quoted 27 lines · Derrick Stolee
Quoted 19 lines · Duy Nguyen
Quoted 5 lines · Jeff Hostetler

I'm curious how big these EWAHs will be in practice and how useful an array of integers will be (especially as the pretty format will be one integer per line). Perhaps it would helpful to have an extended example in one of the tests.

It's one integer per updated entry. So if you have a giant index and updated every single one of them, the EWAH bitmap contains that many integers.

If it was easy to just merge these bitmaps back to the entry (e.g. in this example, add "replaced": true to entry zero) I would have done it. But we dump as we stream and it's already too late to do it.

Quoted 3 lines

Would it be better to have the caller of ewah_each_bit() build a hex or bit string in a strbuf and then write it as a single string?

Quoted 5 lines

I don't think the current EWAH representation is easy to read in the first place. You'll probably have to run through some script to update the main entries part and will have a much better view, but that's pretty quick. If it's for scripts, then it's probably best to keep as an array of integers, not a string. Less post processing.

Quoted 8 lines

I don't think the intent is to dump the EWAH directly, but instead to dump a string of the uncompressed bitmap. Something like:

         "delete_bitmap" : "01101101101"

instead of

         "delete_bitmap" : [ 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1 ]

I get this part. But the numbers in the array were the position of the set bits. It's not showing just the actual bit map.

The same bitmap would be currently displayed as

"delete_bitmap": [ 1, 2, 4, 5, 7, 8, 9, 11 ]

And that maps back to the entry[1], entry[2], entry[4]... in the index being deleted from the base index. So displaying as a real bit map actually adds more work for both the reader and the tool because you have to calculate the position either way. And it gets harder if the bit you're intereted in is on the far right.

Quoted 3 lines
Quoted 1 line

Thanks for the clarification. That helps.

Same here! We expect these to be much smaller than the full set, correct?

For split-index, the number of 1 bits should be about the size of your working set, not the index size. In the normal case, then yes it should be much smaller. After a big merge or branch switch, it could get as big as the index. But I would hope the logic to re-split the index kicks in, which essentially empties these bitmaps.

EWAH bitmap is also used in UNTR extension if I remember correctly. Those bitmaps may have as many bits as the directories you have in the index.

Quoted 3 lines

Thanks,
-Stolee

Signature · 1 line
Duy
#25 lore raw

Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json

In reply to Nguyễn Thái Ngọc Duy
Quoted 11 lines · Nguyễn Thái Ngọc Duy
t/t3011-ls-files-json.sh +3 -0
index 082fe8e966..dbb572ce9d 100755
@@ -44,4 +44,18 @@ test_expect_success 'ls-files --json, main entries, UNTR and TREE' '
 	compare_json basic
 '
 
+test_expect_success 'ls-files --json, split index' '
+	git init split &&
+	(

t/t3011-ls-files-json.sh test_expect_success 'ls-files --json, main entries, UNTR and TREE' ' · line 59

Quoted 10 lines
t/t3011-ls-files-json.sh +10 -0
+		cd split &&
+		echo one >one &&
+		git add one &&
+		git update-index --split-index &&
+		echo updated >>one &&
+		test_must_fail git -c splitIndex.maxPercentChange=100 update-index --refresh &&
+		cp ../filter.sed . &&
+		compare_json split-index
+	)
+'

I think this test should 'sane_unset GIT_TEST_SPLIT_INDEX'. Maybe it's not absolutely necessary, because the explicit '--split-index' and '-c splitIndex.maxPercentChange=100' would already fully control when index splitting is performed, eliminating any indeterminism inherent to GIT_TEST_SPLIT_INDEX... but unsetting it would reduce the cognitive load on future readers.

The same might apply to GIT_TEST_FSMONITOR in the following patch, and perhaps even to GIT_TEST_INDEX_THREADS.

#26 lore raw

Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json

In reply to Nguyễn Thái Ngọc Duy
Quoted 58 lines · Nguyễn Thái Ngọc Duy
t/t3011-ls-files-json.sh +14 -0
index 082fe8e966..dbb572ce9d 100755
@@ -44,4 +44,18 @@ test_expect_success 'ls-files --json, main entries, UNTR and TREE' '
 	compare_json basic
 '
 
+test_expect_success 'ls-files --json, split index' '
+	git init split &&
+	(
+		cd split &&
+		echo one >one &&
+		git add one &&
+		git update-index --split-index &&
+		echo updated >>one &&
+		test_must_fail git -c splitIndex.maxPercentChange=100 update-index --refresh &&
+		cp ../filter.sed . &&
+		compare_json split-index
+	)
+'
+
 test_done
t/t3011/split-index new file +29 -0
new file mode 100644
index 0000000000..cdcc4ddded
@@ -0,0 +1,39 @@
+{
+  "version": 2,
+  "oid": <string>,
+  "mtime_sec": <number>,
+  "mtime_nsec": <number>,
+  "entries": [
+    {
+      "id": 0,
+      "name": "",
+      "mode": "100644",
+      "flags": 0,
+      "oid": <string>,
+      "stat": {
+        "ctime_sec": <number>,
+        "ctime_nsec": <number>,
+        "mtime_sec": <number>,
+        "mtime_nsec": <number>,
+        "device": <number>,
+        "inode": <number>,
+        "uid": <number>,
+        "gid": <number>,
+        "size": 4
+      },
+      "file_offset": <number>
+    }
+  ],
+  "extensions": {
+    "link": {
+      "file_offset": <number>,

t/t3011/split-index line 39

Quoted 10 lines
t/t3011/split-index new file +10 -0
+      "ext_size": <number>,
+      "oid": <string>,
+      "delete_bitmap": [
+      ],
+      "replace_bitmap": [
+        0
+      ]
+    }
+  }
+}

This test is flaky, as reported in:

https://public-inbox.org/git/[email protected]/

This is because it relies on racy behaviour, namely that the following three commands

    echo one >one &&
    git add one &&
    git update-index --split-index &&

are executed within the same second, leaving 'one' racily clean. To deal with the racily clean file, 5581a019ba (split-index: smudge and add racily clean cache entries to split index, 2018-10-11) kicks in, and 'one's smudged index entry is stored both in the shared index and in the split index. That's why this test expects the offset 0 in the "replace_bitmap" array.

However, it's possible that a second boundary is crossed between writing to 'one' and splitting the index, and then 'one' is not racily clean, and its index entry is only stored in the shared index. Consequently, there are no index entries in the split index, so the "replace_bitmap" array ends up being empty, ultimately failing the test.

A 'test-tool chmtime' invocation or two could make the test deterministic (i.e it could make sure that 'one' is either always racily clean or it never is, whichever is preferred).

What I still don't understand, however, is that when the test fails this way, then the "entries" array ends up being empty as well. It looks as if the JSON dump included only index entries that were actually stored in '.git/index', but omitted entries that were only present in the shared index. I think this is wrong, and it should dump the unified view of the split and shared indexes. Or include all entries from the shared index as well. Or perhaps I'm completely missing something...

author
#27 lore raw

Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json

Quoted 89 lines · SZEDER Gábor
Quoted 58 lines · Nguyễn Thái Ngọc Duy
t/t3011-ls-files-json.sh +14 -0
index 082fe8e966..dbb572ce9d 100755
@@ -44,4 +44,18 @@ test_expect_success 'ls-files --json, main entries, UNTR and TREE' '
      compare_json basic
 '
 
+test_expect_success 'ls-files --json, split index' '
+     git init split &&
+     (
+             cd split &&
+             echo one >one &&
+             git add one &&
+             git update-index --split-index &&
+             echo updated >>one &&
+             test_must_fail git -c splitIndex.maxPercentChange=100 update-index --refresh &&
+             cp ../filter.sed . &&
+             compare_json split-index
+     )
+'
+
 test_done
t/t3011/split-index new file +29 -0
new file mode 100644
index 0000000000..cdcc4ddded
@@ -0,0 +1,39 @@
+{
+  "version": 2,
+  "oid": <string>,
+  "mtime_sec": <number>,
+  "mtime_nsec": <number>,
+  "entries": [
+    {
+      "id": 0,
+      "name": "",
+      "mode": "100644",
+      "flags": 0,
+      "oid": <string>,
+      "stat": {
+        "ctime_sec": <number>,
+        "ctime_nsec": <number>,
+        "mtime_sec": <number>,
+        "mtime_nsec": <number>,
+        "device": <number>,
+        "inode": <number>,
+        "uid": <number>,
+        "gid": <number>,
+        "size": 4
+      },
+      "file_offset": <number>
+    }
+  ],
+  "extensions": {
+    "link": {
+      "file_offset": <number>,

t/t3011/split-index line 39

Quoted 10 lines
t/t3011/split-index new file +10 -0
+      "ext_size": <number>,
+      "oid": <string>,
+      "delete_bitmap": [
+      ],
+      "replace_bitmap": [
+        0
+      ]
+    }
+  }
+}

This test is flaky, as reported in:

https://public-inbox.org/git/[email protected]/

This is because it relies on racy behaviour, namely that the following three commands

    echo one >one &&
    git add one &&
    git update-index --split-index &&

are executed within the same second, leaving 'one' racily clean. To deal with the racily clean file, 5581a019ba (split-index: smudge and add racily clean cache entries to split index, 2018-10-11) kicks in, and 'one's smudged index entry is stored both in the shared index and in the split index. That's why this test expects the offset 0 in the "replace_bitmap" array.

Quoted 6 lines

However, it's possible that a second boundary is crossed between writing to 'one' and splitting the index, and then 'one' is not racily clean, and its index entry is only stored in the shared index. Consequently, there are no index entries in the split index, so the "replace_bitmap" array ends up being empty, ultimately failing the test.

Yep. I came up with the same conclusion. But I still have a couple other things to update before resending.

Quoted 4 lines

A 'test-tool chmtime' invocation or two could make the test deterministic (i.e it could make sure that 'one' is either always racily clean or it never is, whichever is preferred).

Quoted 8 lines

What I still don't understand, however, is that when the test fails this way, then the "entries" array ends up being empty as well. It looks as if the JSON dump included only index entries that were actually stored in '.git/index', but omitted entries that were only present in the shared index. I think this is wrong, and it should dump the unified view of the split and shared indexes. Or include all entries from the shared index as well. Or perhaps I'm completely missing something...

The command is to dump .git/index, not the shared one. And since this is not a split index test, rather a (quite low-level) json dump test, I did not bother to also dump the shared index, which should look like a regular one. Producing a unified view in json might not be easy with the current code because it's tied to file reading code, nearly stream out json as we read the file, and split-index requires a post processing step. I could contribute a python script or something to combine shared/main index together. That way you can still see the combined one, but we don't have to add/maintain more C code.

Signature · 1 line
Duy
#28 lore raw

Re: [PATCH v2 05/10] split-index.c: dump "link" extension as json

Quoted 9 lines · Duy Nguyen

The command is to dump .git/index, not the shared one. And since this is not a split index test, rather a (quite low-level) json dump test, I did not bother to also dump the shared index, which should look like a regular one. Producing a unified view in json might not be easy with the current code because it's tied to file reading code, nearly stream out json as we read the file, and split-index requires a post processing step. I could contribute a python script or something to combine shared/main index together. That way you can still see the combined one, but we don't have to add/maintain more C code.

Well, such a post-processing is something external scripts shine at and exporting the internal data in json format is exactly to support these scripts, so it may make a good first test case ;-)

#29 lore raw

PATCH v2 6/10 fsmonitor.c: dump "FSMN" extension as json

In reply to Nguyễn Thái Ngọc Duy
---
 fsmonitor.c              |  6 ++++++
 t/t3011-ls-files-json.sh | 14 +++++++++++++-
 t/t3011/fsmonitor (new)  | 38 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 57 insertions(+), 1 deletion(-)
fsmonitor.c +6 -0
index 1dee0aded1..5ed55ad176 100644
@@ -3,6 +3,7 @@
 #include "dir.h"
 #include "ewah/ewok.h"
 #include "fsmonitor.h"
+#include "json-writer.h"
 #include "run-command.h"
 #include "strbuf.h"
 
@@ -50,6 +51,11 @@ int read_fsmonitor_extension(struct index_state *istate, const void *data,
 	}
 	istate->fsmonitor_dirty = fsmonitor_dirty;
 
+	if (istate->jw) {
+		jw_object_intmax(istate->jw, "version", hdr_version);
+		jw_object_intmax(istate->jw, "last_update", istate->fsmonitor_last_update);
+		jw_object_ewah(istate->jw, "dirty", fsmonitor_dirty);
+	}
 	trace_printf_key(&trace_fsmonitor, "read fsmonitor extension successful");
 	return 0;
 }
t/t3011-ls-files-json.sh +13 -1
index dbb572ce9d..25215f83ae 100755
@@ -36,7 +36,7 @@ test_expect_success 'setup' '
 	git add -N ita &&
 
 	strip_number ctime_sec ctime_nsec mtime_sec mtime_nsec &&
-	strip_number device inode uid gid file_offset ext_size &&
+	strip_number device inode uid gid file_offset ext_size last_update &&
 	strip_string oid ident
 '
 
@@ -58,4 +58,16 @@ test_expect_success 'ls-files --json, split index' '
 	)
 '
 
+test_expect_success 'ls-files --json, fsmonitor extension ' '
+	git init fsmonitor &&
+	(
+		cd fsmonitor &&
+		echo one >one &&
+		git add one &&
+		git update-index --fsmonitor &&
+		cp ../filter.sed . &&
+		compare_json fsmonitor
+	)
+'
+
 test_done
t/t3011/fsmonitor new file +38 -0
new file mode 100644
index 0000000000..17f2d4a664
@@ -0,0 +1,38 @@
+{
+  "version": 2,
+  "oid": <string>,
+  "mtime_sec": <number>,
+  "mtime_nsec": <number>,
+  "entries": [
+    {
+      "id": 0,
+      "name": "one",
+      "mode": "100644",
+      "flags": 0,
+      "oid": <string>,
+      "stat": {
+        "ctime_sec": <number>,
+        "ctime_nsec": <number>,
+        "mtime_sec": <number>,
+        "mtime_nsec": <number>,
+        "device": <number>,
+        "inode": <number>,
+        "uid": <number>,
+        "gid": <number>,
+        "size": 4
+      },
+      "file_offset": <number>
+    }
+  ],
+  "extensions": {
+    "FSMN": {
+      "file_offset": <number>,
+      "ext_size": <number>,
+      "version": 1,
+      "last_update": <number>,
+      "dirty": [
+        0
+      ]
+    }
+  }
+}
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
#30 lore raw

PATCH v2 7/10 resolve-undo.c: dump "REUC" extension as json

In reply to Nguyễn Thái Ngọc Duy
---
 read-cache.c             |  2 +-
 resolve-undo.c           | 30 +++++++++++++++++-
 resolve-undo.h           |  4 ++-
 t/t3011-ls-files-json.sh | 20 ++++++++++++
 t/t3011/rerere (new)     | 66 ++++++++++++++++++++++++++++++++++++++++
 5 files changed, 119 insertions(+), 3 deletions(-)
read-cache.c +1 -1
index a70df4b0a5..e5183636fc 100644
@@ -1719,7 +1719,7 @@ static int read_index_extension(struct index_state *istate,
 		istate->cache_tree = cache_tree_read(data, sz, istate->jw);
 		break;
 	case CACHE_EXT_RESOLVE_UNDO:
-		istate->resolve_undo = resolve_undo_read(data, sz);
+		istate->resolve_undo = resolve_undo_read(data, sz, istate->jw);
 		break;
 	case CACHE_EXT_LINK:
 		ret = read_link_extension(istate, data, sz);
resolve-undo.c +29 -1
index 236320f179..68921e3dfe 100644
@@ -1,5 +1,6 @@
 #include "cache.h"
 #include "dir.h"
+#include "json-writer.h"
 #include "resolve-undo.h"
 #include "string-list.h"
 
@@ -49,7 +50,30 @@ void resolve_undo_write(struct strbuf *sb, struct string_list *resolve_undo)
 	}
 }
 
-struct string_list *resolve_undo_read(const char *data, unsigned long size)
+static void dump_resolve_undo(struct json_writer *jw,
+			      const char *path,
+			      const struct resolve_undo_info *ui)
+{
+	int i;
+
+	if (!jw)
+		return;
+
+	jw_array_inline_begin_object(jw);
+	jw_object_string(jw, "path", path);
+
+	jw_object_inline_begin_array(jw, "stages");
+	for (i = 0; i < 3; i++) {
+		jw_array_inline_begin_object(jw);
+		jw_object_filemode(jw, "mode", ui->mode[i]);
+		jw_object_string(jw, "oid", oid_to_hex(&ui->oid[i]));
+		jw_end(jw);
+	}
+	jw_end(jw);
+}
+
+struct string_list *resolve_undo_read(const char *data, unsigned long size,
+				      struct json_writer *jw)
 {
 	struct string_list *resolve_undo;
 	size_t len;
@@ -59,6 +83,7 @@ struct string_list *resolve_undo_read(const char *data, unsigned long size)
 
 	resolve_undo = xcalloc(1, sizeof(*resolve_undo));
 	resolve_undo->strdup_strings = 1;
+	jw_object_inline_begin_array_gently(jw, "entries");
 
 	while (size) {
 		struct string_list_item *lost;
@@ -94,7 +119,10 @@ struct string_list *resolve_undo_read(const char *data, unsigned long size)
 			size -= rawsz;
 			data += rawsz;
 		}
+
+		dump_resolve_undo(jw, lost->string, ui);
 	}
+	jw_end_gently(jw);
 	return resolve_undo;
 
 error:
resolve-undo.h +3 -1
index 2b3f0f901e..46b4e93a7e 100644
@@ -3,6 +3,8 @@
 
 #include "cache.h"
 
+struct json_writer;
+
 struct resolve_undo_info {
 	unsigned int mode[3];
 	struct object_id oid[3];
@@ -10,7 +12,7 @@ struct resolve_undo_info {
 
 void record_resolve_undo(struct index_state *, struct cache_entry *);
 void resolve_undo_write(struct strbuf *, struct string_list *);
-struct string_list *resolve_undo_read(const char *, unsigned long);
+struct string_list *resolve_undo_read(const char *, unsigned long, struct json_writer *);
 void resolve_undo_clear_index(struct index_state *);
 int unmerge_index_entry_at(struct index_state *, int);
 void unmerge_index(struct index_state *, const struct pathspec *);
t/t3011-ls-files-json.sh +20 -0
index 25215f83ae..dc57138f5b 100755
@@ -70,4 +70,24 @@ test_expect_success 'ls-files --json, fsmonitor extension ' '
 	)
 '
 
+test_expect_success 'ls-files --json, rerere extension' '
+	git init rerere &&
+	(
+		cd rerere &&
+		mkdir fi &&
+		test_commit initial fi/le first &&
+		git branch side &&
+		test_commit second fi/le second &&
+		git checkout side &&
+		test_commit third fi/le third &&
+		git checkout master &&
+		git config rerere.enabled true &&
+		test_must_fail git merge side &&
+		echo resolved >fi/le &&
+		git add fi/le &&
+		cp ../filter.sed . &&
+		compare_json rerere
+	)
+'
+
 test_done
t/t3011/rerere new file +66 -0

66 lines not on this page · raw message

Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
#31 lore raw

PATCH v2 8/10 read-cache.c: dump "EOIE" extension as json

In reply to Nguyễn Thái Ngọc Duy
---
 read-cache.c             |  8 ++++
 t/t3011-ls-files-json.sh | 13 ++++++
 t/t3011/eoie (new)       | 96 ++++++++++++++++++++++++++++++++++++++++
 t/test-lib.sh            |  4 ++
 4 files changed, 121 insertions(+)
read-cache.c +8 -0
index e5183636fc..37491dd03d 100644
@@ -1731,6 +1731,14 @@ static int read_index_extension(struct index_state *istate,
 		read_fsmonitor_extension(istate, data, sz);
 		break;
 	case CACHE_EXT_ENDOFINDEXENTRIES:
+		if (istate->jw) {
+			/* must be synchronized with read_eoie_extension() */
+			jw_object_intmax(istate->jw, "offset", get_be32(data));
+			jw_object_string(istate->jw, "oid",
+					 hash_to_hex((const unsigned char*)data + sizeof(uint32_t)));
+		}
+		/* already handled in do_read_index() */
+		break;
 	case CACHE_EXT_INDEXENTRYOFFSETTABLE:
 		/* already handled in do_read_index() */
 		break;
t/t3011-ls-files-json.sh +13 -0
index dc57138f5b..9f4ad4c9cf 100755
@@ -90,4 +90,17 @@ test_expect_success 'ls-files --json, rerere extension' '
 	)
 '
 
+test_expect_success !SINGLE_CPU 'ls-files --json and multicore extensions' '
+	git init eoie &&
+	(
+		cd eoie &&
+		git config index.threads 2 &&
+		touch one two three four &&
+		git add . &&
+		cp ../filter.sed . &&
+		strip_number offset &&
+		compare_json eoie
+	)
+'
+
 test_done
t/t3011/eoie new file +96 -0

96 lines not on this page · raw message

t/test-lib.sh +4 -0
index 4b346467df..9d5b273b40 100644
@@ -1611,3 +1611,7 @@ test_lazy_prereq REBASE_P '
 test_lazy_prereq FAIL_PREREQS '
 	test -n "$GIT_TEST_FAIL_PREREQS"
 '
+
+test_lazy_prereq SINGLE_CPU '
+	test "$(test-tool online-cpus)" -eq 1
+'
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
#32 lore raw

PATCH v2 9/10 read-cache.c: dump "IEOT" extension as json

In reply to Nguyễn Thái Ngọc Duy
---
 read-cache.c | 43 ++++++++++++++++++++++++++++++++++++-------
 t/t3011/eoie | 13 ++++++++++++-
 2 files changed, 48 insertions(+), 8 deletions(-)
read-cache.c +36 -7
index 37491dd03d..c26edcc9d9 100644
@@ -1693,6 +1693,7 @@ static int verify_hdr(const struct cache_header *hdr, unsigned long size)
 	return 0;
 }
 
+static struct index_entry_offset_table *do_read_ieot_extension(struct index_state *, const char *, uint32_t);
 static int read_index_extension(struct index_state *istate,
 				const char *map,
 				unsigned long *offset)
@@ -1740,7 +1741,11 @@ static int read_index_extension(struct index_state *istate,
 		/* already handled in do_read_index() */
 		break;
 	case CACHE_EXT_INDEXENTRYOFFSETTABLE:
-		/* already handled in do_read_index() */
+		if (istate->jw) {
+			free(do_read_ieot_extension(istate, data, sz));
+		} else {
+			/* already handled in do_read_index() */
+		}
 		break;
 	default:
 		if (*ext < 'A' || 'Z' < *ext)
@@ -1938,7 +1943,7 @@ struct index_entry_offset_table
 	struct index_entry_offset entries[FLEX_ARRAY];
 };
 
-static struct index_entry_offset_table *read_ieot_extension(const char *mmap, size_t mmap_size, size_t offset);
+static struct index_entry_offset_table *read_ieot_extension(struct index_state *istate, const char *mmap, size_t mmap_size, size_t offset);
 static void write_ieot_extension(struct strbuf *sb, struct index_entry_offset_table *ieot);
 
 static size_t read_eoie_extension(const char *mmap, size_t mmap_size);
@@ -2292,7 +2297,7 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist)
 	 * to multi-thread the reading of the cache entries.
 	 */
 	if (extension_offset && nr_threads > 1)
-		ieot = read_ieot_extension(mmap, mmap_size, extension_offset);
+		ieot = read_ieot_extension(istate, mmap, mmap_size, extension_offset);
 
 	if (ieot) {
 		src_offset += load_cache_entries_threaded(istate, mmap, mmap_size, nr_threads, ieot);
@@ -3634,12 +3639,13 @@ static void write_eoie_extension(struct strbuf *sb, git_hash_ctx *eoie_context,
 
 #define IEOT_VERSION	(1)
 
-static struct index_entry_offset_table *read_ieot_extension(const char *mmap, size_t mmap_size, size_t offset)
+static struct index_entry_offset_table *read_ieot_extension(
+		struct index_state *istate,
+		const char *mmap, size_t mmap_size,
+		size_t offset)
 {
 	const char *index = NULL;
-	uint32_t extsize, ext_version;
-	struct index_entry_offset_table *ieot;
-	int i, nr;
+	uint32_t extsize;
 
 	/* find the IEOT extension */
 	if (!offset)
@@ -3655,6 +3661,17 @@ static struct index_entry_offset_table *read_ieot_extension(const char *mmap, si
 	}
 	if (!index)
 		return NULL;
+	return do_read_ieot_extension(istate, index, extsize);
+}
+
+static struct index_entry_offset_table *do_read_ieot_extension(
+		struct index_state *istate,
+		const char *index,
+		uint32_t extsize)
+{
+	uint32_t ext_version;
+	struct index_entry_offset_table *ieot;
+	int i, nr;
 
 	/* validate the version is IEOT_VERSION */
 	ext_version = get_be32(index);
@@ -3670,6 +3687,10 @@ static struct index_entry_offset_table *read_ieot_extension(const char *mmap, si
 		error("invalid number of IEOT entries %d", nr);
 		return NULL;
 	}
+	if (istate->jw) {
+		jw_object_intmax(istate->jw, "version", ext_version);
+		jw_object_inline_begin_array(istate->jw, "entries");
+	}
 	ieot = xmalloc(sizeof(struct index_entry_offset_table)
 		       + (nr * sizeof(struct index_entry_offset)));
 	ieot->nr = nr;
@@ -3678,7 +3699,15 @@ static struct index_entry_offset_table *read_ieot_extension(const char *mmap, si
 		index += sizeof(uint32_t);
 		ieot->entries[i].nr = get_be32(index);
 		index += sizeof(uint32_t);
+
+		if (istate->jw) {
+			jw_array_inline_begin_object(istate->jw);
+			jw_object_intmax(istate->jw, "offset", ieot->entries[i].offset);
+			jw_object_intmax(istate->jw, "count", ieot->entries[i].nr);
+			jw_end(istate->jw);
+		}
 	}
+	jw_end_gently(istate->jw);
 
 	return ieot;
 }
t/t3011/eoie +12 -1
index 85ec61517b..66a0feb3b6 100644
@@ -84,7 +84,18 @@
   "extensions": {
     "IEOT": {
       "file_offset": <number>,
-      "ext_size": <number>
+      "ext_size": <number>,
+      "version": 1,
+      "entries": [
+        {
+          "offset": <number>,
+          "count": 2
+        },
+        {
+          "offset": <number>,
+          "count": 2
+        }
+      ]
     },
     "EOIE": {
       "file_offset": <number>,
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
#33 lore raw

PATCH v2 10/10 t3008: use the new SINGLE_CPU prereq

In reply to Nguyễn Thái Ngọc Duy
---
 t/t3008-ls-files-lazy-init-name-hash.sh | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)
t/t3008-ls-files-lazy-init-name-hash.sh +1 -7
index 64f047332b..7f918c05f6 100755
@@ -4,15 +4,9 @@ test_description='Test the lazy init name hash with various folder structures'
 
 . ./test-lib.sh
 
-if test 1 -eq $($GIT_BUILD_DIR/t/helper/test-tool online-cpus)
-then
-	skip_all='skipping lazy-init tests, single cpu'
-	test_done
-fi
-
 LAZY_THREAD_COST=2000
 
-test_expect_success 'no buffer overflow in lazy_init_name_hash' '
+test_expect_success !SINGLE_CPU 'no buffer overflow in lazy_init_name_hash' '
 	(
 	    test_seq $LAZY_THREAD_COST | sed "s/^/a_/" &&
 	    echo b/b/b &&
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a
#34 lore raw
In reply to Nguyễn Thái Ngọc Duy

Hi Duy,

Quoted 3 lines · Nguyễn Thái Ngọc Duy

- json field names now use '_' instead of '.' to be friendlier to some languages. I stick to underscore_name instead of camelCase because the former is closer to what we use

This is not a good reason. People who are used to read JSON will stumble over this all the time because it is so uncommon.

Quoted 7 lines

- extension location is printed, in case you need to decode the extension by yourself (previously only the size is printed)
- all extensions are printed in the same order they appear in the file (previously eoie and ieot are printed first because that's how we parse)
- resolve undo extension is reorganized a bit to be easier to read
- tests added. Example json files are in t/t3011

It might actually make sense to optionally disable showing extensions.

You also forgot to mention that you explicitly disable handling <pathspec>, which I find a bit odd, personally, as that would probably come in real handy at times, especially when we offer this as a better way for 3rd-party applications to interact with Git (which I think will be the use case for this feature that will be _far_ more common than using it for debugging).

Ciao,
Dscho

#35 lore raw
Quoted 10 lines · Johannes Schindelin

Hi Duy,

Quoted 3 lines · Nguyễn Thái Ngọc Duy

- json field names now use '_' instead of '.' to be friendlier to some languages. I stick to underscore_name instead of camelCase because the former is closer to what we use

This is not a good reason. People who are used to read JSON will stumble over this all the time because it is so uncommon.

Getting rid of "." and "-" in field names is the important part. These confuse some languages and make us drop into object["<field>"] syntax in my experience.

As for "_" or camelCase (or PascalCase), I'm not sure it matters one way or the other. Personally, I'd vote for underscores.

Jeff

author
#36 lore raw
In reply to Johannes Schindelin
Quoted 8 lines · Johannes Schindelin
Quoted 7 lines

- extension location is printed, in case you need to decode the extension by yourself (previously only the size is printed)
- all extensions are printed in the same order they appear in the file (previously eoie and ieot are printed first because that's how we parse)
- resolve undo extension is reorganized a bit to be easier to read
- tests added. Example json files are in t/t3011

Quoted 5 lines

It might actually make sense to optionally disable showing extensions.

You also forgot to mention that you explicitly disable handling <pathspec>, which I find a bit odd, personally, as that would probably come in real handy at times,

No. I mentioned the land of high level languages before. Filtering in any Python, Ruby, Scheme, JavaScript, Java is a piece of cake and much more flexible than pathspec. Even with shell scripts, jq could do a much better job than pathspec. If you filter by pathspec, good luck trying that on extensions.

It's the same reason why I will not provide a flexible way to disable extensions. I'm not starting a JSON API for Git. I provide an index file in JSON format. You do what you want with it. You have a format easy enough to import to native data structures of your favorite language.

Quoted 4 lines

especially when we offer this as a better way
for 3rd-party applications to interact with Git (which I think will be the use case for this feature that will be _far_ more common than using it for debugging).

We may have conflicting goals. For me, first priority is the debug tool for Git developers. 3rd-party support is a stretch. I could move all this back to test-tool, then you can provide a 3rd-party API if you want. Or I'll withdraw this series and go back to my original plan.

Signature · 1 line
Duy
#37 lore raw
Quoted 7 lines · Duy Nguyen
Quoted 4 lines · Johannes Schindelin

especially when we offer this as a better way
for 3rd-party applications to interact with Git (which I think will be the use case for this feature that will be _far_ more common than using it for debugging).

Quoted 5 lines

We may have conflicting goals. For me, first priority is the debug tool for Git developers. 3rd-party support is a stretch. I could move all this back to test-tool, then you can provide a 3rd-party API if you want. Or I'll withdraw this series and go back to my original plan.

FWIW, I am very much in favor of this series, and would have found something like this very useful many times in the past when I was digging into the index code. So I'd be more than happy to just have this as debug tool, rather than as 3rd-party API.

I'd also be fine with this living in test-tool, as long as it's somewhere in git.git where it's easily usable, I'd find it helpful.

Thanks for working on this!

#38 lore raw
In reply to Duy Nguyen

Hi Duy,

Quoted 12 lines · Duy Nguyen
Quoted 10 lines · Johannes Schindelin
Quoted 7 lines

- extension location is printed, in case you need to decode the extension by yourself (previously only the size is printed)
- all extensions are printed in the same order they appear in the file (previously eoie and ieot are printed first because that's how we parse)
- resolve undo extension is reorganized a bit to be easier to read
- tests added. Example json files are in t/t3011

It might actually make sense to optionally disable showing extensions.

Quoted 7 lines
Quoted 3 lines

You also forgot to mention that you explicitly disable handling <pathspec>, which I find a bit odd, personally, as that would probably come in real handy at times,

No. I mentioned the land of high level languages before. Filtering in any Python, Ruby, Scheme, JavaScript, Java is a piece of cake and much more flexible than pathspec.

I heard that type of argument before. I was working on the initial Windows port of Git, uh, of course I was working on a big refactoring of a big C++ application backed by a database. A colleague suggested that filtering could be done much better in C++, on the desktop, than in SQL. And so they changed the paradigm to "simplify" the SQL query, and instead dropped the unwanted data after it had hit the RAM of the client machine.

Turns out it was a bad idea. A _really_ bad idea. Because it required downloading 30MB of data for about several dozens computers in parallel, at the start of every shift.

This change was reverted in one big hurry, and the colleague was tasked to learn them some SQL.

Why am I telling you this story? Because you fall into the exact same trap as my colleague.

In this instance, it may not be so much network bandwidth, but it is still quite a suboptimal idea to render JSON for possibly tens of thousands of files, then parse the same JSON on the receiving side, the spend even more time to drop all but a dozen files.

And this is _even more_ relevant when you want to debug things.

In short: I am quite puzzled why this is even debated here. There is a reason, a good reason, why git ls-files accepts pathspecs. I would not want to ignore the lessons of history as willfully here.

Quoted 2 lines

Even with shell scripts, jq could do a much better job than pathspec. If you filter by pathspec, good luck trying that on extensions.

You keep harping on extensions, but the reality of the matter is that they are rarely interesting. I would even wager a bet that we will end up excluding them from the JSON output by default.

Most of the times when I had to decode the index file manually in the past, it was about the regular file entries.

There was *one* week in which I had to decode the untracked cache a bit, to the point where I patched the test helper locally to help me with that.

If my experience in debugging these things is any indicator, extensions do not matter even 10% of the non-extension data.

And that's not even taking into account the third-party software that could definitely benefit from having this JSON format as query result.

In my work as Git for Windows maintainer, I do hear about the needs of third-party software developers quite a bit, so I would claim that I know a bit about what they need, why the NUL-terminated format is not a good match, and how much a JSON-based API would help.

So while that is not your pet, it will be the most useful part of the outcome of your work.

Quoted 5 lines

It's the same reason why I will not provide a flexible way to disable extensions. I'm not starting a JSON API for Git. I provide an index file in JSON format. You do what you want with it. You have a format easy enough to import to native data structures of your favorite language.

I understand that you don't care.

Your patch series is just too good a start on something truly useful to pass up on the opportunity.

Quoted 10 lines
Quoted 4 lines

especially when we offer this as a better way for 3rd-party applications to interact with Git (which I think will be the use case for this feature that will be _far_ more common than using it for debugging).

We may have conflicting goals. For me, first priority is the debug tool for Git developers. 3rd-party support is a stretch. I could move all this back to test-tool, then you can provide a 3rd-party API if you want. Or I'll withdraw this series and go back to my original plan.

You don't need JSON if you want to debug things. That would be a lot of love lost, if debugging was your goal.

I guess I'll wait until your patch series hits next, and then try to find some time to work on that feature.

Ciao,
Johannes

author
#39 lore raw
Quoted 39 lines · Johannes Schindelin

Hi Duy,

Quoted 12 lines · Duy Nguyen
Quoted 10 lines · Johannes Schindelin
Quoted 7 lines

- extension location is printed, in case you need to decode the extension by yourself (previously only the size is printed)
- all extensions are printed in the same order they appear in the file (previously eoie and ieot are printed first because that's how we parse)
- resolve undo extension is reorganized a bit to be easier to read
- tests added. Example json files are in t/t3011

It might actually make sense to optionally disable showing extensions.

Quoted 7 lines
Quoted 3 lines

You also forgot to mention that you explicitly disable handling <pathspec>, which I find a bit odd, personally, as that would probably come in real handy at times,

No. I mentioned the land of high level languages before. Filtering in any Python, Ruby, Scheme, JavaScript, Java is a piece of cake and much more flexible than pathspec.

I heard that type of argument before. I was working on the initial Windows port of Git, uh, of course I was working on a big refactoring of a big C++ application backed by a database. A colleague suggested that filtering could be done much better in C++, on the desktop, than in SQL. And so they changed the paradigm to "simplify" the SQL query, and instead dropped the unwanted data after it had hit the RAM of the client machine.

Turns out it was a bad idea. A _really_ bad idea. Because it required downloading 30MB of data for about several dozens computers in parallel, at the start of every shift.

This change was reverted in one big hurry, and the colleague was tasked to learn them some SQL.

Quoted 7 lines

Why am I telling you this story? Because you fall into the exact same trap as my colleague.

In this instance, it may not be so much network bandwidth, but it is still quite a suboptimal idea to render JSON for possibly tens of thousands of files, then parse the same JSON on the receiving side, the spend even more time to drop all but a dozen files.

This was mentioned before [1]. Of course I don't work on giant index files, but I would assume the cost of parsing JSON (at least with a stream-based one, not loading the whole thing in core) is still cheaper. And you could still do it in iteration, saving every step until you have the reasonable small dataset to work on. The other side of the story is, are we sure parsing and executing pathspec is cheap? I'm not so sure, especially when pathspec code is not exactly optimized.

Consider the amount of code to support something like that. I'd rather wait until a real example come up and no good solution found without modify git.git, before actually supporting it.

[1] https://public-inbox.org/git/[email protected]/

Quoted 5 lines

And this is _even more_ relevant when you want to debug things.

In short: I am quite puzzled why this is even debated here. There is a reason, a good reason, why git ls-files accepts pathspecs. I would not want to ignore the lessons of history as willfully here.

I guess you and I have different ways of debugging things.

Quoted 7 lines
Quoted 2 lines

Even with shell scripts, jq could do a much better job than pathspec. If you filter by pathspec, good luck trying that on extensions.

You keep harping on extensions, but the reality of the matter is that they are rarely interesting. I would even wager a bet that we will end up excluding them from the JSON output by default.

Quoted 8 lines

Most of the times when I had to decode the index file manually in the past, it was about the regular file entries.

There was *one* week in which I had to decode the untracked cache a bit, to the point where I patched the test helper locally to help me with that.

If my experience in debugging these things is any indicator, extensions do not matter even 10% of the non-extension data.

Again our experiences differ. Mine is mostly about extensions, probably because I had to work on them more often. For normal entries "ls-files --debug" gives you 99% what's in the index file already.

Quoted 5 lines
Quoted 5 lines
Quoted 4 lines

especially when we offer this as a better way for 3rd-party applications to interact with Git (which I think will be the use case for this feature that will be _far_ more common than using it for debugging).

Quoted 8 lines
Quoted 5 lines

We may have conflicting goals. For me, first priority is the debug tool for Git developers. 3rd-party support is a stretch. I could move all this back to test-tool, then you can provide a 3rd-party API if you want. Or I'll withdraw this series and go back to my original plan.

You don't need JSON if you want to debug things. That would be a lot of love lost, if debugging was your goal.

No, I did think of some other line-based format before I ended up with JSON. I did not want to use it in the beginning.

The thing is, a giant table to cover all fields and entries in the main index is not as easy to navigate, or search even in 'less'. And the hierarchical structure of some extensions is hard to represent in good way (at least without writing lots of code). On top of that I still need some easy way to parse and post-process, like how much saving I could gain if I compressed stat data. And the final nail is json-writer.c is already there, much less work.

So JSON was the best option I found to meet all those points.

Signature · 1 line
Duy
#40 lore raw

Hi Duy,

Quoted 47 lines · Duy Nguyen
Quoted 37 lines · Johannes Schindelin
Quoted 12 lines · Duy Nguyen
Quoted 10 lines · Johannes Schindelin
Quoted 7 lines

- extension location is printed, in case you need to decode the extension by yourself (previously only the size is printed)
- all extensions are printed in the same order they appear in the file (previously eoie and ieot are printed first because that's how we parse)
- resolve undo extension is reorganized a bit to be easier to read
- tests added. Example json files are in t/t3011

It might actually make sense to optionally disable showing extensions.

Quoted 7 lines
Quoted 3 lines

You also forgot to mention that you explicitly disable handling <pathspec>, which I find a bit odd, personally, as that would probably come in real handy at times,

No. I mentioned the land of high level languages before. Filtering in any Python, Ruby, Scheme, JavaScript, Java is a piece of cake and much more flexible than pathspec.

I heard that type of argument before. I was working on the initial Windows port of Git, uh, of course I was working on a big refactoring of a big C++ application backed by a database. A colleague suggested that filtering could be done much better in C++, on the desktop, than in SQL. And so they changed the paradigm to "simplify" the SQL query, and instead dropped the unwanted data after it had hit the RAM of the client machine.

Turns out it was a bad idea. A _really_ bad idea. Because it required downloading 30MB of data for about several dozens computers in parallel, at the start of every shift.

This change was reverted in one big hurry, and the colleague was tasked to learn them some SQL.

Quoted 7 lines

Why am I telling you this story? Because you fall into the exact same trap as my colleague.

In this instance, it may not be so much network bandwidth, but it is still quite a suboptimal idea to render JSON for possibly tens of thousands of files, then parse the same JSON on the receiving side, the spend even more time to drop all but a dozen files.

Quoted 4 lines

This was mentioned before [1]. Of course I don't work on giant index files, but I would assume the cost of parsing JSON (at least with a stream-based one, not loading the whole thing in core) is still cheaper.

You may have heard that a few thousand of my colleagues are working on what they call the largest repository on this planet.

No, the cost of parsing JSON only to throw away the majority of the parsed information is not cheap. It is a clear sign of a design in want of being improved.

Quoted 4 lines

And you could still do it in iteration, saving every step until you have the reasonable small dataset to work on. The other side of the story is, are we sure parsing and executing pathspec is cheap? I'm not so sure, especially when pathspec code is not exactly optimized.

Let's not try to slap on workaround over workaround. Let's fix the root cause. (Being: don't filter at the wrong end.)

Quoted 1 line

Consider the amount of code to support something like that.

Given that I am pretty familiar with the pathspec machinery due to working with it in the git stash and git add -p built-ins, I have a very easy time considering the amount of code. It makes me smile how little code will be needed.

Quoted 2 lines

I'd rather wait until a real example come up and no good solution found without modify git.git, before actually supporting it.

Oh hey, there you go: Team Explorer. Visual Studio Code. Literally every single 3rd-party application that needs to deal with real-world loads. Every single one.

Quoted 7 lines
Quoted 5 lines

And this is _even more_ relevant when you want to debug things.

In short: I am quite puzzled why this is even debated here. There is a reason, a good reason, why git ls-files accepts pathspecs. I would not want to ignore the lessons of history as willfully here.

I guess you and I have different ways of debugging things.

Yep, I'm with Lincoln here: Give me six hours to debug a problem and I will spend the first four optimizing the edit-build-test cycle.

Quoted 13 lines
Quoted 13 lines
Quoted 2 lines

Even with shell scripts, jq could do a much better job than pathspec. If you filter by pathspec, good luck trying that on extensions.

You keep harping on extensions, but the reality of the matter is that they are rarely interesting. I would even wager a bet that we will end up excluding them from the JSON output by default.

Most of the times when I had to decode the index file manually in the past, it was about the regular file entries.

There was *one* week in which I had to decode the untracked cache a bit, to the point where I patched the test helper locally to help me with that.

Quoted 6 lines
Quoted 2 lines

If my experience in debugging these things is any indicator, extensions do not matter even 10% of the non-extension data.

Again our experiences differ. Mine is mostly about extensions, probably because I had to work on them more often. For normal entries "ls-files --debug" gives you 99% what's in the index file already.

Like the device. And the ctime. And the file size. And the uid/gid. Is that what you mean?

I don't know whether I missed a joke or not.

Quoted 5 lines
Quoted 2 lines

You don't need JSON if you want to debug things. That would be a lot of love lost, if debugging was your goal.

No, I did think of some other line-based format before I ended up with JSON. I did not want to use it in the beginning.

Then why bother.

Quoted 9 lines

The thing is, a giant table to cover all fields and entries in the main index is not as easy to navigate, or search even in 'less'. And the hierarchical structure of some extensions is hard to represent in good way (at least without writing lots of code). On top of that I still need some easy way to parse and post-process, like how much saving I could gain if I compressed stat data. And the final nail is json-writer.c is already there, much less work.

So JSON was the best option I found to meet all those points.

Well, as I said: you're obviously dead-set to optimize this for debugging your own problems. The beauty of open source is that it can be turned into something of wider use.

Ciao,
Johannes

#41 lore raw
Quoted 1 line · Johannes Schindelin

Hi Duy,

[snip]

Quoted 6 lines
Quoted 3 lines

Again our experiences differ. Mine is mostly about extensions, probably because I had to work on them more often. For normal entries "ls-files --debug" gives you 99% what's in the index file already.

Like the device. And the ctime. And the file size. And the uid/gid. Is that what you mean?

Hmm, well I think so:

  $ git ls-files --debug git.c git-compat-util.h 
  git-compat-util.h
    ctime: 1561457278:502638001
    mtime: 1561457278:502638001
    dev: 2049	ino: 262663
    uid: 1000	gid: 1000
    size: 35440	flags: 0
  git.c
    ctime: 1561457278:518646000
    mtime: 1561457278:518646000
    dev: 2049	ino: 263083
    uid: 1000	gid: 1000
    size: 26837	flags: 0
  $ 

I have occasionally added stuff to the '--debug' output while debugging something, but the above is usually sufficient for my uses. (Having said that, I have not had the need to debug extensions [yet!]).

ATB,
Ramsay Jones

#42 lore raw

Hi Ramsay,

Quoted 28 lines · Ramsay Jones
Quoted 1 line · Johannes Schindelin

Hi Duy,

[snip]

Quoted 6 lines
Quoted 3 lines

Again our experiences differ. Mine is mostly about extensions, probably because I had to work on them more often. For normal entries "ls-files --debug" gives you 99% what's in the index file already.

Like the device. And the ctime. And the file size. And the uid/gid. Is that what you mean?

Hmm, well I think so:

  $ git ls-files --debug git.c git-compat-util.h
  git-compat-util.h
    ctime: 1561457278:502638001
    mtime: 1561457278:502638001
    dev: 2049	ino: 262663
    uid: 1000	gid: 1000
    size: 35440	flags: 0
  git.c
    ctime: 1561457278:518646000
    mtime: 1561457278:518646000
    dev: 2049	ino: 263083
    uid: 1000	gid: 1000
    size: 26837	flags: 0
  $
Quoted 4 lines

I have occasionally added stuff to the '--debug' output while debugging something, but the above is usually sufficient for my uses. (Having said that, I have not had the need to debug extensions [yet!]).

Well, live and learn. So for debugging purposes, we already have a good facility and would not need JSON at all.

Good to know,
Dscho