Skip to content

Add 'ls-files --debug-json' to dump the index in json

Nguyễn Thái Ngọc Duy git 2 versions – In review

Get series with b4
b4 am [email protected]

Versions newest first · linked by author and title

  1. v2 Latest10 patches 32 repliesIn review
  2. v18 patches 27 repliesSuperseded

Patches in v2 No applies recorded yet

#Subject and filesSizeRepliesApplied
1/10ls-files: add --json to dump the indexDocumentation/git-ls-files.txt builtin/ls-files.c cache.h +5 more+265 -811 replies
2/10read-cache.c: dump common extension info in jsonread-cache.c+35 -140 replies
3/10cache-tree.c: dump "TREE" extension as jsoncache-tree.c cache-tree.h read-cache.c +2 more+58 -90 replies
4/10dir.c: dump "UNTR" extension as jsondir.c dir.h json-writer.h +3 more+107 -41 reply
5/10split-index.c: dump "link" extension as jsonjson-writer.c json-writer.h split-index.c +2 more+78 -111 replies
6/10fsmonitor.c: dump "FSMN" extension as jsonfsmonitor.c t/t3011-ls-files-json.sh t/t3011/fsmonitor+57 -10 replies
7/10resolve-undo.c: dump "REUC" extension as jsonread-cache.c resolve-undo.c resolve-undo.h +2 more+119 -30 replies
8/10read-cache.c: dump "EOIE" extension as jsonread-cache.c t/t3011-ls-files-json.sh t/t3011/eoie +1 more+121 -00 replies
9/10read-cache.c: dump "IEOT" extension as jsonread-cache.c t/t3011/eoie+48 -80 replies
10/10t3008: use the new SINGLE_CPU prereqt/t3008-ls-files-lazy-init-name-hash.sh+1 -70 replies

Reviews trailers per patch

No Reviewed-by, Acked-by or Tested-by trailers for v2 yet.

Read the v2 thread

Cover letter [PATCH v2 00/10] Add 'ls-files --debug-json' to dump the index in json

v2 main changes:

- --json is renamed --debug-json
- json field names now use '_' instead of '.' to be friendlier to some languages. I stick to underscore_name instead of camelCase because the former is closer to what we use
- extension location is printed, in case you need to decode the extension by yourself (previously only the size is printed)
- all extensions are printed in the same order they appear in the file (previously eoie and ieot are printed first because that's how we parse)
- resolve undo extension is reorganized a bit to be easier to read
- tests added. Example json files are in t/t3011

Interdiff

Documentation/git-ls-files.txt +5 -5
index 54011c8f65..fec5cb7170 100644
@@ -60,11 +60,6 @@ OPTIONS
 --stage::
 	Show staged contents' mode bits, object name and stage number in the output.
 
---json::
-	Dump the entire index content in JSON format. This is for
-	debugging purposes and the JSON structure may change from time
-	to time.
-
 --directory::
 	If a whole directory is classified as "other", show just its
 	name (with a trailing slash) and not its whole contents.
@@ -167,6 +162,11 @@ a space) at the start of each line:
 	possible for manual inspection; the exact format may change at
 	any time.
 
+--debug-json::
+	Dump the entire index content in JSON format. This is for
+	debugging purposes. The JSON structure is subject to change.
+	Note that the strings are not always valid UTF-8.
+
 --eol::
 	Show <eolinfo> and <eolattr> of files.
 	<eolinfo> is the file content identification used by Git when
builtin/ls-files.c +11 -5
index d00f6d3074..b60cd9ab28 100644
@@ -545,8 +545,6 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)
 			N_("show staged contents' object name in the output")),
 		OPT_BOOL('k', "killed", &show_killed,
 			N_("show files on the filesystem that need to be removed")),
-		OPT_BOOL(0, "json", &show_json,
-			N_("dump index content in json format")),
 		OPT_BIT(0, "directory", &dir.flags,
 			N_("show 'other' directories' names only"),
 			DIR_SHOW_OTHER_DIRECTORIES),
@@ -581,6 +579,8 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)
 			N_("pretend that paths removed since <tree-ish> are still present")),
 		OPT__ABBREV(&abbrev),
 		OPT_BOOL(0, "debug", &debug_mode, N_("show debugging data")),
+		OPT_BOOL(0, "debug-json", &show_json,
+			N_("dump index content in JSON format")),
 		OPT_END()
 	};
 
@@ -636,7 +636,7 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)
 		    "--error-unmatch");
 
 	parse_pathspec(&pathspec, 0,
-		       PATHSPEC_PREFER_CWD,
+		       show_json ? PATHSPEC_PREFER_FULL : PATHSPEC_PREFER_CWD,
 		       prefix, argv);
 
 	/*
@@ -668,8 +668,14 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)
 		show_cached = 1;
 	if (show_json && (show_stage || show_deleted || show_others ||
 			  show_unmerged || show_killed || show_modified ||
-			  show_cached || show_resolve_undo || with_tree))
-		die(_("--show-json cannot be used with other --show- options, or --with-tree"));
+			  show_cached || pathspec.nr))
+		die(_("--debug-json cannot be used with other file selection options"));
+	if (show_json && show_resolve_undo)
+		die(_("--debug-json cannot be used with %s"), "--resolve-undo");
+	if (show_json && with_tree)
+		die(_("--debug-json cannot be used with %s"), "--with-tree");
+	if (show_json && debug_mode)
+		die(_("--debug-json cannot be used with %s"), "--debug");
 
 	if (with_tree) {
 		/*
cache-tree.c +1 -6
index fc44016fe8..b6a233307e 100644
@@ -599,18 +599,13 @@ struct cache_tree *cache_tree_read(const char *buffer, unsigned long size,
 	struct cache_tree *ret;
 
 	if (jw) {
-		jw_object_inline_begin_object(jw, "cache-tree");
-		jw_object_intmax(jw, "ext-size", size);
 		jw_object_inline_begin_object(jw, "root");
 	}
 	if (buffer[0])
 		ret = NULL; /* not the whole tree */
 	else
 		ret = read_one(&buffer, &size, jw);
-	if (jw) {
-		jw_end(jw);	/* root */
-		jw_end(jw);	/* cache-tree */
-	}
+	jw_end_gently(jw);
 	return ret;
 }
 
dir.c +7 -6
index f389eee24a..8808577ea3 100644
@@ -2902,13 +2902,15 @@ struct untracked_cache *read_untracked_extension(const void *data,
 	uc->exclude_per_dir = xstrdup(exclude_per_dir);
 
 	if (jw) {
-		jw_object_inline_begin_object(jw, "untracked-cache");
-		jw_object_intmax(jw, "ext-size", sz);
 		jw_object_string(jw, "ident", ident);
-		jw_object_oid_stat(jw, "info/exclude", &uc->ss_info_exclude);
-		jw_object_oid_stat(jw, "excludes-file", &uc->ss_excludes_file);
+		jw_object_oid_stat(jw, "info_exclude", &uc->ss_info_exclude);
+		jw_object_oid_stat(jw, "excludes_file", &uc->ss_excludes_file);
 		jw_object_intmax(jw, "flags", uc->dir_flags);
-		jw_object_string(jw, "excludes-per-dir", uc->exclude_per_dir);
+		if (uc->dir_flags & DIR_SHOW_OTHER_DIRECTORIES)
+			jw_object_bool(jw, "show_other_directories", 1);
+		if (uc->dir_flags & DIR_HIDE_EMPTY_DIRECTORIES)
+			jw_object_bool(jw, "hide_empty_directories", 1);
+		jw_object_string(jw, "excludes_per_dir", uc->exclude_per_dir);
 	}
 
 	/* NUL after exclude_per_dir is covered by sizeof(*ouc) */
@@ -2968,7 +2970,6 @@ struct untracked_cache *read_untracked_extension(const void *data,
 		free_untracked_cache(uc);
 		uc = NULL;
 	}
-	jw_end_gently(jw);
 	return uc;
 }
 
fsmonitor.c +1 -4
index f6ba437255..5ed55ad176 100644
@@ -52,12 +52,9 @@ int read_fsmonitor_extension(struct index_state *istate, const void *data,
 	istate->fsmonitor_dirty = fsmonitor_dirty;
 
 	if (istate->jw) {
-		jw_object_inline_begin_object(istate->jw, "fsmonitor");
 		jw_object_intmax(istate->jw, "version", hdr_version);
-		jw_object_intmax(istate->jw, "last-update", istate->fsmonitor_last_update);
+		jw_object_intmax(istate->jw, "last_update", istate->fsmonitor_last_update);
 		jw_object_ewah(istate->jw, "dirty", fsmonitor_dirty);
-		jw_object_intmax(istate->jw, "ext-size", sz);
-		jw_end(istate->jw);
 	}
 	trace_printf_key(&trace_fsmonitor, "read fsmonitor extension successful");
 	return 0;
json-writer.c +15 -9
index 70403580ca..c0bd302e4e 100644
@@ -203,19 +203,25 @@ void jw_object_null(struct json_writer *jw, const char *key)
 	strbuf_addstr(&jw->json, "null");
 }
 
+void jw_object_filemode(struct json_writer *jw, const char *key, mode_t mode)
+{
+	object_common(jw, key);
+	strbuf_addf(&jw->json, "\"%06o\"", mode);
+}
+
 void jw_object_stat_data(struct json_writer *jw, const char *name,
 			 const struct stat_data *sd)
 {
 	jw_object_inline_begin_object(jw, name);
-	jw_object_intmax(jw, "st_ctime.sec", sd->sd_ctime.sec);
-	jw_object_intmax(jw, "st_ctime.nsec", sd->sd_ctime.nsec);
-	jw_object_intmax(jw, "st_mtime.sec", sd->sd_mtime.sec);
-	jw_object_intmax(jw, "st_mtime.nsec", sd->sd_mtime.nsec);
-	jw_object_intmax(jw, "st_dev", sd->sd_dev);
-	jw_object_intmax(jw, "st_ino", sd->sd_ino);
-	jw_object_intmax(jw, "st_uid", sd->sd_uid);
-	jw_object_intmax(jw, "st_gid", sd->sd_gid);
-	jw_object_intmax(jw, "st_size", sd->sd_size);
+	jw_object_intmax(jw, "ctime_sec", sd->sd_ctime.sec);
+	jw_object_intmax(jw, "ctime_nsec", sd->sd_ctime.nsec);
+	jw_object_intmax(jw, "mtime_sec", sd->sd_mtime.sec);
+	jw_object_intmax(jw, "mtime_nsec", sd->sd_mtime.nsec);
+	jw_object_intmax(jw, "device", sd->sd_dev);
+	jw_object_intmax(jw, "inode", sd->sd_ino);
+	jw_object_intmax(jw, "uid", sd->sd_uid);
+	jw_object_intmax(jw, "gid", sd->sd_gid);
+	jw_object_intmax(jw, "size", sd->sd_size);
 	jw_end(jw);
 }
 
json-writer.h +3 -0
index f778e019a2..07d841d52a 100644
@@ -42,8 +42,10 @@
  * of the given strings.
  */
 
+#include "git-compat-util.h"
 #include "strbuf.h"
 
+struct ewah_bitmap;
 struct stat_data;
 
 struct json_writer
@@ -83,6 +85,7 @@ void jw_object_true(struct json_writer *jw, const char *key);
 void jw_object_false(struct json_writer *jw, const char *key);
 void jw_object_bool(struct json_writer *jw, const char *key, int value);
 void jw_object_null(struct json_writer *jw, const char *key);
+void jw_object_filemode(struct json_writer *jw, const char *key, mode_t value);
 void jw_object_stat_data(struct json_writer *jw, const char *key,
 			 const struct stat_data *sd);
 void jw_object_ewah(struct json_writer *jw, const char *key,
read-cache.c +87 -78

317 lines not on this page · raw message

resolve-undo.c +25 -31

75 lines not on this page · raw message

split-index.c +2 -6
index d7b4420c92..41552bf771 100644
@@ -17,7 +17,6 @@ int read_link_extension(struct index_state *istate,
 {
 	const unsigned char *data = data_;
 	struct split_index *si;
-	unsigned long original_sz = sz;
 	int ret;
 
 	if (sz < the_hash_algo->rawsz)
@@ -42,12 +41,9 @@ int read_link_extension(struct index_state *istate,
 		return error("garbage at the end of link extension");
 done:
 	if (istate->jw) {
-		jw_object_inline_begin_object(istate->jw, "split-index");
 		jw_object_string(istate->jw, "oid", oid_to_hex(&si->base_oid));
-		jw_object_ewah(istate->jw, "delete-bitmap", si->delete_bitmap);
-		jw_object_ewah(istate->jw, "replace-bitmap", si->replace_bitmap);
-		jw_object_intmax(istate->jw, "ext-size", original_sz);
-		jw_end(istate->jw);
+		jw_object_ewah(istate->jw, "delete_bitmap", si->delete_bitmap);
+		jw_object_ewah(istate->jw, "replace_bitmap", si->replace_bitmap);
 	}
 	return 0;
 }
t/t3008-ls-files-lazy-init-name-hash.sh +1 -7
index 64f047332b..7f918c05f6 100755
@@ -4,15 +4,9 @@ test_description='Test the lazy init name hash with various folder structures'
 
 . ./test-lib.sh
 
-if test 1 -eq $($GIT_BUILD_DIR/t/helper/test-tool online-cpus)
-then
-	skip_all='skipping lazy-init tests, single cpu'
-	test_done
-fi
-
 LAZY_THREAD_COST=2000
 
-test_expect_success 'no buffer overflow in lazy_init_name_hash' '
+test_expect_success !SINGLE_CPU 'no buffer overflow in lazy_init_name_hash' '
 	(
 	    test_seq $LAZY_THREAD_COST | sed "s/^/a_/" &&
 	    echo b/b/b &&
t/t3011-ls-files-json.sh new file +106 -0

106 lines not on this page · raw message

t/t3011/basic new file +124 -0

124 lines not on this page · raw message

t/t3011/eoie new file +107 -0

107 lines not on this page · raw message

t/t3011/fsmonitor new file +38 -0
new file mode 100644
index 0000000000..17f2d4a664
@@ -0,0 +1,38 @@
+{
+  "version": 2,
+  "oid": <string>,
+  "mtime_sec": <number>,
+  "mtime_nsec": <number>,
+  "entries": [
+    {
+      "id": 0,
+      "name": "one",
+      "mode": "100644",
+      "flags": 0,
+      "oid": <string>,
+      "stat": {
+        "ctime_sec": <number>,
+        "ctime_nsec": <number>,
+        "mtime_sec": <number>,
+        "mtime_nsec": <number>,
+        "device": <number>,
+        "inode": <number>,
+        "uid": <number>,
+        "gid": <number>,
+        "size": 4
+      },
+      "file_offset": <number>
+    }
+  ],
+  "extensions": {
+    "FSMN": {
+      "file_offset": <number>,
+      "ext_size": <number>,
+      "version": 1,
+      "last_update": <number>,
+      "dirty": [
+        0
+      ]
+    }
+  }
+}
t/t3011/rerere new file +66 -0

66 lines not on this page · raw message

t/t3011/split-index new file +39 -0
new file mode 100644
index 0000000000..cdcc4ddded
@@ -0,0 +1,39 @@
+{
+  "version": 2,
+  "oid": <string>,
+  "mtime_sec": <number>,
+  "mtime_nsec": <number>,
+  "entries": [
+    {
+      "id": 0,
+      "name": "",
+      "mode": "100644",
+      "flags": 0,
+      "oid": <string>,
+      "stat": {
+        "ctime_sec": <number>,
+        "ctime_nsec": <number>,
+        "mtime_sec": <number>,
+        "mtime_nsec": <number>,
+        "device": <number>,
+        "inode": <number>,
+        "uid": <number>,
+        "gid": <number>,
+        "size": 4
+      },
+      "file_offset": <number>
+    }
+  ],
+  "extensions": {
+    "link": {
+      "file_offset": <number>,
+      "ext_size": <number>,
+      "oid": <string>,
+      "delete_bitmap": [
+      ],
+      "replace_bitmap": [
+        0
+      ]
+    }
+  }
+}
t/test-lib.sh +4 -0
index 4b346467df..9d5b273b40 100644
@@ -1611,3 +1611,7 @@ test_lazy_prereq REBASE_P '
 test_lazy_prereq FAIL_PREREQS '
 	test -n "$GIT_TEST_FAIL_PREREQS"
 '
+
+test_lazy_prereq SINGLE_CPU '
+	test "$(test-tool online-cpus)" -eq 1
+'

Nguyễn Thái Ngọc Duy (10):
ls-files: add --json to dump the index read-cache.c: dump common extension info in json cache-tree.c: dump "TREE" extension as json dir.c: dump "UNTR" extension as json split-index.c: dump "link" extension as json fsmonitor.c: dump "FSMN" extension as json resolve-undo.c: dump "REUC" extension as json read-cache.c: dump "EOIE" extension as json read-cache.c: dump "IEOT" extension as json
t3008: use the new SINGLE_CPU prereq

 Documentation/git-ls-files.txt          |   5 +
 builtin/ls-files.c                      |  38 ++++-
 cache-tree.c                            |  36 ++++-
 cache-tree.h                            |   5 +-
 cache.h                                 |   2 +
 dir.c                                   |  57 +++++++-
 dir.h                                   |   4 +-
 fsmonitor.c                             |   6 +
 json-writer.c                           |  36 +++++
 json-writer.h                           |  32 +++++
 read-cache.c                            | 178 ++++++++++++++++++++----
 resolve-undo.c                          |  30 +++-
 resolve-undo.h                          |   4 +-
 split-index.c                           |   9 +-
 t/t3008-ls-files-lazy-init-name-hash.sh |   8 +-
 t/t3011-ls-files-json.sh (new +x)       | 106 ++++++++++++++
 t/t3011/basic (new)                     | 124 +++++++++++++++++
 t/t3011/eoie (new)                      | 107 ++++++++++++++
 t/t3011/fsmonitor (new)                 |  38 +++++
 t/t3011/rerere (new)                    |  66 +++++++++
 t/t3011/split-index (new)               |  39 ++++++
 t/test-lib.sh                           |   4 +
 22 files changed, 884 insertions(+), 50 deletions(-)
 create mode 100755 t/t3011-ls-files-json.sh
 create mode 100644 t/t3011/basic
 create mode 100644 t/t3011/eoie
 create mode 100644 t/t3011/fsmonitor
 create mode 100644 t/t3011/rerere
 create mode 100644 t/t3011/split-index
Signature · 1 line
2.22.0.rc0.322.g2b0371e29a