Skip to contentSkip to conversation

ref consistency check infra setup

PATCH Superseded shejialuo 7 participants 282 messages May 30, 2024 – Oct 6, 2024 Documentation Page 1 of 3

lore
Share & tools
lore

shejialuo

author
#0 lore raw

cover ref consistency check infra setup

Hi,

This patch mainly focuses on two things:

1. Setup the infrastructure of the ref consistency check.
2. Implement ref name and content checks for files backend to verify the correctness about the infrastructure with corresponding unit test.

See https://github.com/shejialuo/git/pull/1 for CI.

Thanks,
Jialuo

shejialuo (2):
refs: setup ref consistency check infrastructure
refs: add name and content check for file backend

 builtin/fsck.c           |   5 ++
 refs.c                   |   5 ++
 refs.h                   |   5 ++
 refs/files-backend.c     | 119 ++++++++++++++++++++++++++++++++++-
 refs/packed-backend.c    |   7 +++
 refs/refs-internal.h     |   4 ++
 refs/reftable-backend.c  |   7 +++
 t/t0602-reffiles-fsck.sh | 132 +++++++++++++++++++++++++++++++++++++++
 8 files changed, 283 insertions(+), 1 deletion(-)
 create mode 100755 t/t0602-reffiles-fsck.sh
Signature · 1 line
2.45.1
author
#1 lore raw

PATCH 1/2 refs: setup ref consistency check infrastructure

Add a new interface fsck_refs_fn for the ref_storage_be. Implement dummy method for files and reftable backends.

---
 builtin/fsck.c          | 5 +++++
 refs.c                  | 5 +++++
 refs.h                  | 5 +++++
 refs/files-backend.c    | 9 ++++++++-
 refs/packed-backend.c   | 7 +++++++
 refs/refs-internal.h    | 4 ++++
 refs/reftable-backend.c | 7 +++++++
 7 files changed, 41 insertions(+), 1 deletion(-)
builtin/fsck.c +5 -0
index d13a226c2e..65a26e2d1b 100644
@@ -1065,6 +1065,11 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
 
 	check_connectivity();
 
+	if (refs_fsck(get_main_ref_store(the_repository))) {
+		error("ref database is corrupt");
+		errors_found |= ERROR_REFS;
+	}
+
 	if (the_repository->settings.core_commit_graph) {
 		struct child_process commit_graph_verify = CHILD_PROCESS_INIT;
 
refs.c +5 -0
index 8260c27cde..5ac61d551c 100644
@@ -316,6 +316,11 @@ int check_refname_format(const char *refname, int flags)
 	return check_or_sanitize_refname(refname, flags, NULL);
 }
 
+int refs_fsck(struct ref_store *refs)
+{
+	return refs->be->fsck(refs);
+}
+
 void sanitize_refname_component(const char *refname, struct strbuf *out)
 {
 	if (check_or_sanitize_refname(refname, REFNAME_ALLOW_ONELEVEL, out))
refs.h +5 -0
index 34568ee1fb..2799820c40 100644
@@ -544,6 +544,11 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat
  */
 int check_refname_format(const char *refname, int flags);
 
+/*
+  * Return 0 iff all refs in filesystem are consistent.
+*/
+int refs_fsck(struct ref_store *refs);
+
 /*
  * Apply the rules from check_refname_format, but mutate the result until it
  * is acceptable, and place the result in "out".
refs/files-backend.c +8 -1
index 5f3089d947..b6147c588b 100644
@@ -3299,6 +3299,11 @@ static int files_init_db(struct ref_store *ref_store,
 	return 0;
 }
 
+static int files_fsck(struct ref_store *ref_store)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_files = {
 	.name = "files",
 	.init = files_ref_store_create,
@@ -3322,5 +3327,7 @@ struct ref_storage_be refs_be_files = {
 	.reflog_exists = files_reflog_exists,
 	.create_reflog = files_create_reflog,
 	.delete_reflog = files_delete_reflog,
-	.reflog_expire = files_reflog_expire
+	.reflog_expire = files_reflog_expire,
+
+	.fsck = files_fsck,
 };
refs/packed-backend.c +7 -0
index a937e7dbfc..0617321634 100644
@@ -1704,6 +1704,11 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_s
 	return empty_ref_iterator_begin();
 }
 
+static int packed_fsck(struct ref_store *ref_store)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_packed = {
 	.name = "packed",
 	.init = packed_ref_store_create,
@@ -1728,4 +1733,6 @@ struct ref_storage_be refs_be_packed = {
 	.create_reflog = NULL,
 	.delete_reflog = NULL,
 	.reflog_expire = NULL,
+
+	.fsck = packed_fsck,
 };
refs/refs-internal.h +4 -0
index 53a6c5d842..ef697bf3bf 100644
@@ -675,6 +675,8 @@ typedef int read_raw_ref_fn(struct ref_store *ref_store, const char *refname,
 typedef int read_symbolic_ref_fn(struct ref_store *ref_store, const char *refname,
 				 struct strbuf *referent);
 
+typedef int fsck_fn(struct ref_store *ref_store);
+
 struct ref_storage_be {
 	const char *name;
 	ref_store_init_fn *init;
@@ -700,6 +702,8 @@ struct ref_storage_be {
 	create_reflog_fn *create_reflog;
 	delete_reflog_fn *delete_reflog;
 	reflog_expire_fn *reflog_expire;
+
+	fsck_fn *fsck;
 };
 
 extern struct ref_storage_be refs_be_files;
refs/reftable-backend.c +7 -0
index 1af86bbdec..f3f85cd2f0 100644
@@ -2167,6 +2167,11 @@ static int reftable_be_reflog_expire(struct ref_store *ref_store,
 	return ret;
 }
 
+static int reftable_be_fsck(struct ref_store *ref_store)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_reftable = {
 	.name = "reftable",
 	.init = reftable_be_init,
@@ -2191,4 +2196,6 @@ struct ref_storage_be refs_be_reftable = {
 	.create_reflog = reftable_be_create_reflog,
 	.delete_reflog = reftable_be_delete_reflog,
 	.reflog_expire = reftable_be_reflog_expire,
+
+	.fsck = reftable_be_fsck,
 };
Signature · 1 line
2.45.1
#2 lore raw

Re: [GSoC][PATCH 1/2] refs: setup ref consistency check infrastructure

Quoted 17 lines · shejialuo
refs/files-backend.c +5 -0
index 5f3089d947..b6147c588b 100644
@@ -3299,6 +3299,11 @@ static int files_init_db(struct ref_store *ref_store,
 	return 0;
 }
 
+static int files_fsck(struct ref_store *ref_store)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_files = {
 	.name = "files",
 	.init = files_ref_store_create,

refs/files-backend.c struct ref_storage_be refs_be_files = { · line 3332

Quoted 7 lines
refs/files-backend.c +3 -1
@@ -3322,5 +3327,7 @@ struct ref_storage_be refs_be_files = {
 	.reflog_exists = files_reflog_exists,
 	.create_reflog = files_create_reflog,
 	.delete_reflog = files_delete_reflog,
-	.reflog_expire = files_reflog_expire
+	.reflog_expire = files_reflog_expire,
+
+	.fsck = files_fsck,

What is the extra blank line doing there? It makes reader wonder why the .fsck member is somehow very special and different from others. Is there a valid reason to single it out (and no, "yes this is special because I invented it" does not count as a valid reason)?

The same comment applies to a few other places in this patch.

author
#3 lore raw

Re: [GSoC][PATCH 1/2] refs: setup ref consistency check infrastructure

Quoted 7 lines · Junio C Hamano

What is the extra blank line doing there? It makes reader wonder why the .fsck member is somehow very special and different from others. Is there a valid reason to single it out (and no, "yes this is special because I invented it" does not count as a valid reason)?

The same comment applies to a few other places in this patch.

The interfaces defined in the ref_storage_be are carefully structured in semantic. It is organized as the five parts at now:

1. The name and the initialization interfaces.
2. The ref transaction interfaces.
3. The ref internal interfaces (pack, rename and copy).
4. The ref filesystem interfaces.
5. The reflog related interfaces.

I firstly thought that we could just add a new function into the ref internal interfaces part with the name check_refs_fn. However, the ultimate goal is not only to achieve consistency check of refs but also consistency check of reflogs. So it's not suitable to name this interface to check_refs_fn.

In order to keep consistent with the git-fsck, we decide to name this interface to fsck_fn. This semantic cannot be grouped into any above five categories in my view. So I deliberately add a blank line to indicate its independence.

#4 lore raw

Re: [GSoC][PATCH 1/2] refs: setup ref consistency check infrastructure

In reply to shejialuo
Quoted 21 lines · shejialuo

Add a new interface fsck_refs_fn for the ref_storage_be. Implement dummy method for files and reftable backends.

---
 builtin/fsck.c          | 5 +++++
 refs.c                  | 5 +++++
 refs.h                  | 5 +++++
 refs/files-backend.c    | 9 ++++++++-
 refs/packed-backend.c   | 7 +++++++
 refs/refs-internal.h    | 4 ++++
 refs/reftable-backend.c | 7 +++++++
 7 files changed, 41 insertions(+), 1 deletion(-)
builtin/fsck.c +0 -0
index d13a226c2e..65a26e2d1b 100644

builtin/fsck.c int cmd_fsck(int argc, const char **argv, const char *prefix) · line 1071

Quoted 7 lines
builtin/fsck.c +4 -0
@@ -1065,6 +1065,11 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
 
 	check_connectivity();
 
+	if (refs_fsck(get_main_ref_store(the_repository))) {
+		error("ref database is corrupt");
+		errors_found |= ERROR_REFS;
+	}

One thing I was wondering is whether we want to make the infrastructure part of the new git-refs(1) command, which is about to land via my series that introduces ref backend migrations. We already make it a habit that we spawn new processes here, as you can see in two lines below for example. It would also make it trivial to execute ref checks standalone, by saying e.g. git refs verify.

Quoted 3 lines
builtin/fsck.c +0 -0
 	if (the_repository->settings.core_commit_graph) {
 		struct child_process commit_graph_verify = CHILD_PROCESS_INIT;
 

[snip]

refs.h int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat · line 550

Quoted 12 lines
refs.h +4 -0
index 34568ee1fb..2799820c40 100644
@@ -544,6 +544,11 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat
  */
 int check_refname_format(const char *refname, int flags);
 
+/*
+  * Return 0 iff all refs in filesystem are consistent.
+*/
+int refs_fsck(struct ref_store *refs);

We should probably also mention that errors will be written to stderr?

Also, I noticed that you are missing the implementation for the debug backend in "refs/debug.c".

Quoted 8 lines
refs.h +0 -0
 /*
  * Apply the rules from check_refname_format, but mutate the result until it
  * is acceptable, and place the result in "out".
refs/files-backend.c +0 -0
index 5f3089d947..b6147c588b 100644

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3305

Quoted 7 lines
refs/files-backend.c +4 -0
@@ -3299,6 +3299,11 @@ static int files_init_db(struct ref_store *ref_store,
 	return 0;
 }
 
+static int files_fsck(struct ref_store *ref_store)
+{
+	return 0;
+}

We can already wire up the call to the embedded packed-refs ref store here. We always want to call that from the files backend.

Patrick

#5 lore raw

Re: [GSoC][PATCH 1/2] refs: setup ref consistency check infrastructure

In reply to shejialuo

Hello,

shejialuo <[email protected]> writes:

[snip]

refs.h int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat · line 549

Quoted 11 lines
refs.h +3 -0
index 34568ee1fb..2799820c40 100644
@@ -544,6 +544,11 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat
  */
 int check_refname_format(const char *refname, int flags);
 
+/*
+  * Return 0 iff all refs in filesystem are consistent.
+*/

s/iff/if

The indentation seems to be wrong here. Also since we want to check refs and reflogs, it might be better to mention that the function checks the reference database for consistency.

Quoted 100 lines
refs.h +2 -0
+int refs_fsck(struct ref_store *refs);
+
 /*
  * Apply the rules from check_refname_format, but mutate the result until it
  * is acceptable, and place the result in "out".
refs/files-backend.c +8 -1
index 5f3089d947..b6147c588b 100644
@@ -3299,6 +3299,11 @@ static int files_init_db(struct ref_store *ref_store,
 	return 0;
 }
 
+static int files_fsck(struct ref_store *ref_store)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_files = {
 	.name = "files",
 	.init = files_ref_store_create,
@@ -3322,5 +3327,7 @@ struct ref_storage_be refs_be_files = {
 	.reflog_exists = files_reflog_exists,
 	.create_reflog = files_create_reflog,
 	.delete_reflog = files_delete_reflog,
-	.reflog_expire = files_reflog_expire
+	.reflog_expire = files_reflog_expire,
+
+	.fsck = files_fsck,
 };
refs/packed-backend.c +7 -0
index a937e7dbfc..0617321634 100644
@@ -1704,6 +1704,11 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_s
 	return empty_ref_iterator_begin();
 }
 
+static int packed_fsck(struct ref_store *ref_store)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_packed = {
 	.name = "packed",
 	.init = packed_ref_store_create,
@@ -1728,4 +1733,6 @@ struct ref_storage_be refs_be_packed = {
 	.create_reflog = NULL,
 	.delete_reflog = NULL,
 	.reflog_expire = NULL,
+
+	.fsck = packed_fsck,
 };
refs/refs-internal.h +4 -0
index 53a6c5d842..ef697bf3bf 100644
@@ -675,6 +675,8 @@ typedef int read_raw_ref_fn(struct ref_store *ref_store, const char *refname,
 typedef int read_symbolic_ref_fn(struct ref_store *ref_store, const char *refname,
 				 struct strbuf *referent);
 
+typedef int fsck_fn(struct ref_store *ref_store);
+
 struct ref_storage_be {
 	const char *name;
 	ref_store_init_fn *init;
@@ -700,6 +702,8 @@ struct ref_storage_be {
 	create_reflog_fn *create_reflog;
 	delete_reflog_fn *delete_reflog;
 	reflog_expire_fn *reflog_expire;
+
+	fsck_fn *fsck;
 };
 
 extern struct ref_storage_be refs_be_files;
refs/reftable-backend.c +7 -1
index 1af86bbdec..f3f85cd2f0 100644
@@ -2167,6 +2167,11 @@ static int reftable_be_reflog_expire(struct ref_store *ref_store,
 	return ret;
 }
 
+static int reftable_be_fsck(struct ref_store *ref_store)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_reftable = {
 	.name = "reftable",
 	.init = reftable_be_init,
@@ -2191,4 +2196,6 @@ struct ref_storage_be refs_be_reftable = {
 	.create_reflog = reftable_be_create_reflog,
 	.delete_reflog = reftable_be_delete_reflog,
 	.reflog_expire = reftable_be_reflog_expire,
+
+	.fsck = reftable_be_fsck,
 };
--

2.45.1

#6 lore raw

Re: [GSoC][PATCH 1/2] refs: setup ref consistency check infrastructure

Quoted 6 lines · Karthik Nayak

refs.h int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat · line 549

Quoted 3 lines · shejialuo
refs.h +3 -0
+/*
+  * Return 0 iff all refs in filesystem are consistent.
+*/

s/iff/if

I had spotted this, as well, but figured that the use of "iff" was intentional to mean "if and only if".

author
#7 lore raw

PATCH 2/2 refs: add name and content check for file backend

In reply to shejialuo

The existing git-fsck does not fully check bad format ref name such as standalone '@' or name ending with ".lock". And also git-fsck does not fully check ref content, the following contents will not be reported by the original git-fsck command.

1. "c71dba5654404b9b0d378a6cbff1b743b9d31a34 garbage": with trailing characters.
2. "c71dba5654404b9b0d378a6cbff1b743b9d31a34 ": with trailing empty
characters.
3. "C71DBA5654404b9b0d378a6cbff1b743b9d31A34": with uppercase hex.
4. "z71dba5654404b9b0d378a6cbff1b743b9d31a34": with not hex character.

Provide functionality to support such consistency checks for branch and tag refs and add a new unit test to verify this functionality.

---
 refs/files-backend.c     | 112 ++++++++++++++++++++++++++++++++-
 t/t0602-reffiles-fsck.sh | 132 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 243 insertions(+), 1 deletion(-)
 create mode 100755 t/t0602-reffiles-fsck.sh
refs/files-backend.c +111 -1
index b6147c588b..3c811f5c03 100644
@@ -3299,11 +3299,121 @@ static int files_init_db(struct ref_store *ref_store,
 	return 0;
 }
 
-static int files_fsck(struct ref_store *ref_store)
+typedef int (*files_fsck_refs_fn)(const char *refs_check_dir,
+				struct dir_iterator *iter);
+
+static int files_fsck_refs_name(const char *refs_check_dir,
+				struct dir_iterator *iter)
 {
+	if (check_refname_format(iter->basename, REFNAME_ALLOW_ONELEVEL)) {
+		error(_("%s/%s: invalid refname format"), refs_check_dir, iter->basename);
+		return -1;
+	}
+
 	return 0;
 }
 
+static int files_fsck_refs_content(const char *refs_check_dir,
+				struct dir_iterator *iter)
+{
+	struct strbuf ref_content = STRBUF_INIT;
+
+	if (strbuf_read_file(&ref_content, iter->path.buf, 0) < 0) {
+		error(_("%s/%s: unable to read the ref"), refs_check_dir, iter->basename);
+		goto clean;
+	}
+
+	/*
+	 * Case 1: check if the ref content length is valid and the last
+	 * character is a newline.
+	 */
+	if (ref_content.len != the_hash_algo->hexsz + 1 ||
+			ref_content.buf[ref_content.len - 1] != '\n') {
+		goto failure;
+	}
+
+	/*
+	 * Case 2: the content should be range of [0-9a-f].
+	 */
+	for (size_t i = 0; i < the_hash_algo->hexsz; i++) {
+		if (!isdigit(ref_content.buf[i]) &&
+				(ref_content.buf[i] < 'a' || ref_content.buf[i] > 'f')) {
+			goto failure;
+		}
+	}
+
+	strbuf_release(&ref_content);
+	return 0;
+
+failure:
+	error(_("%s/%s: invalid ref content"), refs_check_dir, iter->basename);
+
+clean:
+	strbuf_release(&ref_content);
+	return -1;
+}
+
+static int files_fsck_refs(struct ref_store *ref_store,
+				const char* refs_check_dir,
+				files_fsck_refs_fn *fsck_refs_fns)
+{
+	struct dir_iterator *iter;
+	struct strbuf sb = STRBUF_INIT;
+	int ret = 0;
+	int iter_status;
+
+	strbuf_addf(&sb, "%s/%s", ref_store->gitdir, refs_check_dir);
+
+	iter = dir_iterator_begin(sb.buf, 0);
+
+	/*
+	 * The current implementation does not care about the worktree, the worktree
+	 * may have no refs/heads or refs/tags directory. Simply return 0 now.
+	*/
+	if (!iter) {
+		return 0;
+	}
+
+	while ((iter_status = dir_iterator_advance(iter)) == ITER_OK) {
+		if (S_ISDIR(iter->st.st_mode)) {
+			continue;
+		} else if (S_ISREG(iter->st.st_mode)) {
+			for (files_fsck_refs_fn *fsck_refs_fn = fsck_refs_fns;
+					*fsck_refs_fn; fsck_refs_fn++) {
+				ret |= (*fsck_refs_fn)(refs_check_dir, iter);
+			}
+		} else {
+			error(_("unexpected file type for '%s'"), iter->basename);
+			ret = -1;
+		}
+	}
+
+	if (iter_status != ITER_DONE) {
+		ret = -1;
+		error(_("failed to iterate over '%s'"), sb.buf);
+	}
+
+	strbuf_release(&sb);
+
+	return ret;
+}
+
+static int files_fsck(struct ref_store *ref_store)
+{
+	int ret = 0;
+
+	files_fsck_refs_fn fsck_refs_fns[] = {
+		files_fsck_refs_name,
+		files_fsck_refs_content,
+		NULL
+	};
+
+	ret = files_fsck_refs(ref_store, "refs/heads",fsck_refs_fns)
+	    | files_fsck_refs(ref_store, "refs/tags", fsck_refs_fns);
+
+	return ret;
+}
+
 struct ref_storage_be refs_be_files = {
 	.name = "files",
 	.init = files_ref_store_create,
t/t0602-reffiles-fsck.sh new file +132 -0

132 lines not on this page · raw message

Signature · 1 line
2.45.1
#8 lore raw

Re: [GSoC][PATCH 2/2] refs: add name and content check for file backend

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3319

Quoted 4 lines · shejialuo
refs/files-backend.c +4 -0
+static int files_fsck_refs_content(const char *refs_check_dir,
+				struct dir_iterator *iter)
+{
+	struct strbuf ref_content = STRBUF_INIT;

The caller makes sure that this gets called only on a regular file, so ...

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3321

Quoted 1 line
refs/files-backend.c +1 -0
+	if (strbuf_read_file(&ref_content, iter->path.buf, 0) < 0) {

... the use of strbuf_read_file() is fine (i.e. we do not have to worry about the iter->path to be pointing at a symbolic link) here.

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3331

Quoted 10 lines
refs/files-backend.c +10 -0
+		error(_("%s/%s: unable to read the ref"), refs_check_dir, iter->basename);
+		goto clean;
+	}
+
+	/*
+	 * Case 1: check if the ref content length is valid and the last
+	 * character is a newline.
+	 */
+	if (ref_content.len != the_hash_algo->hexsz + 1 ||
+			ref_content.buf[ref_content.len - 1] != '\n') {

Funny indentation.

	if (ref_content.len != the_hash_algo->hexsz + 1 ||
	    ref_content.buf[ref_content.len - 1] != '\n') {

Also, we do not want {braces} around a single statement block.

In any case, the two checks are good ONLY for regular refs and not for symbolic refs. The users are free to create symbolic refs next to their branches, e.g. here is a way to say, "among the maintenance tracks maint-2.30, maint-2.31, ... maint-2.44, maint-2.45, what I consider the primary maintenance track is currently maint-2.45":

    $ git symbolic-ref refs/heads/maint
    refs/heads/maint-2.45

and if your directory walk encounters such a symbolic ref, your strbuf_read_file() will read something like:

    $ cat .git/refs/heads/maint
    ref: refs/heads/maint-2.45

Also, the caller also needs to be prepared to find a real symbolic link that is used as a symbolic ref, but for them you'd need to do a readlink() and the expected contents would not have "ref: " prefix, so the code to implement actual check would have to be different. The way how refs/files-backend.c:read_ref_internal() handles S_ISLNK would be illuminating.

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3341

Quoted 9 lines
refs/files-backend.c +9 -0
+		goto failure;
+	}
+	/*
+	 * Case 2: the content should be range of [0-9a-f].
+	 */
+	for (size_t i = 0; i < the_hash_algo->hexsz; i++) {
+		if (!isdigit(ref_content.buf[i]) &&
+				(ref_content.buf[i] < 'a' || ref_content.buf[i] > 'f')) {
+			goto failure;

I do not think it is a good idea to suddenly redefine what a valid way to write object names in a loose ref file after ~20 years. Given the popularity of Git, I would not be surprised at all if a third-party tool or two have been writing their own refs with uppercase hexadecimal and we have been happily using them as everybody expects. It is a good idea to be strict when you are a producer, and we do write object names always in lowercase hex, but we are lenient when consuming what is possibly written by others. As long as the hexadecimal string names an existing object (otherwise we have a dangling ref which would be another kind of error, I presume), it should be at most FSCK_WARN when it does not look like what _we_ wrote (e.g., if it uses uppercase hex, or if it has extra bytes after the 40-hex (or 64-hex) other than the final LF). If it is shorter, of course that is an outright FSCK_ERROR.

How well does this interact with the fsck error levels (aka fsck_msg_type), by the way? It should be made to work well if the current design does not.

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3349

Quoted 8 lines
refs/files-backend.c +8 -0
+		}
+	}
+
+	strbuf_release(&ref_content);
+	return 0;
+
+failure:
+	error(_("%s/%s: invalid ref content"), refs_check_dir, iter->basename);

In addition, when we honor fsck error levels, such an unconditional call to "error" may become an issue. If the user says a certain kind of anomaly is tolerated by setting a specific finding to FSCK_IGNORE, we should be silent about our finding.

Quoted 22 lines
refs/files-backend.c +22 -0
+clean:
+	strbuf_release(&ref_content);
+	return -1;
+}
+
+static int files_fsck_refs(struct ref_store *ref_store,
+				const char* refs_check_dir,
+				files_fsck_refs_fn *fsck_refs_fns)
+{
+	struct dir_iterator *iter;
+	struct strbuf sb = STRBUF_INIT;
+	int ret = 0;
+	int iter_status;
+
+	strbuf_addf(&sb, "%s/%s", ref_store->gitdir, refs_check_dir);
+
+	iter = dir_iterator_begin(sb.buf, 0);
+
+	/*
+	 * The current implementation does not care about the worktree, the worktree
+	 * may have no refs/heads or refs/tags directory. Simply return 0 now.
+	*/

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3382

Quoted 10 lines
refs/files-backend.c +10 -0
+	if (!iter) {
+		return 0;
+	}
+
+	while ((iter_status = dir_iterator_advance(iter)) == ITER_OK) {
+		if (S_ISDIR(iter->st.st_mode)) {
+			continue;
+		} else if (S_ISREG(iter->st.st_mode)) {
+			for (files_fsck_refs_fn *fsck_refs_fn = fsck_refs_fns;
+					*fsck_refs_fn; fsck_refs_fn++) {

Funny indentation (again---the patch 2/2 has funny "two extra HT" indentation style all over the place). Write it like so:

			for (files_fsck_refs_fn *fn = fsck_refs_fns;
			     fn;
                             fn++)

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3383

Quoted 1 line
refs/files-backend.c +1 -0
+				ret |= (*fsck_refs_fn)(refs_check_dir, iter);

A pointer to a function does not need to be dereferenced explicitly like so. Also, writing

				if (fn(refs_check_dir, iter))
					ret = -1;

would be more consistent with the rest of the code around here, which does not OR int ret but explicitly set it to -1 when any helper function detects an error.

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3387

Quoted 4 lines
refs/files-backend.c +4 -0
+			}
+		} else {
+			error(_("unexpected file type for '%s'"), iter->basename);
+			ret = -1;

This is wrong. A symbolic link is a valid symbolic ref, even though we no longer create such a symbolic ref by default.

Quoted 14 lines
refs/files-backend.c +14 -0
+		}
+	}
+
+	if (iter_status != ITER_DONE) {
+		ret = -1;
+		error(_("failed to iterate over '%s'"), sb.buf);
+	}
+
+	strbuf_release(&sb);
+
+	return ret;
+}
+
+static int files_fsck(struct ref_store *ref_store)

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3411

Quoted 10 lines
refs/files-backend.c +10 -0
+{
+	int ret = 0;
+
+	files_fsck_refs_fn fsck_refs_fns[] = {
+		files_fsck_refs_name,
+		files_fsck_refs_content,
+		NULL
+	};
+
+	ret = files_fsck_refs(ref_store, "refs/heads",fsck_refs_fns)

Missing SP after a comma.

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3412

Quoted 1 line
refs/files-backend.c +1 -0
+	    | files_fsck_refs(ref_store, "refs/tags", fsck_refs_fns);

Why only these two hierarchies? Shouldn't it also be checking the remote tracking branches and notes?

Quoted 6 lines
refs/files-backend.c +3 -0
+	return ret;
+}
+
 struct ref_storage_be refs_be_files = {
 	.name = "files",
 	.init = files_ref_store_create,
author
#9 lore raw

Re: [GSoC][PATCH 2/2] refs: add name and content check for file backend

Quoted 28 lines · Junio C Hamano

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3319

Quoted 4 lines · shejialuo
refs/files-backend.c +4 -0
+static int files_fsck_refs_content(const char *refs_check_dir,
+				struct dir_iterator *iter)
+{
+	struct strbuf ref_content = STRBUF_INIT;

The caller makes sure that this gets called only on a regular file, so ...

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3321

Quoted 1 line
refs/files-backend.c +1 -0
+	if (strbuf_read_file(&ref_content, iter->path.buf, 0) < 0) {

... the use of strbuf_read_file() is fine (i.e. we do not have to worry about the iter->path to be pointing at a symbolic link) here.

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3331

Quoted 10 lines
refs/files-backend.c +10 -0
+		error(_("%s/%s: unable to read the ref"), refs_check_dir, iter->basename);
+		goto clean;
+	}
+
+	/*
+	 * Case 1: check if the ref content length is valid and the last
+	 * character is a newline.
+	 */
+	if (ref_content.len != the_hash_algo->hexsz + 1 ||
+			ref_content.buf[ref_content.len - 1] != '\n') {

Funny indentation.

Quoted 5 lines
	if (ref_content.len != the_hash_algo->hexsz + 1 ||
	    ref_content.buf[ref_content.len - 1] != '\n') {

Also, we do not want {braces} around a single statement block.

I will clean the code. I am used to applying one extra HT.

Quoted 6 lines

In any case, the two checks are good ONLY for regular refs and not for symbolic refs. The users are free to create symbolic refs next to their branches, e.g. here is a way to say, "among the maintenance tracks maint-2.30, maint-2.31, ... maint-2.44, maint-2.45, what I consider the primary maintenance track is currently maint-2.45":

Quoted 9 lines
    $ git symbolic-ref refs/heads/maint
    refs/heads/maint-2.45

and if your directory walk encounters such a symbolic ref, your strbuf_read_file() will read something like:

    $ cat .git/refs/heads/maint
    ref: refs/heads/maint-2.45

Yes, currently I intentionally ignore the symbolic ref. My original idea was to make this patch as small as possible to set up the infrastructure of the ref consistence check.

So this is why I only choose "refs/heads" and "refs/tags" at current. The "refs/remote" will contain "HEAD" which is a symbolic ref. So I ignore "refs/remote".

I just thought git would ONLY have one symbolic ref "HEAD". I will add more patches in the next version to support the symbolic ref checks.

Quoted 7 lines

Also, the caller also needs to be prepared to find a real symbolic link that is used as a symbolic ref, but for them you'd need to do a readlink() and the expected contents would not have "ref: " prefix, so the code to implement actual check would have to be different. The way how refs/files-backend.c:read_ref_internal() handles S_ISLNK would be illuminating.

I will handle this in the next version. However, I am a little curious about the reason why people use a real symbolic in git. git has provided the symbolic ref which should be more consistent and generic against the OS.

Quoted 16 lines

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3341

Quoted 9 lines
refs/files-backend.c +9 -0
+		goto failure;
+	}
+	/*
+	 * Case 2: the content should be range of [0-9a-f].
+	 */
+	for (size_t i = 0; i < the_hash_algo->hexsz; i++) {
+		if (!isdigit(ref_content.buf[i]) &&
+				(ref_content.buf[i] < 'a' || ref_content.buf[i] > 'f')) {
+			goto failure;

I do not think it is a good idea to suddenly redefine what a valid way to write object names in a loose ref file after ~20 years. Given the popularity of Git, I would not be surprised at all if a third-party tool or two have been writing their own refs with uppercase hexadecimal and we have been happily using them as everybody expects. It is a good idea to be strict when you are a

Quoted 9 lines

producer, and we do write object names always in lowercase hex, but we are lenient when consuming what is possibly written by others. As long as the hexadecimal string names an existing object (otherwise we have a dangling ref which would be another kind of error, I presume), it should be at most FSCK_WARN when it does not look like what _we_ wrote (e.g., if it uses uppercase hex, or if it has extra bytes after the 40-hex (or 64-hex) other than the final LF). If it is shorter, of course that is an outright FSCK_ERROR.

Yes, actually I have done some experiments using git-fsck. I am really surprised by the fact that the few bytes will cause FSCK_ERROR. But the situation with extra bytes are OK. I don't know this design is intentional.

Quoted 4 lines

How well does this interact with the fsck error levels (aka fsck_msg_type), by the way? It should be made to work well if the current design does not.

Unfortunately, I totally ignored this. I will try to implement the fsck error levels in the next version.

Quoted 5 lines
Quoted 5 lines
refs/files-backend.c +5 -0
+		}
+	}
+
+	strbuf_release(&ref_content);
+	return 0;
Quoted 9 lines

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3349

Quoted 3 lines
refs/files-backend.c +3 -0
+
+failure:
+	error(_("%s/%s: invalid ref content"), refs_check_dir, iter->basename);

In addition, when we honor fsck error levels, such an unconditional call to "error" may become an issue. If the user says a certain kind of anomaly is tolerated by setting a specific finding to FSCK_IGNORE, we should be silent about our finding.

Yes, I will add the code to support this.

Quoted 45 lines
Quoted 22 lines
refs/files-backend.c +22 -0
+clean:
+	strbuf_release(&ref_content);
+	return -1;
+}
+
+static int files_fsck_refs(struct ref_store *ref_store,
+				const char* refs_check_dir,
+				files_fsck_refs_fn *fsck_refs_fns)
+{
+	struct dir_iterator *iter;
+	struct strbuf sb = STRBUF_INIT;
+	int ret = 0;
+	int iter_status;
+
+	strbuf_addf(&sb, "%s/%s", ref_store->gitdir, refs_check_dir);
+
+	iter = dir_iterator_begin(sb.buf, 0);
+
+	/*
+	 * The current implementation does not care about the worktree, the worktree
+	 * may have no refs/heads or refs/tags directory. Simply return 0 now.
+	*/

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3382

Quoted 10 lines
refs/files-backend.c +10 -0
+	if (!iter) {
+		return 0;
+	}
+
+	while ((iter_status = dir_iterator_advance(iter)) == ITER_OK) {
+		if (S_ISDIR(iter->st.st_mode)) {
+			continue;
+		} else if (S_ISREG(iter->st.st_mode)) {
+			for (files_fsck_refs_fn *fsck_refs_fn = fsck_refs_fns;
+					*fsck_refs_fn; fsck_refs_fn++) {

Funny indentation (again---the patch 2/2 has funny "two extra HT" indentation style all over the place). Write it like so:

			for (files_fsck_refs_fn *fn = fsck_refs_fns;
			     fn;
                             fn++)

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3383

Quoted 1 line
refs/files-backend.c +1 -0
+				ret |= (*fsck_refs_fn)(refs_check_dir, iter);

A pointer to a function does not need to be dereferenced explicitly like so. Also, writing

Quoted 7 lines
				if (fn(refs_check_dir, iter))
					ret = -1;

would be more consistent with the rest of the code around here, which does not OR int ret but explicitly set it to -1 when any helper function detects an error.

I will handle this in the next version.

Quoted 8 lines

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3387

Quoted 4 lines
refs/files-backend.c +4 -0
+			}
+		} else {
+			error(_("unexpected file type for '%s'"), iter->basename);
+			ret = -1;

This is wrong. A symbolic link is a valid symbolic ref, even though we no longer create such a symbolic ref by default.

As above said, I have thought git should not handle this situation. I will implement support for real symbolic file.

Quoted 18 lines
Quoted 18 lines
refs/files-backend.c +18 -0
+		}
+	}
+
+	if (iter_status != ITER_DONE) {
+		ret = -1;
+		error(_("failed to iterate over '%s'"), sb.buf);
+	}
+
+	strbuf_release(&sb);
+
+	return ret;
+}
+
+static int files_fsck(struct ref_store *ref_store)
+{
+	int ret = 0;
+
+	files_fsck_refs_fn fsck_refs_fns[] = {
Quoted 9 lines

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3411

Quoted 6 lines
refs/files-backend.c +6 -0
+		files_fsck_refs_name,
+		files_fsck_refs_content,
+		NULL
+	};
+
+	ret = files_fsck_refs(ref_store, "refs/heads",fsck_refs_fns)

Missing SP after a comma.

I will fix it in the next version.

Quoted 5 lines

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3412

Quoted 1 line
refs/files-backend.c +1 -0
+	    | files_fsck_refs(ref_store, "refs/tags", fsck_refs_fns);

Why only these two hierarchies? Shouldn't it also be checking the remote tracking branches and notes?

I have explained above, because I intentionally ignored the symbolic ref in this patch, so I don't handle "refs/remote" and notes. I guess I will handle these situations in the next patch.

Quoted 6 lines

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3419

Quoted 6 lines
refs/files-backend.c +3 -0
+	return ret;
+}
+
 struct ref_storage_be refs_be_files = {
 	.name = "files",
 	.init = files_ref_store_create,

Thanks for your dedicated reviews. I have a more clean direction to setup the consistence check for file backend.

Thanks,
Jialuo

#10 lore raw

Re: [GSoC][PATCH 2/2] refs: add name and content check for file backend

In reply to Junio C Hamano
Quoted 6 lines · Junio C Hamano

Review

Quoted 2 lines · shejialuo
(quoted diff) +2 -0
+                     error(_("unexpected file type for '%s'"), iter->basename);
+                     ret = -1;

This is wrong. A symbolic link is a valid symbolic ref, even though we no longer create such a symbolic ref by default.

Very minor additional note: error() unconditionally returns -1 in order to support this pattern:

    ret = error(_("..."));
#11 lore raw

Re: [GSoC][PATCH 2/2] refs: add name and content check for file backend

In reply to shejialuo
Quoted 12 lines · shejialuo

The existing git-fsck does not fully check bad format ref name such as standalone '@' or name ending with ".lock". And also git-fsck does not fully check ref content, the following contents will not be reported by the original git-fsck command.

1. "c71dba5654404b9b0d378a6cbff1b743b9d31a34 garbage": with trailing characters.
2. "c71dba5654404b9b0d378a6cbff1b743b9d31a34 ": with trailing empty
characters.
3. "C71DBA5654404b9b0d378a6cbff1b743b9d31A34": with uppercase hex.
4. "z71dba5654404b9b0d378a6cbff1b743b9d31a34": with not hex character.

Quoted 2 lines

Provide functionality to support such consistency checks for branch and tag refs and add a new unit test to verify this functionality.

I would recommend to only introduce one check per commit. This commit introduces multiple different checks to the files backend, which makes it harder to review.

[snip]

Quoted 28 lines
refs/files-backend.c +28 -0
+static int files_fsck_refs_content(const char *refs_check_dir,
+				struct dir_iterator *iter)
+{
+	struct strbuf ref_content = STRBUF_INIT;
+
+	if (strbuf_read_file(&ref_content, iter->path.buf, 0) < 0) {
+		error(_("%s/%s: unable to read the ref"), refs_check_dir, iter->basename);
+		goto clean;
+	}
+
+	/*
+	 * Case 1: check if the ref content length is valid and the last
+	 * character is a newline.
+	 */
+	if (ref_content.len != the_hash_algo->hexsz + 1 ||
+			ref_content.buf[ref_content.len - 1] != '\n') {
+		goto failure;
+	}
+
+	/*
+	 * Case 2: the content should be range of [0-9a-f].
+	 */
+	for (size_t i = 0; i < the_hash_algo->hexsz; i++) {
+		if (!isdigit(ref_content.buf[i]) &&
+				(ref_content.buf[i] < 'a' || ref_content.buf[i] > 'f')) {
+			goto failure;
+		}
+	}

refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3353

Quoted 10 lines
refs/files-backend.c +10 -0
+
+	strbuf_release(&ref_content);
+	return 0;
+
+failure:
+	error(_("%s/%s: invalid ref content"), refs_check_dir, iter->basename);
+
+clean:
+	strbuf_release(&ref_content);
+	return -1;

Can we reuse parse_loose_ref_contents() to get rid of much of the duplication here? That function would need to learn to indicate to the caller that there is trailing data, but other than that it's already as strict as we want it to be.

Other than that, Junio has already added quite a bunch of comments, so I'll refrain from repeating them here.

Thanks!

Patrick

author
#12 lore raw

v2 cover ref consistency check infra setup

In reply to shejialuo

This patch sets up the ref consistency check infrastructure. We need to integrate the fsck error messages. However, the git-fsck(1) is highly coupled with the object database. And it uses git_fsck_config to initialize the fsck_options->fsck_msg_type. For my implementation, I do not add some new files into fsck_options due to the following reasons:

1. The fileds in fsck_options are all related to objects. We only can reuse the fsck_msg_type and strict member, adding some ref-related fileds into fsck_options will break the semantics.
2. We may laterc hange fsck_options and fsck_objs_options.

The code for integrating fsck error messages for refs may be a liitle redudant now, but if we change the fsck_options, it will be way complicated.

After integrating the fsck error messages, setup the consistency check infrastructure like patch v1. And introduce a new subcommand for builtin git-refs and in git-fsck(1), use a child process to call this subcommand.

At last, like patch v1, implement the consistency check for files backend and corresponding unit tests.

Because I have added more commits compare to v1. So I provide the innerdiff between the v1 and v2.

Thanks,
Jialuo

shejialuo (7):
fsck: add refs check interfaces to interface with fsck error levels
refs: set up ref consistency check infrastructure builtin/refs: add verify subcommand
builtin/fsck: add git-refs verify child process
files-backend: add unified interface for refs scanning
fsck: add ref name check for files backend
fsck: add ref content check for files backend

 Documentation/fsck-msgids.txt |  12 ++
 Documentation/git-refs.txt    |  11 ++
 builtin/fsck.c                |  17 +++
 builtin/refs.c                |  45 +++++++
 fsck.c                        |  69 ++++++++++
 fsck.h                        |  43 ++++++-
 refs.c                        |   7 +-
 refs.h                        |   8 ++
 refs/debug.c                  |   9 ++
 refs/files-backend.c          | 234 +++++++++++++++++++++++++++++++++-
 refs/packed-backend.c         |   8 ++
 refs/refs-internal.h          |  11 +-
 refs/reftable-backend.c       |   8 ++
 t/t0602-reffiles-fsck.sh      | 211 ++++++++++++++++++++++++++++++
 14 files changed, 685 insertions(+), 8 deletions(-)
 create mode 100755 t/t0602-reffiles-fsck.sh

Interdiff between patch v1 and patch v2:

Documentation/fsck-msgids.txt +12 -1
index 5edc06c658..69f86c5345 100644
@@ -19,6 +19,12 @@
 `badParentSha1`::
 	(ERROR) A commit object has a bad parent sha1.
 
+`badRefName`::
+	(ERROR) A ref has a bad name.
+
+`badRefSha`::
+	(ERROR) A ref has a bad sha.
+
 `badTagName`::
 	(INFO) A tag has an invalid format.
 
@@ -34,6 +40,9 @@
 `badType`::
 	(ERROR) Found an invalid object type.
 
+`danglingSymref`::
+	(WARN) Found a dangling symref.
+
 `duplicateEntries`::
 	(ERROR) A tree contains duplicate file entries.
 
@@ -176,6 +185,9 @@
 `symlinkTargetMissing`::
 	(ERROR) Unable to read symbolic link target's blob.
 
+`trailingRefContent`::
+	(WARN) A ref content has trailing contents.
+
 `treeNotSorted`::
 	(ERROR) A tree is not properly sorted.
-
Documentation/git-refs.txt +11 -0
index 5b99e04385..f9d36ea19d 100644
@@ -10,6 +10,7 @@ SYNOPSIS
 --------
 [verse]
 'git refs migrate' --ref-format=<format> [--dry-run]
+'git refs verify' [--strict] [--verbose]
 
 DESCRIPTION
 -----------
@@ -22,6 +23,9 @@ COMMANDS
 migrate::
 	Migrate ref store between different formats.
 
+verify::
+	Verify reference database consistency.
+
 OPTIONS
 -------
 
@@ -39,6 +43,13 @@ include::ref-storage-format.txt[]
 	can be used to double check that the migration works as expected before
 	performing the actual migration.
 
+--strict::
+	Enable more strict checking, every WARN severity for the `Fsck Messages`
+	be seen as ERROR.
+
+--verbose::
+	When verifying the reference database consistency, be chatty.
+
 KNOWN LIMITATIONS
 -----------------
 
builtin/fsck.c +16 -4
index 65a26e2d1b..10d73f534f 100644
@@ -896,6 +896,21 @@ static int check_pack_rev_indexes(struct repository *r, int show_progress)
 	return res;
 }
 
+static void fsck_refs(void)
+{
+	struct child_process refs_verify = CHILD_PROCESS_INIT;
+	child_process_init(&refs_verify);
+	refs_verify.git_cmd = 1;
+	strvec_pushl(&refs_verify.args, "refs", "verify", NULL);
+	if (verbose)
+		strvec_push(&refs_verify.args, "--verbose");
+	if (check_strict)
+		strvec_push(&refs_verify.args, "--strict");
+
+	if (run_command(&refs_verify))
+		errors_found |= ERROR_REFS;
+}
+
 static char const * const fsck_usage[] = {
 	N_("git fsck [--tags] [--root] [--unreachable] [--cache] [--no-reflogs]\n"
 	   "         [--[no-]full] [--strict] [--verbose] [--lost-found]\n"
@@ -1065,10 +1080,7 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
 
 	check_connectivity();
 
-	if (refs_fsck(get_main_ref_store(the_repository))) {
-		error("ref database is corrupt");
-		errors_found |= ERROR_REFS;
-	}
+	fsck_refs();
 
 	if (the_repository->settings.core_commit_graph) {
 		struct child_process commit_graph_verify = CHILD_PROCESS_INIT;

-

builtin/refs.c +45 -0

70 lines not on this page · raw message

fsck.c +69 -0

81 lines not on this page · raw message

fsck.h +41 -2

93 lines not on this page · raw message

refs.c +3 -3
index 06e191a316..1325f83269 100644
@@ -316,9 +316,9 @@ int check_refname_format(const char *refname, int flags)
 	return check_or_sanitize_refname(refname, flags, NULL);
 }
 
-int refs_fsck(struct ref_store *refs)
+int refs_fsck(struct ref_store *refs, struct fsck_refs_options *o)
 {
-	return refs->be->fsck(refs);
+	return refs->be->fsck(refs, o);
 }
 
 void sanitize_refname_component(const char *refname, struct strbuf *out)
@@ -1744,7 +1744,7 @@ static int refs_read_special_head(struct ref_store *ref_store,
 	}
 
 	result = parse_loose_ref_contents(content.buf, oid, referent, type,
-					  failure_errno);
+					  failure_errno, NULL);
 
 done:
 	strbuf_release(&full_path);
refs.h +6 -3
index 55160085b6..5a042695f5 100644
@@ -3,6 +3,7 @@
 
 #include "commit.h"
 
+struct fsck_refs_options;
 struct object_id;
 struct ref_store;
 struct repository;
@@ -548,9 +549,11 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat
 int check_refname_format(const char *refname, int flags);
 
 /*
-  * Return 0 iff all refs in filesystem are consistent.
-*/
-int refs_fsck(struct ref_store *refs);
+ * Check the reference database for consistency. Return 0 if refs and
+ * reflogs are consistent, and non-zero otherwise. The errors will be
+ * written to stderr.
+ */
+int refs_fsck(struct ref_store *refs, struct fsck_refs_options *o);
 
 /*
  * Apply the rules from check_refname_format, but mutate the result until it
refs/debug.c +9 -0
index 547d9245b9..110a264522 100644
@@ -419,6 +419,13 @@ static int debug_reflog_expire(struct ref_store *ref_store, const char *refname,
 	return res;
 }
 
+static int debug_fsck(struct ref_store *ref_store,
+		      struct fsck_refs_options *o)
+{
+	trace_printf_key(&trace_refs, "fsck\n");
+	return 0;
+}
+
 struct ref_storage_be refs_be_debug = {
 	.name = "debug",
 	.init = NULL,
@@ -451,4 +458,6 @@ struct ref_storage_be refs_be_debug = {
 	.create_reflog = debug_create_reflog,
 	.delete_reflog = debug_delete_reflog,
 	.reflog_expire = debug_reflog_expire,
+
+	.fsck = debug_fsck,
 };
refs/files-backend.c +172 -63

325 lines not on this page · raw message

refs/packed-backend.c +2 -1
index 3de0fd59a7..db152053f8 100644
@@ -1733,7 +1733,8 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_s
 	return empty_ref_iterator_begin();
 }
 
-static int packed_fsck(struct ref_store *ref_store)
+static int packed_fsck(struct ref_store *ref_store,
+		       struct fsck_refs_options *o)
 {
 	return 0;
 }
refs/refs-internal.h +6 -3
index 16d92b252e..eb3a7cdcc1 100644
@@ -4,6 +4,7 @@
 #include "refs.h"
 #include "iterator.h"
 
+struct fsck_refs_options;
 struct ref_transaction;
 
 /*
@@ -650,7 +651,8 @@ typedef int read_raw_ref_fn(struct ref_store *ref_store, const char *refname,
 typedef int read_symbolic_ref_fn(struct ref_store *ref_store, const char *refname,
 				 struct strbuf *referent);
 
-typedef int fsck_fn(struct ref_store *ref_store);
+typedef int fsck_fn(struct ref_store *ref_store,
+		    struct fsck_refs_options *o);
 
 struct ref_storage_be {
 	const char *name;
@@ -707,11 +709,12 @@ struct ref_store {
 
 /*
  * Parse contents of a loose ref file. *failure_errno maybe be set to EINVAL for
- * invalid contents.
+ * invalid contents. Also trailing is set to 1 when there is any bytes after the
+ * hex.
  */
 int parse_loose_ref_contents(const char *buf, struct object_id *oid,
 			     struct strbuf *referent, unsigned int *type,
-			     int *failure_errno);
+			     int *failure_errno, unsigned int *trailing);
 
 /*
  * Fill in the generic part of refs and add it to our collection of
refs/reftable-backend.c +2 -1
index 8f0ce5aed0..7f606faa9e 100644
@@ -2242,7 +2242,8 @@ static int reftable_be_reflog_expire(struct ref_store *ref_store,
 	return ret;
 }
 
-static int reftable_be_fsck(struct ref_store *ref_store)
+static int reftable_be_fsck(struct ref_store *ref_store,
+			    struct fsck_refs_options *o)
 {
 	return 0;
 }
t/t0602-reffiles-fsck.sh +107 -28

206 lines not on this page · raw message

t/t0610-reftable-basics.sh +0 -1
index b06c46999d..cc5bbfd732 100755
@@ -10,7 +10,6 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
 GIT_TEST_DEFAULT_REF_FORMAT=reftable
 export GIT_TEST_DEFAULT_REF_FORMAT
 
-TEST_PASSES_SANITIZE_LEAK=true
 . ./test-lib.sh
 
 INVALID_OID=$(test_oid 001)
t/t1300-config.sh +1 -8
index 9de2d95f06..f3c4d28e06 100755
@@ -596,8 +596,7 @@ test_expect_success 'get bool variable with empty value' '
 
 test_expect_success 'no arguments, but no crash' '
 	test_must_fail git config >output 2>&1 &&
-	echo "error: no action specified" >expect &&
-	test_cmp expect output
+	test_grep usage output
 '
 
 cat > .git/config << EOF
@@ -2835,12 +2834,6 @@ test_expect_success 'specifying multiple modes causes failure' '
 	test_cmp expect err
 '
 
-test_expect_success 'writing to stdin is rejected' '
-	echo "fatal: writing to stdin is not supported" >expect &&
-	test_must_fail git config ${mode_set} --file - foo.bar baz 2>err &&
-	test_cmp expect err
-'
-
 done
 
 test_done
Signature · 1 line
2.45.2
author
#13 lore raw

PATCH v2 1/7 fsck: add refs check interfaces to interface with fsck error levels

The git-fsck(1) focuses on object database consistency check. It relies on the "fsck_options" to interact with fsck error levels. However, "fsck_options" aims at checking the object database which makes it unsuitable to change the semantics of it. Instead, create "fsck_refs_options" structure to handle refs consistency check.

The "git_fsck_config" sets up the "msg_type" and "skiplist" member of the "fsck_options". For refs, we just need the "msg_type". In order to allow setting up more refs-specific options easily later, add a separate function "git_fsck_refs_config" to initialize the refs-specific options.

Move the "msg_type" and "strict" member to the top of the "fsck_options" which allows us to convert "fsck_refs_options *" to "fsck_options *" to reuse the interfaces provided by "fsck.h" without changing the original code.

The static function "report" provided by "fsck.c" aims at reporting the problems related to object database which cannot be reused for refs. Provide "fsck_refs_report" function to integrate the fsck error levels into reference consistency check.

---
 fsck.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 fsck.h | 39 +++++++++++++++++++++++++++++++--
 2 files changed, 106 insertions(+), 2 deletions(-)
fsck.c +69 -0
index 8ef962199f..13528c646e 100644
@@ -1249,6 +1249,20 @@ int fsck_buffer(const struct object_id *oid, enum object_type type,
 		      type);
 }
 
+int fsck_refs_error_function(struct fsck_refs_options *o UNUSED,
+			     const char *name,
+			     enum fsck_msg_type msg_type,
+			     enum fsck_msg_id msg_id UNUSED,
+			     const char *message)
+{
+	if (msg_type == FSCK_WARN) {
+		warning("%s: %s", name, message);
+		return 0;
+	}
+	error("%s: %s", name, message);
+	return 1;
+}
+
 int fsck_error_function(struct fsck_options *o,
 			const struct object_id *oid,
 			enum object_type object_type UNUSED,
@@ -1323,6 +1337,61 @@ int fsck_finish(struct fsck_options *options)
 	return ret;
 }
 
+int fsck_refs_report(struct fsck_refs_options *o,
+		     const char *name,
+		     enum fsck_msg_id msg_id,
+		     const char *fmt, ...)
+{
+	va_list ap;
+	struct strbuf sb = STRBUF_INIT;
+	enum fsck_msg_type msg_type =
+		fsck_msg_type(msg_id, (struct fsck_options*)o);
+	int ret = 0;
+
+	if (msg_type == FSCK_IGNORE)
+		return 0;
+
+	if (msg_type == FSCK_FATAL)
+		msg_type = FSCK_ERROR;
+	else if (msg_type == FSCK_INFO)
+		msg_type = FSCK_WARN;
+
+	prepare_msg_ids();
+	strbuf_addf(&sb, "%s: ", msg_id_info[msg_id].camelcased);
+
+	va_start(ap, fmt);
+	strbuf_vaddf(&sb, fmt, ap);
+	ret = o->error_func(o, name, msg_type, msg_id, sb.buf);
+	strbuf_release(&sb);
+	va_end(ap);
+
+	return ret;
+}
+
+int git_fsck_refs_config(const char *var, const char *value,
+			 const struct config_context *ctx, void *cb)
+{
+	struct fsck_refs_options *options = cb;
+	const char *msg_id;
+
+	/*
+	 * We don't check the value of fsck.skiplist here, because it
+	 * is specific to object database, not reference database.
+	 */
+	if (strcmp(var, "fsck.skiplist") == 0) {
+		return 0;
+	}
+
+	if (skip_prefix(var, "fsck.", &msg_id)) {
+		if (!value)
+			return config_error_nonbool(var);
+		fsck_set_msg_type((struct fsck_options*)options, msg_id, value);
+		return 0;
+	}
+
+	return git_default_config(var, value, ctx, cb);
+}
+
 int git_fsck_config(const char *var, const char *value,
 		    const struct config_context *ctx, void *cb)
 {
fsck.h +37 -2
index 17fa2dda5d..7451b1f91b 100644
@@ -96,6 +96,7 @@ enum fsck_msg_id {
 };
 #undef MSG_ID
 
+struct fsck_refs_options;
 struct fsck_options;
 struct object;
 
@@ -107,6 +108,21 @@ void fsck_set_msg_type(struct fsck_options *options,
 void fsck_set_msg_types(struct fsck_options *options, const char *values);
 int is_valid_msg_type(const char *msg_id, const char *msg_type);
 
+/*
+ * callback function for fsck refs and reflogs.
+ */
+typedef int (*fsck_refs_error)(struct fsck_refs_options *o,
+			       const char *name,
+			       enum fsck_msg_type msg_type,
+			       enum fsck_msg_id msg_id,
+			       const char *message);
+
+int fsck_refs_error_function(struct fsck_refs_options *o,
+			     const char *name,
+			     enum fsck_msg_type msg_type,
+			     enum fsck_msg_id msg_id,
+			     const char *message);
+
 /*
  * callback function for fsck_walk
  * type is the expected type of the object or OBJ_ANY
@@ -135,11 +151,22 @@ int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o,
 					   enum fsck_msg_id msg_id,
 					   const char *message);
 
+struct fsck_refs_options {
+	enum fsck_msg_type *msg_type;
+	unsigned strict:1;
+	fsck_refs_error error_func;
+	unsigned verbose:1;
+};
+
+#define FSCK_REFS_OPTIONS_DEFAULT { \
+	.error_func = fsck_refs_error_function, \
+}
+
 struct fsck_options {
+	enum fsck_msg_type *msg_type;
+	unsigned strict:1;
 	fsck_walk_func walk;
 	fsck_error error_func;
-	unsigned strict:1;
-	enum fsck_msg_type *msg_type;
 	struct oidset skiplist;
 	struct oidset gitmodules_found;
 	struct oidset gitmodules_done;
@@ -221,6 +248,12 @@ int fsck_tag_standalone(const struct object_id *oid, const char *buffer,
  */
 int fsck_finish(struct fsck_options *options);
 
+__attribute__((format (printf, 4, 5)))
+int fsck_refs_report(struct fsck_refs_options *o,
+		     const char *name,
+		     enum fsck_msg_id msg_id,
+		     const char *fmt, ...);
+
 /*
  * Subsystem for storing human-readable names for each object.
  *
@@ -247,6 +280,8 @@ const char *fsck_describe_object(struct fsck_options *options,
 				 const struct object_id *oid);
 
 struct key_value_info;
+int git_fsck_refs_config(const char *var, const char *value,
+			 const struct config_context *ctx, void *cb);
 /*
  * git_config() callback for use by fsck-y tools that want to support
  * fsck.<msg> fsck.skipList etc.
Signature · 1 line
2.45.2
author
#14 lore raw

PATCH v2 2/7 refs: set up ref consistency check infrastructure

In reply to shejialuo

The interfaces defined in the ref_storage_be are carefully structured in semantic. It's organized as the five parts:

1. The name and the initialization interfaces.
2. The ref transaction interfaces.
3. The ref internal interfaces (pack, rename and copy).
4. The ref filesystem interfaces.
5. The reflog related interfaces.

To keep consistent with the git-fsck(1), add a new interface named "fsck_refs_fn" to the end of "ref_storage_be". This semantic cannot be grouped into any above five categories. Explicitly add blank line to make it different from others.

---
 refs.c                  |  5 +++++
 refs.h                  |  8 ++++++++
 refs/debug.c            |  9 +++++++++
 refs/files-backend.c    | 15 ++++++++++++++-
 refs/packed-backend.c   |  8 ++++++++
 refs/refs-internal.h    |  6 ++++++
 refs/reftable-backend.c |  8 ++++++++
 7 files changed, 58 insertions(+), 1 deletion(-)
refs.c +5 -0
index f7c7765d23..0922439275 100644
@@ -316,6 +316,11 @@ int check_refname_format(const char *refname, int flags)
 	return check_or_sanitize_refname(refname, flags, NULL);
 }
 
+int refs_fsck(struct ref_store *refs, struct fsck_refs_options *o)
+{
+	return refs->be->fsck(refs, o);
+}
+
 void sanitize_refname_component(const char *refname, struct strbuf *out)
 {
 	if (check_or_sanitize_refname(refname, REFNAME_ALLOW_ONELEVEL, out))
refs.h +8 -0
index 76d25df4de..5a042695f5 100644
@@ -3,6 +3,7 @@
 
 #include "commit.h"
 
+struct fsck_refs_options;
 struct object_id;
 struct ref_store;
 struct repository;
@@ -547,6 +548,13 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat
  */
 int check_refname_format(const char *refname, int flags);
 
+/*
+ * Check the reference database for consistency. Return 0 if refs and
+ * reflogs are consistent, and non-zero otherwise. The errors will be
+ * written to stderr.
+ */
+int refs_fsck(struct ref_store *refs, struct fsck_refs_options *o);
+
 /*
  * Apply the rules from check_refname_format, but mutate the result until it
  * is acceptable, and place the result in "out".
refs/debug.c +9 -0
index 547d9245b9..110a264522 100644
@@ -419,6 +419,13 @@ static int debug_reflog_expire(struct ref_store *ref_store, const char *refname,
 	return res;
 }
 
+static int debug_fsck(struct ref_store *ref_store,
+		      struct fsck_refs_options *o)
+{
+	trace_printf_key(&trace_refs, "fsck\n");
+	return 0;
+}
+
 struct ref_storage_be refs_be_debug = {
 	.name = "debug",
 	.init = NULL,
@@ -451,4 +458,6 @@ struct ref_storage_be refs_be_debug = {
 	.create_reflog = debug_create_reflog,
 	.delete_reflog = debug_delete_reflog,
 	.reflog_expire = debug_reflog_expire,
+
+	.fsck = debug_fsck,
 };
refs/files-backend.c +14 -1
index cb752d32b6..e965345ad8 100644
@@ -3402,6 +3402,17 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,
 	return ret;
 }
 
+static int files_fsck(struct ref_store *ref_store,
+		      struct fsck_refs_options *o)
+{
+	int ret;
+	struct files_ref_store *refs =
+		files_downcast(ref_store, REF_STORE_READ, "fsck");
+
+	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o);
+	return ret;
+}
+
 struct ref_storage_be refs_be_files = {
 	.name = "files",
 	.init = files_ref_store_init,
@@ -3428,5 +3439,7 @@ struct ref_storage_be refs_be_files = {
 	.reflog_exists = files_reflog_exists,
 	.create_reflog = files_create_reflog,
 	.delete_reflog = files_delete_reflog,
-	.reflog_expire = files_reflog_expire
+	.reflog_expire = files_reflog_expire,
+
+	.fsck = files_fsck,
 };
refs/packed-backend.c +8 -0
index c4c1e36aa2..db152053f8 100644
@@ -1733,6 +1733,12 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_s
 	return empty_ref_iterator_begin();
 }
 
+static int packed_fsck(struct ref_store *ref_store,
+		       struct fsck_refs_options *o)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_packed = {
 	.name = "packed",
 	.init = packed_ref_store_init,
@@ -1760,4 +1766,6 @@ struct ref_storage_be refs_be_packed = {
 	.create_reflog = NULL,
 	.delete_reflog = NULL,
 	.reflog_expire = NULL,
+
+	.fsck = packed_fsck,
 };
refs/refs-internal.h +6 -0
index cbcb6f9c36..8f42f21e77 100644
@@ -4,6 +4,7 @@
 #include "refs.h"
 #include "iterator.h"
 
+struct fsck_refs_options;
 struct ref_transaction;
 
 /*
@@ -650,6 +651,9 @@ typedef int read_raw_ref_fn(struct ref_store *ref_store, const char *refname,
 typedef int read_symbolic_ref_fn(struct ref_store *ref_store, const char *refname,
 				 struct strbuf *referent);
 
+typedef int fsck_fn(struct ref_store *ref_store,
+		    struct fsck_refs_options *o);
+
 struct ref_storage_be {
 	const char *name;
 	ref_store_init_fn *init;
@@ -677,6 +681,8 @@ struct ref_storage_be {
 	create_reflog_fn *create_reflog;
 	delete_reflog_fn *delete_reflog;
 	reflog_expire_fn *reflog_expire;
+
+	fsck_fn *fsck;
 };
 
 extern struct ref_storage_be refs_be_files;
refs/reftable-backend.c +8 -0
index e555be4671..7f606faa9e 100644
@@ -2242,6 +2242,12 @@ static int reftable_be_reflog_expire(struct ref_store *ref_store,
 	return ret;
 }
 
+static int reftable_be_fsck(struct ref_store *ref_store,
+			    struct fsck_refs_options *o)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_reftable = {
 	.name = "reftable",
 	.init = reftable_be_init,
@@ -2269,4 +2275,6 @@ struct ref_storage_be refs_be_reftable = {
 	.create_reflog = reftable_be_create_reflog,
 	.delete_reflog = reftable_be_delete_reflog,
 	.reflog_expire = reftable_be_reflog_expire,
+
+	.fsck = reftable_be_fsck,
 };
Signature · 1 line
2.45.2
#15 lore raw

Re: [GSoC][PATCH v2 2/7] refs: set up ref consistency check infrastructure

shejialuo <[email protected]> writes:

[snip]

Quoted 6 lines
refs/debug.c +0 -0
index 547d9245b9..110a264522 100644
@@ -419,6 +419,13 @@ static int debug_reflog_expire(struct ref_store *ref_store, const char *refname,
 	return res;

refs/debug.c static int debug_reflog_expire(struct ref_store *ref_store, const char *refname, · line 429

Quoted 10 lines
refs/debug.c +7 -0
 }
 
+static int debug_fsck(struct ref_store *ref_store,
+		      struct fsck_refs_options *o)
+{
+	trace_printf_key(&trace_refs, "fsck\n");
+	return 0;
+}
+
 

The debug interface acts as a wrapper around the actual refs backend being used, as such, you need to still call the underlying backend here.

[snip]

author
#16 lore raw

PATCH v2 3/7 builtin/refs: add verify subcommand

In reply to shejialuo

Introduce a new subcommand "verify" in git-refs(1) to allow the user to check the reference database consistency.

---
 Documentation/git-refs.txt | 11 ++++++++++
 builtin/refs.c             | 45 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
Documentation/git-refs.txt +11 -0
index 5b99e04385..f9d36ea19d 100644
@@ -10,6 +10,7 @@ SYNOPSIS
 --------
 [verse]
 'git refs migrate' --ref-format=<format> [--dry-run]
+'git refs verify' [--strict] [--verbose]
 
 DESCRIPTION
 -----------
@@ -22,6 +23,9 @@ COMMANDS
 migrate::
 	Migrate ref store between different formats.
 
+verify::
+	Verify reference database consistency.
+
 OPTIONS
 -------
 
@@ -39,6 +43,13 @@ include::ref-storage-format.txt[]
 	can be used to double check that the migration works as expected before
 	performing the actual migration.
 
+--strict::
+	Enable more strict checking, every WARN severity for the `Fsck Messages`
+	be seen as ERROR.
+
+--verbose::
+	When verifying the reference database consistency, be chatty.
+
 KNOWN LIMITATIONS
 -----------------
 
builtin/refs.c +45 -0

70 lines not on this page · raw message

Signature · 1 line
2.45.2
#17 lore raw

Re: [GSoC][PATCH v2 3/7] builtin/refs: add verify subcommand

Quoted 34 lines · shejialuo

Introduce a new subcommand "verify" in git-refs(1) to allow the user to check the reference database consistency.

---
 Documentation/git-refs.txt | 11 ++++++++++
 builtin/refs.c             | 45 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
Documentation/git-refs.txt +4 -0
index 5b99e04385..f9d36ea19d 100644
@@ -10,6 +10,7 @@ SYNOPSIS
 --------
 [verse]
 'git refs migrate' --ref-format=<format> [--dry-run]
+'git refs verify' [--strict] [--verbose]
 
 DESCRIPTION
 -----------
@@ -22,6 +23,9 @@ COMMANDS
 migrate::
 	Migrate ref store between different formats.
 
+verify::
+	Verify reference database consistency.
+
 OPTIONS
 -------
 

Documentation/git-refs.txt include::ref-storage-format.txt[] · line 45

Quoted 3 lines
Documentation/git-refs.txt +0 -0
@@ -39,6 +43,13 @@ include::ref-storage-format.txt[]
 	can be used to double check that the migration works as expected before
 	performing the actual migration.
 

Above it states that the options mentioned are "specific to 'git refs migrate'", shouldn't we do something similar here?

Quoted 60 lines
Documentation/git-refs.txt +7 -0
+--strict::
+	Enable more strict checking, every WARN severity for the `Fsck Messages`
+	be seen as ERROR.
+
+--verbose::
+	When verifying the reference database consistency, be chatty.
+
 KNOWN LIMITATIONS
 -----------------
 
builtin/refs.c +30 -0
index 46dcd150d4..82ed0d57c1 100644
@@ -1,4 +1,6 @@
 #include "builtin.h"
+#include "config.h"
+#include "fsck.h"
 #include "parse-options.h"
 #include "refs.h"
 #include "repository.h"
@@ -7,6 +9,9 @@
 #define REFS_MIGRATE_USAGE \
 	N_("git refs migrate --ref-format=<format> [--dry-run]")
 
+#define REFS_VERIFY_USAGE \
+	N_("git refs verify [--strict] [--verbose]")
+
 static int cmd_refs_migrate(int argc, const char **argv, const char *prefix)
 {
 	const char * const migrate_usage[] = {
@@ -58,15 +63,55 @@ static int cmd_refs_migrate(int argc, const char **argv, const char *prefix)
 	return err;
 }
 
+static int cmd_refs_verify(int argc, const char **argv, const char *prefix)
+{
+	const char * const verify_usage[] = {
+		REFS_VERIFY_USAGE,
+		NULL,
+	};
+	int ret = 0;
+	unsigned int verbose = 0, strict = 0;
+	struct fsck_refs_options fsck_refs_options = FSCK_REFS_OPTIONS_DEFAULT;
+	struct option options[] = {
+		OPT__VERBOSE(&verbose, N_("be verbose")),
+		OPT_BOOL(0, "strict", &strict, N_("enable strict checking")),
+		OPT_END(),
+	};
+
+	argc = parse_options(argc, argv, prefix, options, verify_usage, 0);
+	if (argc)
+		usage(_("too many arguments"));
+
+	if (verbose)
+		fsck_refs_options.verbose = 1;
+	if (strict)
+		fsck_refs_options.strict = 1;
+
+	git_config(git_fsck_refs_config, &fsck_refs_options);

builtin/refs.c static int cmd_refs_migrate(int argc, const char **argv, const char *prefix) · line 100

Quoted 10 lines
builtin/refs.c +10 -0
+	prepare_repo_settings(the_repository);
+
+	ret = refs_fsck(get_main_ref_store(the_repository), &fsck_refs_options);
+
+	/*
+	 * Explicitly free the allocated array. This is necessary because
+	 * this program is executed as child process of git-fsck(1) and the
+	 * allocated array may not freed when git-fsck(1) aborts somewhere.
+	 */
+	free(fsck_refs_options.msg_type);

Nice, this is a good thing to do even if this isn't called as a child process.

Quoted 19 lines
builtin/refs.c +5 -1
+	return ret;
+}
+
 int cmd_refs(int argc, const char **argv, const char *prefix)
 {
 	const char * const refs_usage[] = {
 		REFS_MIGRATE_USAGE,
+		REFS_VERIFY_USAGE,
 		NULL,
 	};
 	parse_opt_subcommand_fn *fn = NULL;
 	struct option opts[] = {
 		OPT_SUBCOMMAND("migrate", &fn, cmd_refs_migrate),
+		OPT_SUBCOMMAND("verify", &fn, cmd_refs_verify),
 		OPT_END(),
 	};
 
--

2.45.2

author
#18 lore raw

PATCH v2 4/7 builtin/fsck: add `git-refs verify` child process

In reply to shejialuo

Introduce a new function "fsck_refs" that initializes and runs a child process to execute the "git-refs verify" command.

---
 builtin/fsck.c | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
builtin/fsck.c +17 -0
index d13a226c2e..10d73f534f 100644
@@ -896,6 +896,21 @@ static int check_pack_rev_indexes(struct repository *r, int show_progress)
 	return res;
 }
 
+static void fsck_refs(void)
+{
+	struct child_process refs_verify = CHILD_PROCESS_INIT;
+	child_process_init(&refs_verify);
+	refs_verify.git_cmd = 1;
+	strvec_pushl(&refs_verify.args, "refs", "verify", NULL);
+	if (verbose)
+		strvec_push(&refs_verify.args, "--verbose");
+	if (check_strict)
+		strvec_push(&refs_verify.args, "--strict");
+
+	if (run_command(&refs_verify))
+		errors_found |= ERROR_REFS;
+}
+
 static char const * const fsck_usage[] = {
 	N_("git fsck [--tags] [--root] [--unreachable] [--cache] [--no-reflogs]\n"
 	   "         [--[no-]full] [--strict] [--verbose] [--lost-found]\n"
@@ -1065,6 +1080,8 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
 
 	check_connectivity();
 
+	fsck_refs();
+
 	if (the_repository->settings.core_commit_graph) {
 		struct child_process commit_graph_verify = CHILD_PROCESS_INIT;
 
Signature · 1 line
2.45.2
#19 lore raw

Re: [GSoC][PATCH v2 4/7] builtin/fsck: add `git-refs verify` child process

Quoted 23 lines · shejialuo

Introduce a new function "fsck_refs" that initializes and runs a child process to execute the "git-refs verify" command.

---
 builtin/fsck.c | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
builtin/fsck.c +5 -0
index d13a226c2e..10d73f534f 100644
@@ -896,6 +896,21 @@ static int check_pack_rev_indexes(struct repository *r, int show_progress)
 	return res;
 }
 
+static void fsck_refs(void)
+{
+	struct child_process refs_verify = CHILD_PROCESS_INIT;
+	child_process_init(&refs_verify);
+	refs_verify.git_cmd = 1;

builtin/fsck.c static int check_pack_rev_indexes(struct repository *r, int show_progress) · line 913

Quoted 10 lines
builtin/fsck.c +10 -0
+	strvec_pushl(&refs_verify.args, "refs", "verify", NULL);
+	if (verbose)
+		strvec_push(&refs_verify.args, "--verbose");
+	if (check_strict)
+		strvec_push(&refs_verify.args, "--strict");
+
+	if (run_command(&refs_verify))
+		errors_found |= ERROR_REFS;
+}
+

At first I thought we need to call child_process_clear() here, but seems like run_command does that internally.

Quoted 14 lines
builtin/fsck.c +2 -1
 static char const * const fsck_usage[] = {
 	N_("git fsck [--tags] [--root] [--unreachable] [--cache] [--no-reflogs]\n"
 	   "         [--[no-]full] [--strict] [--verbose] [--lost-found]\n"
@@ -1065,6 +1080,8 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
 
 	check_connectivity();
 
+	fsck_refs();
+
 	if (the_repository->settings.core_commit_graph) {
 		struct child_process commit_graph_verify = CHILD_PROCESS_INIT;
 
--

2.45.2

author
#20 lore raw

PATCH v2 5/7 files-backend: add unified interface for refs scanning

In reply to shejialuo

For refs and reflogs, we need to scan its corresponding directories to check every regular file or symbolic link which shares the same pattern. Introduce a unified interface for scanning directories for files-backend.

---
 refs/files-backend.c | 75 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)
refs/files-backend.c +75 -0
index e965345ad8..b26cfb8ba6 100644
@@ -4,6 +4,7 @@
 #include "../gettext.h"
 #include "../hash.h"
 #include "../hex.h"
+#include "../fsck.h"
 #include "../refs.h"
 #include "refs-internal.h"
 #include "ref-cache.h"
@@ -3402,6 +3403,78 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,
 	return ret;
 }
 
+/*
+ * For refs and reflogs, they share a unified interface when scanning
+ * the whole directory. This function is used as the callback for each
+ * regular file or symlink in the directory.
+ */
+typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o,
+				  const char *gitdir,
+				  const char *refs_check_dir,
+				  struct dir_iterator *iter);
+
+static int files_fsck_refs_dir(struct ref_store *ref_store,
+			       struct fsck_refs_options *o,
+			       const char *refs_check_dir,
+			       files_fsck_refs_fn *fsck_refs_fns)
+{
+	const char *gitdir = ref_store->gitdir;
+	struct strbuf sb = STRBUF_INIT;
+	struct dir_iterator *iter;
+	int iter_status;
+	int ret = 0;
+
+	strbuf_addf(&sb, "%s/%s", gitdir, refs_check_dir);
+
+	iter = dir_iterator_begin(sb.buf, 0);
+
+	if (!iter) {
+		ret = error_errno("cannot open directory %s", sb.buf);
+		goto out;
+	}
+
+	while ((iter_status = dir_iterator_advance(iter)) == ITER_OK) {
+		if (S_ISDIR(iter->st.st_mode)) {
+			continue;
+		} else if (S_ISREG(iter->st.st_mode) ||
+			   S_ISLNK(iter->st.st_mode)) {
+			if (o->verbose)
+				fprintf_ln(stderr, "Checking %s/%s",
+					   refs_check_dir, iter->relative_path);
+			for (size_t i = 0; fsck_refs_fns[i]; i++) {
+				if (fsck_refs_fns[i](o, gitdir, refs_check_dir, iter))
+					ret = -1;
+			}
+		} else {
+			ret = error(_("unexpected file type for '%s'"),
+				    iter->basename);
+		}
+	}
+
+	if (iter_status != ITER_DONE)
+		ret = error(_("failed to iterate over '%s'"), sb.buf);
+
+out:
+	strbuf_release(&sb);
+	return ret;
+}
+
+static int files_fsck_refs(struct ref_store *ref_store,
+			   struct fsck_refs_options *o)
+{
+	int ret;
+	files_fsck_refs_fn fsck_refs_fns[]= {
+		NULL
+	};
+
+	if (o->verbose)
+		fprintf_ln(stderr, "Checking references consistency");
+
+	ret = files_fsck_refs_dir(ref_store, o, "refs", fsck_refs_fns);
+
+	return ret;
+}
+
 static int files_fsck(struct ref_store *ref_store,
 		      struct fsck_refs_options *o)
 {
@@ -3410,6 +3483,8 @@ static int files_fsck(struct ref_store *ref_store,
 		files_downcast(ref_store, REF_STORE_READ, "fsck");
 
 	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o);
+	ret = files_fsck_refs(ref_store, o);
+
 	return ret;
 }
 
Signature · 1 line
2.45.2
#21 lore raw

Re: [GSoC][PATCH v2 5/7] files-backend: add unified interface for refs scanning

Quoted 5 lines · shejialuo

For refs and reflogs, we need to scan its corresponding directories to check every regular file or symbolic link which shares the same pattern. Introduce a unified interface for scanning directories for files-backend.

Here we talk about reflogs, but we only add a iterator for "$GIT_DIR/refs". Should we add something for reflogs too?

Quoted 84 lines
---
 refs/files-backend.c | 75 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)
refs/files-backend.c +62 -0
index e965345ad8..b26cfb8ba6 100644
@@ -4,6 +4,7 @@
 #include "../gettext.h"
 #include "../hash.h"
 #include "../hex.h"
+#include "../fsck.h"
 #include "../refs.h"
 #include "refs-internal.h"
 #include "ref-cache.h"
@@ -3402,6 +3403,78 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,
 	return ret;
 }
 
+/*
+ * For refs and reflogs, they share a unified interface when scanning
+ * the whole directory. This function is used as the callback for each
+ * regular file or symlink in the directory.
+ */
+typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o,
+				  const char *gitdir,
+				  const char *refs_check_dir,
+				  struct dir_iterator *iter);
+
+static int files_fsck_refs_dir(struct ref_store *ref_store,
+			       struct fsck_refs_options *o,
+			       const char *refs_check_dir,
+			       files_fsck_refs_fn *fsck_refs_fns)
+{
+	const char *gitdir = ref_store->gitdir;
+	struct strbuf sb = STRBUF_INIT;
+	struct dir_iterator *iter;
+	int iter_status;
+	int ret = 0;
+
+	strbuf_addf(&sb, "%s/%s", gitdir, refs_check_dir);
+
+	iter = dir_iterator_begin(sb.buf, 0);
+
+	if (!iter) {
+		ret = error_errno("cannot open directory %s", sb.buf);
+		goto out;
+	}
+
+	while ((iter_status = dir_iterator_advance(iter)) == ITER_OK) {
+		if (S_ISDIR(iter->st.st_mode)) {
+			continue;
+		} else if (S_ISREG(iter->st.st_mode) ||
+			   S_ISLNK(iter->st.st_mode)) {
+			if (o->verbose)
+				fprintf_ln(stderr, "Checking %s/%s",
+					   refs_check_dir, iter->relative_path);
+			for (size_t i = 0; fsck_refs_fns[i]; i++) {
+				if (fsck_refs_fns[i](o, gitdir, refs_check_dir, iter))
+					ret = -1;
+			}
+		} else {
+			ret = error(_("unexpected file type for '%s'"),
+				    iter->basename);
+		}
+	}
+
+	if (iter_status != ITER_DONE)
+		ret = error(_("failed to iterate over '%s'"), sb.buf);
+
+out:
+	strbuf_release(&sb);
+	return ret;
+}
+
+static int files_fsck_refs(struct ref_store *ref_store,
+			   struct fsck_refs_options *o)
+{
+	int ret;
+	files_fsck_refs_fn fsck_refs_fns[]= {

refs/files-backend.c static int files_ref_store_remove_on_disk(struct ref_store *ref_store, · line 3476

Quoted 10 lines
refs/files-backend.c +10 -0
+		NULL
+	};
+
+	if (o->verbose)
+		fprintf_ln(stderr, "Checking references consistency");
+
+	ret = files_fsck_refs_dir(ref_store, o, "refs", fsck_refs_fns);
+
+	return ret;
+}

I'm not fully sure why this needs to be separate from files_fsck. Is that because we plan to also introduce files_fsck_reflogs?

Quoted 15 lines
refs/files-backend.c +3 -1
+
 static int files_fsck(struct ref_store *ref_store,
 		      struct fsck_refs_options *o)
 {
@@ -3410,6 +3483,8 @@ static int files_fsck(struct ref_store *ref_store,
 		files_downcast(ref_store, REF_STORE_READ, "fsck");
 
 	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o);
+	ret = files_fsck_refs(ref_store, o);
+
 	return ret;
 }
 
--

2.45.2

author
#22 lore raw

Re: [GSoC][PATCH v2 5/7] files-backend: add unified interface for refs scanning

Quoted 11 lines · Karthik Nayak
Quoted 5 lines · shejialuo

For refs and reflogs, we need to scan its corresponding directories to check every regular file or symbolic link which shares the same pattern. Introduce a unified interface for scanning directories for files-backend.

Here we talk about reflogs, but we only add a iterator for "$GIT_DIR/refs". Should we add something for reflogs too?

Here, we simply set up the basic structure. For both refs and reflogs, we will traverse the file system. What we concern about is the regular file and symlink file. The "refs_check_dir" parameter is used to indicate what we will check. Actually, we do not add any actual check operations for refs in this commit.

Quoted 89 lines
Quoted 89 lines
---
 refs/files-backend.c | 75 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)
refs/files-backend.c +67 -0
index e965345ad8..b26cfb8ba6 100644
@@ -4,6 +4,7 @@
 #include "../gettext.h"
 #include "../hash.h"
 #include "../hex.h"
+#include "../fsck.h"
 #include "../refs.h"
 #include "refs-internal.h"
 #include "ref-cache.h"
@@ -3402,6 +3403,78 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,
 	return ret;
 }
 
+/*
+ * For refs and reflogs, they share a unified interface when scanning
+ * the whole directory. This function is used as the callback for each
+ * regular file or symlink in the directory.
+ */
+typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o,
+				  const char *gitdir,
+				  const char *refs_check_dir,
+				  struct dir_iterator *iter);
+
+static int files_fsck_refs_dir(struct ref_store *ref_store,
+			       struct fsck_refs_options *o,
+			       const char *refs_check_dir,
+			       files_fsck_refs_fn *fsck_refs_fns)
+{
+	const char *gitdir = ref_store->gitdir;
+	struct strbuf sb = STRBUF_INIT;
+	struct dir_iterator *iter;
+	int iter_status;
+	int ret = 0;
+
+	strbuf_addf(&sb, "%s/%s", gitdir, refs_check_dir);
+
+	iter = dir_iterator_begin(sb.buf, 0);
+
+	if (!iter) {
+		ret = error_errno("cannot open directory %s", sb.buf);
+		goto out;
+	}
+
+	while ((iter_status = dir_iterator_advance(iter)) == ITER_OK) {
+		if (S_ISDIR(iter->st.st_mode)) {
+			continue;
+		} else if (S_ISREG(iter->st.st_mode) ||
+			   S_ISLNK(iter->st.st_mode)) {
+			if (o->verbose)
+				fprintf_ln(stderr, "Checking %s/%s",
+					   refs_check_dir, iter->relative_path);
+			for (size_t i = 0; fsck_refs_fns[i]; i++) {
+				if (fsck_refs_fns[i](o, gitdir, refs_check_dir, iter))
+					ret = -1;
+			}
+		} else {
+			ret = error(_("unexpected file type for '%s'"),
+				    iter->basename);
+		}
+	}
+
+	if (iter_status != ITER_DONE)
+		ret = error(_("failed to iterate over '%s'"), sb.buf);
+
+out:
+	strbuf_release(&sb);
+	return ret;
+}
+
+static int files_fsck_refs(struct ref_store *ref_store,
+			   struct fsck_refs_options *o)
+{
+	int ret;
+	files_fsck_refs_fn fsck_refs_fns[]= {
+		NULL
+	};
+
+	if (o->verbose)
+		fprintf_ln(stderr, "Checking references consistency");
Quoted 9 lines

refs/files-backend.c static int files_ref_store_remove_on_disk(struct ref_store *ref_store, · line 3476

Quoted 5 lines
refs/files-backend.c +5 -0
+
+	ret = files_fsck_refs_dir(ref_store, o, "refs", fsck_refs_fns);
+
+	return ret;
+}

I'm not fully sure why this needs to be separate from files_fsck. Is that because we plan to also introduce files_fsck_reflogs?

Yes, exactly. Later we need to introduce "files_fsck_reflogs" which should define the different files_fsck_refs_fn here. So just introduce a function here.

Quoted 15 lines
Quoted 15 lines
refs/files-backend.c +3 -1
+
 static int files_fsck(struct ref_store *ref_store,
 		      struct fsck_refs_options *o)
 {
@@ -3410,6 +3483,8 @@ static int files_fsck(struct ref_store *ref_store,
 		files_downcast(ref_store, REF_STORE_READ, "fsck");
 
 	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o);
+	ret = files_fsck_refs(ref_store, o);
+
 	return ret;
 }
 
--

2.45.2

author
#23 lore raw

PATCH v2 6/7 fsck: add ref name check for files backend

In reply to shejialuo

The git-fsck(1) only implicitly checks the reference, it does not fully check refs with bad format name such as standalone "@" and name ending with ".lock".

In order to provide such checks, add a new fsck message id "badRefName" with default ERROR type. Use existing "check_refname_format" to explicit check the ref name. And add a new unit test to verify the functionality.

---
 Documentation/fsck-msgids.txt |   3 +
 fsck.h                        |   1 +
 refs/files-backend.c          |  20 +++++++
 t/t0602-reffiles-fsck.sh      | 101 ++++++++++++++++++++++++++++++++++
 4 files changed, 125 insertions(+)
 create mode 100755 t/t0602-reffiles-fsck.sh
Documentation/fsck-msgids.txt +3 -0
index 5edc06c658..cc85c897cc 100644
@@ -19,6 +19,9 @@
 `badParentSha1`::
 	(ERROR) A commit object has a bad parent sha1.
 
+`badRefName`::
+	(ERROR) A ref has a bad name.
+
 `badTagName`::
 	(INFO) A tag has an invalid format.
 
fsck.h +1 -0
index 7451b1f91b..1423a5e428 100644
@@ -31,6 +31,7 @@ enum fsck_msg_type {
 	FUNC(BAD_NAME, ERROR) \
 	FUNC(BAD_OBJECT_SHA1, ERROR) \
 	FUNC(BAD_PARENT_SHA1, ERROR) \
+	FUNC(BAD_REF_NAME, ERROR) \
 	FUNC(BAD_TIMEZONE, ERROR) \
 	FUNC(BAD_TREE, ERROR) \
 	FUNC(BAD_TREE_SHA1, ERROR) \
refs/files-backend.c +20 -0
index b26cfb8ba6..266f1ffe8a 100644
@@ -3413,6 +3413,25 @@ typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o,
 				  const char *refs_check_dir,
 				  struct dir_iterator *iter);
 
+static int files_fsck_refs_name(struct fsck_refs_options *o,
+				const char *gitdir UNUSED,
+				const char *refs_check_dir,
+				struct dir_iterator *iter)
+{
+	struct strbuf sb = STRBUF_INIT;
+	int ret = 0;
+
+	if (check_refname_format(iter->basename, REFNAME_ALLOW_ONELEVEL)) {
+		strbuf_addf(&sb, "%s/%s", refs_check_dir, iter->relative_path);
+		ret = fsck_refs_report(o, sb.buf,
+				       FSCK_MSG_BAD_REF_NAME,
+				       "invalid refname format");
+	}
+
+	strbuf_release(&sb);
+	return ret;
+}
+
 static int files_fsck_refs_dir(struct ref_store *ref_store,
 			       struct fsck_refs_options *o,
 			       const char *refs_check_dir,
@@ -3464,6 +3483,7 @@ static int files_fsck_refs(struct ref_store *ref_store,
 {
 	int ret;
 	files_fsck_refs_fn fsck_refs_fns[]= {
+		files_fsck_refs_name,
 		NULL
 	};
 
t/t0602-reffiles-fsck.sh new file +101 -0

101 lines not on this page · raw message

Signature · 1 line
2.45.2
author
#24 lore raw

PATCH v2 7/7 fsck: add ref content check for files backend

In reply to shejialuo

Enhance the git-fsck(1) command by adding a check for reference content in the files backend. The new functionality ensures that symrefs, real symbolic link and regular refs are validated correctly.

Add a new function "files_fsck_symref" to check whether the symrefs and symbolic link points to a valid object and a new function "files_fsck_refs_content" handles both regular refs and symbolic refs.

In order to check the trailing content, add a new parameter "trailing" to "parse_loose_ref_contents" function.

Last, add the following FSCK MESSAGEs:

1. "badRefSha(ERROR)": A ref has a bad sha.
2. "danglingSymre(WARN)": Found a dangling symref.
3. "trailingRefContent(WARN)": A ref content has trailing contents.

---
 Documentation/fsck-msgids.txt |   9 +++
 fsck.h                        |   3 +
 refs.c                        |   2 +-
 refs/files-backend.c          | 124 +++++++++++++++++++++++++++++++++-
 refs/refs-internal.h          |   5 +-
 t/t0602-reffiles-fsck.sh      | 110 ++++++++++++++++++++++++++++++
 6 files changed, 248 insertions(+), 5 deletions(-)
Documentation/fsck-msgids.txt +9 -0
index cc85c897cc..69f86c5345 100644
@@ -22,6 +22,9 @@
 `badRefName`::
 	(ERROR) A ref has a bad name.
 
+`badRefSha`::
+	(ERROR) A ref has a bad sha.
+
 `badTagName`::
 	(INFO) A tag has an invalid format.
 
@@ -37,6 +40,9 @@
 `badType`::
 	(ERROR) Found an invalid object type.
 
+`danglingSymref`::
+	(WARN) Found a dangling symref.
+
 `duplicateEntries`::
 	(ERROR) A tree contains duplicate file entries.
 
@@ -179,6 +185,9 @@
 `symlinkTargetMissing`::
 	(ERROR) Unable to read symbolic link target's blob.
 
+`trailingRefContent`::
+	(WARN) A ref content has trailing contents.
+
 `treeNotSorted`::
 	(ERROR) A tree is not properly sorted.
 
fsck.h +3 -0
index 1423a5e428..5a55a567b0 100644
@@ -32,6 +32,7 @@ enum fsck_msg_type {
 	FUNC(BAD_OBJECT_SHA1, ERROR) \
 	FUNC(BAD_PARENT_SHA1, ERROR) \
 	FUNC(BAD_REF_NAME, ERROR) \
+	FUNC(BAD_REF_SHA, ERROR) \
 	FUNC(BAD_TIMEZONE, ERROR) \
 	FUNC(BAD_TREE, ERROR) \
 	FUNC(BAD_TREE_SHA1, ERROR) \
@@ -69,11 +70,13 @@ enum fsck_msg_type {
 	FUNC(SYMLINK_TARGET_BLOB, ERROR) \
 	/* warnings */ \
 	FUNC(EMPTY_NAME, WARN) \
+	FUNC(DANGLING_SYMREF, WARN) \
 	FUNC(FULL_PATHNAME, WARN) \
 	FUNC(HAS_DOT, WARN) \
 	FUNC(HAS_DOTDOT, WARN) \
 	FUNC(HAS_DOTGIT, WARN) \
 	FUNC(NULL_SHA1, WARN) \
+	FUNC(TRAILING_REF_CONTENT, WARN) \
 	FUNC(ZERO_PADDED_FILEMODE, WARN) \
 	FUNC(NUL_IN_COMMIT, WARN) \
 	FUNC(LARGE_PATHNAME, WARN) \
refs.c +1 -1
index 0922439275..1325f83269 100644
@@ -1744,7 +1744,7 @@ static int refs_read_special_head(struct ref_store *ref_store,
 	}
 
 	result = parse_loose_ref_contents(content.buf, oid, referent, type,
-					  failure_errno);
+					  failure_errno, NULL);
 
 done:
 	strbuf_release(&full_path);
refs/files-backend.c +122 -2

154 lines not on this page · raw message

refs/refs-internal.h +3 -2
index 8f42f21e77..eb3a7cdcc1 100644
@@ -709,11 +709,12 @@ struct ref_store {
 
 /*
  * Parse contents of a loose ref file. *failure_errno maybe be set to EINVAL for
- * invalid contents.
+ * invalid contents. Also trailing is set to 1 when there is any bytes after the
+ * hex.
  */
 int parse_loose_ref_contents(const char *buf, struct object_id *oid,
 			     struct strbuf *referent, unsigned int *type,
-			     int *failure_errno);
+			     int *failure_errno, unsigned int *trailing);
 
 /*
  * Fill in the generic part of refs and add it to our collection of
t/t0602-reffiles-fsck.sh +110 -0

114 lines not on this page · raw message

Signature · 1 line
2.45.2
#25 lore raw

Re: [GSoC][PATCH v2 7/7] fsck: add ref content check for files backend

Quoted 2 lines · shejialuo

In order to check the trailing content, add a new parameter "trailing" to "parse_loose_ref_contents" function.

About this one.

Quoted 8 lines

int parse_loose_ref_contents(const char *buf, struct object_id *oid,

refs/files-backend.c +1 -1
 			     struct strbuf *referent, unsigned int *type,
-			     int *failure_errno)
+			     int *failure_errno, unsigned int *trailing)
 {
 	const char *p;
 	if (skip_prefix(buf, "ref:", &buf)) {

refs/files-backend.c int parse_loose_ref_contents(const char *buf, struct object_id *oid, · line 615

Quoted 9 lines
refs/files-backend.c +4 -0
@@ -607,6 +607,10 @@ int parse_loose_ref_contents(const char *buf, struct object_id *oid,
 		*failure_errno = EINVAL;
 		return -1;
 	}
+
+	if (trailing && (*p != '\0' && *p != '\n'))
+		*trailing = 1;
+
 	return 0;
 }

We know what the garbage looked like at this point. The caller owns the "buf" pointer and we are pointing into that buffer with the pointer p, and the garbage is right there.

So I am not sure if losing information by using "uint *" is a good idea. Wouldn't it make more sense to take "const char **trailing" as a parameter and tell the caller where the trailing junk begins?

refs/files-backend.c static int files_fsck_refs_name(struct fsck_refs_options *o, · line 3441

Quoted 3 lines
refs/files-backend.c +3 -0
+static int files_fsck_symref(struct fsck_refs_options *o,
+			     struct strbuf *refname,
+			     struct strbuf *path)

This does not take things like HEAD or refs/remotes/origin/HEAD to validate. Instead, the caller is responsible for either doing a readlink on a symbolic link, or reading a textual symref and stripping "ref: " prefix from it, before calling this function. The "refname" parameter is not HEAD or refs/remotes/origin/HEAD but the pointee of the symref.

So I'd imagine that renaming it to fsck_symref_target or along that line to clarify that we are not checking the symref, but the target of a symref, would be a good idea.

refs/files-backend.c static int files_fsck_refs_name(struct fsck_refs_options *o, · line 3451

Quoted 10 lines
refs/files-backend.c +10 -0
+{
+	struct stat st;
+	int ret = 0;
+
+	if (lstat(path->buf, &st) < 0) {
+		ret = fsck_refs_report(o, refname->buf,
+				       FSCK_MSG_DANGLING_SYMREF,
+				       "point to non-existent ref");
+		goto out;
+	}

Is that an error? Just like being on an unborn branch is not an error, it could be argued that a symref that points at a branch yet to be born wouldn't be an error, either, no?

refs/files-backend.c static int files_fsck_refs_name(struct fsck_refs_options *o, · line 3457

Quoted 5 lines
refs/files-backend.c +5 -0
+	if (!S_ISREG(st.st_mode) && !S_ISLNK(st.st_mode)) {
+		ret = fsck_refs_report(o, refname->buf,
+				       FSCK_MSG_DANGLING_SYMREF,
+				       "point to invalid object");
+		goto out;

The use of "object" here is highly misleading. Yes, you can call a filesystem entity like "a regular file", "a directory", etc. "an object", but the word can refer to many other kinds of "object". In fact, I originally read this to mean "we are referring to an object in the object database that is corrupt" or something, but of course that is not what we are complaining about. We are complaining that the symbolic link points at a file of wrong type (like a directory).

So, in short, missing is probably OK. Pointing at a wrong thing (like a directory or block device) is not. Where, if any, do we catch a symbolic ref that tries to escape the refs/* hierarchy (e.g. ".git/refs/heads/my-crazy-ref" that is a symbolic link that points at "../../../../else/where" that is not even part of the repository), by the way?

Thanks.

author
#26 lore raw

Re: [GSoC][PATCH v2 7/7] fsck: add ref content check for files backend

Quoted 29 lines · Junio C Hamano
Quoted 2 lines · shejialuo

In order to check the trailing content, add a new parameter "trailing" to "parse_loose_ref_contents" function.

About this one.

Quoted 8 lines

int parse_loose_ref_contents(const char *buf, struct object_id *oid,

refs/files-backend.c +1 -1
 			     struct strbuf *referent, unsigned int *type,
-			     int *failure_errno)
+			     int *failure_errno, unsigned int *trailing)
 {
 	const char *p;
 	if (skip_prefix(buf, "ref:", &buf)) {

refs/files-backend.c int parse_loose_ref_contents(const char *buf, struct object_id *oid, · line 615

Quoted 9 lines
refs/files-backend.c +4 -0
@@ -607,6 +607,10 @@ int parse_loose_ref_contents(const char *buf, struct object_id *oid,
 		*failure_errno = EINVAL;
 		return -1;
 	}
+
+	if (trailing && (*p != '\0' && *p != '\n'))
+		*trailing = 1;
+
 	return 0;
 }

We know what the garbage looked like at this point. The caller owns the "buf" pointer and we are pointing into that buffer with the pointer p, and the garbage is right there.

Quoted 4 lines

So I am not sure if losing information by using "uint *" is a good idea. Wouldn't it make more sense to take "const char **trailing" as a parameter and tell the caller where the trailing junk begins?

Yes, I totally agree that using "uint *" will lose a lot of information here. Actually I have used the "const char **trailing", but I made a mistake to result the wild pointer. This is because when "parse_loose_ref_contents" handles symref, it will never handle *p. When the caller defines const char *trailing, it will be wild pointer. But I think we could set it to NULL when handling symref.

I will change the code in the next version.

Quoted 11 lines

refs/files-backend.c static int files_fsck_refs_name(struct fsck_refs_options *o, · line 3441

Quoted 3 lines
refs/files-backend.c +3 -0
+static int files_fsck_symref(struct fsck_refs_options *o,
+			     struct strbuf *refname,
+			     struct strbuf *path)

This does not take things like HEAD or refs/remotes/origin/HEAD to validate. Instead, the caller is responsible for either doing a readlink on a symbolic link, or reading a textual symref and stripping "ref: " prefix from it, before calling this function. The "refname" parameter is not HEAD or refs/remotes/origin/HEAD but the pointee of the symref.

Quoted 4 lines

So I'd imagine that renaming it to fsck_symref_target or along that line to clarify that we are not checking the symref, but the target of a symref, would be a good idea.

That's not correct. The "refname" parameter is EXACTLY the symref itself. What we do here is to check the "path" paramteter, there are two situations:

1. For symref we will strip "ref: " prefix, and combine the girdir and the stripped content to get the "path" parameter.
2. For symbolic we will get its actual path (here I made a mistake, I totally forget the situation when it points to absolute path, I will revise the code for handling it).

The design here is just check whether the symref points to the correct thing. It does not care about the pointee. The code will traverse every regular file under the "refs/" directory, eventually we will check the "pointee" status. For example, a symref "sym-branch" and a regular ref "branch".

sym-branch: "ref: refs/heads/branch".
branch: "xxxx"

The design will not report any error for "sym-branch". I think we should discuss here whether this design is OK.

Quoted 6 lines
Quoted 6 lines
refs/files-backend.c +6 -0
+{
+	struct stat st;
+	int ret = 0;
+
+	if (lstat(path->buf, &st) < 0) {
+		ret = fsck_refs_report(o, refname->buf,
Quoted 9 lines

refs/files-backend.c static int files_fsck_refs_name(struct fsck_refs_options *o, · line 3451

Quoted 4 lines
refs/files-backend.c +4 -0
+				       FSCK_MSG_DANGLING_SYMREF,
+				       "point to non-existent ref");
+		goto out;
+	}

Is that an error? Just like being on an unborn branch is not an error, it could be argued that a symref that points at a branch yet to be born wouldn't be an error, either, no?

The reason why I choose "danglingSymref" and warn severity is that I let the code be align with "git checkout". When we use "git checkout" for a dangling symref. It would produce the following output:

  $ git checkout branch-3
  warning: ignoring dangling symref refs/heads/branch-3
  error: pathspec 'branch-3' did not match any file(s) known to git

So I prefer to warn severity.

Quoted 6 lines

refs/files-backend.c static int files_fsck_refs_name(struct fsck_refs_options *o, · line 3457

Quoted 5 lines
refs/files-backend.c +5 -0
+	if (!S_ISREG(st.st_mode) && !S_ISLNK(st.st_mode)) {
+		ret = fsck_refs_report(o, refname->buf,
+				       FSCK_MSG_DANGLING_SYMREF,
+				       "point to invalid object");
+		goto out;
Quoted 8 lines

The use of "object" here is highly misleading. Yes, you can call a filesystem entity like "a regular file", "a directory", etc. "an object", but the word can refer to many other kinds of "object". In fact, I originally read this to mean "we are referring to an object in the object database that is corrupt" or something, but of course that is not what we are complaining about. We are complaining that the symbolic link points at a file of wrong type (like a directory).

Yes, it brings a lot of misleading here. I will clean the code and commit message (I also used object in commit message).

Quoted 7 lines

So, in short, missing is probably OK. Pointing at a wrong thing (like a directory or block device) is not. Where, if any, do we catch a symbolic ref that tries to escape the refs/* hierarchy (e.g. ".git/refs/heads/my-crazy-ref" that is a symbolic link that points at "../../../../else/where" that is not even part of the repository), by the way?

I intentionally ignored the "escape" situation. Actually, the path could be either absolute or relative. It may be a little complicated. I will find a way to support this in the next version.

Quoted 1 line

Thanks.

Thanks,
Jialuo

#27 lore raw

Re: [GSoC][PATCH v2 7/7] fsck: add ref content check for files backend

Quoted 11 lines · shejialuo
Quoted 11 lines

refs/files-backend.c static int files_fsck_refs_name(struct fsck_refs_options *o, · line 3441

Quoted 3 lines
refs/files-backend.c +3 -0
+static int files_fsck_symref(struct fsck_refs_options *o,
+			     struct strbuf *refname,
+			     struct strbuf *path)

This does not take things like HEAD or refs/remotes/origin/HEAD to validate. Instead, the caller is responsible for either doing a readlink on a symbolic link, or reading a textual symref and stripping "ref: " prefix from it, before calling this function. The "refname" parameter is not HEAD or refs/remotes/origin/HEAD but the pointee of the symref.

Quoted 6 lines
Quoted 3 lines

So I'd imagine that renaming it to fsck_symref_target or along that line to clarify that we are not checking the symref, but the target of a symref, would be a good idea.

That's not correct. The "refname" parameter is EXACTLY the symref itself.

Yeah, but the story is the same. We are not really checking anything about the symref (i.e. the thing in "refname") being funny. We are checking what is in "path" (like "does it exist?") and the "refname" is there only for reporting purposes (i.e. "we have a symbolic ref REFNAME, that points at PATH which is a wrong thing").

Quoted 20 lines
Quoted 6 lines
Quoted 6 lines
refs/files-backend.c +6 -0
+{
+	struct stat st;
+	int ret = 0;
+
+	if (lstat(path->buf, &st) < 0) {
+		ret = fsck_refs_report(o, refname->buf,
Quoted 9 lines

refs/files-backend.c static int files_fsck_refs_name(struct fsck_refs_options *o, · line 3451

Quoted 4 lines
refs/files-backend.c +4 -0
+				       FSCK_MSG_DANGLING_SYMREF,
+				       "point to non-existent ref");
+		goto out;
+	}

Is that an error? Just like being on an unborn branch is not an error, it could be argued that a symref that points at a branch yet to be born wouldn't be an error, either, no?

The reason why I choose "danglingSymref" and warn severity is that I let the code be align with "git checkout". When we use "git checkout" for a dangling symref. It would produce the following output:

Quoted 5 lines
  $ git checkout branch-3
  warning: ignoring dangling symref refs/heads/branch-3
  error: pathspec 'branch-3' did not match any file(s) known to git

So I prefer to warn severity.

If you do this from that situation,

    $ git branch branch-3 master

what happens is that the pointee of branch-3 is created at the commit pointed at by 'master'. No error. No warnings.

In a freshly created respository, HEAD is a dangling symbolic ref, and that is not an error. You can create a root commit from there just fine.

If there is anything that needs improvement in your example, it is that "checkout branch-3" should be taught to either (1) not warn about dangling symbolic link and just give the error, which is in line with how "git checkout HEAD" in a freshly created repository behaves, or (2) just like unborn 'master' pointed at by 'HEAD' is perfectly happy to be checked out, allow the unborn 'branch-3' to be pointed at by 'HEAD', and arrange the first commit (which will be a root commit) you create in that state to be pointed by the ref 'branch-3' points at.

So from all of these reasons, I do not think missing target should be treated as any error worthy event. Not even warning.

On the other hand, the target of the symref in "path" must be checked, even if it does not currently exist, for its validity, the same way an existing ref gets checked (lives inside refs/, passes check-ref-format, etc.).

Quoted 3 lines

I intentionally ignored the "escape" situation. Actually, the path could be either absolute or relative. It may be a little complicated. I will find a way to support this in the next version.

Yes, if this wants to claim to be part of "FSCK", it should catch all the errors the regular runtime would complain about, and "escape" thing is one of the first things that you need to get right. Whatever refs.c:read_ref_internal() for S_ISLNK(st.st_mode) case takes as legit should be considered legit, the "fallback - read through the symlink as if it were a non-symlink" case probably wants to be warned. refs.c:resolve_ref_unsafe(), which is used at the low level from object-name.c:get_oid_basic() via refs.c:repo_dwim_ref(), has further checks to see if a refname that a symref resolves to is valid and the runtime sanity relies on these checks.

Thanks.

author
#28 lore raw

Re: [GSoC][PATCH v2 7/7] fsck: add ref content check for files backend

Junio C Hamano <[email protected]> writes:

Quoted 18 lines
Quoted 15 lines · shejialuo
Quoted 14 lines

refs/files-backend.c static int files_fsck_refs_name(struct fsck_refs_options *o, · line 3441

Quoted 3 lines
refs/files-backend.c +3 -0
+static int files_fsck_symref(struct fsck_refs_options *o,
+                       struct strbuf *refname,
+                       struct strbuf *path)

This does not take things like HEAD or refs/remotes/origin/HEAD to validate. Instead, the caller is responsible for either doing a readlink on a symbolic link, or reading a textual symref and stripping "ref: " prefix from it, before calling this function. The "refname" parameter is not HEAD or refs/remotes/origin/HEAD but the pointee of the symref.

So I'd imagine that renaming it to fsck_symref_target or along that line to clarify that we are not checking the symref, but the target of a symref, would be a good idea.

Quoted 9 lines
Quoted 2 lines

That's not correct. The "refname" parameter is EXACTLY the symref itself.

Yeah, but the story is the same. We are not really checking anything about the symref (i.e. the thing in "refname") being funny. We are checking what is in "path" (like "does it exist?") and the "refname" is there only for reporting purposes (i.e. "we have a symbolic ref REFNAME, that points at PATH which is a wrong thing").

Yes, I agree, will rename the function in the next version.

Quoted 28 lines
Quoted 20 lines
Quoted 6 lines
Quoted 6 lines
refs/files-backend.c +6 -0
+{
+  struct stat st;
+  int ret = 0;
+
+  if (lstat(path->buf, &st) < 0) {
+          ret = fsck_refs_report(o, refname->buf,
Quoted 9 lines

refs/files-backend.c static int files_fsck_refs_name(struct fsck_refs_options *o, · line 3451

Quoted 4 lines
refs/files-backend.c +4 -0
+                                 FSCK_MSG_DANGLING_SYMREF,
+                                 "point to non-existent ref");
+          goto out;
+  }

Is that an error? Just like being on an unborn branch is not an error, it could be argued that a symref that points at a branch yet to be born wouldn't be an error, either, no?

The reason why I choose "danglingSymref" and warn severity is that I let the code be align with "git checkout". When we use "git checkout" for a dangling symref. It would produce the following output:

Quoted 5 lines
  $ git checkout branch-3
  warning: ignoring dangling symref refs/heads/branch-3
  error: pathspec 'branch-3' did not match any file(s) known to git

So I prefer to warn severity.

If you do this from that situation,

Quoted 9 lines
    $ git branch branch-3 master

what happens is that the pointee of branch-3 is created at the commit pointed at by 'master'. No error. No warnings.

In a freshly created respository, HEAD is a dangling symbolic ref, and that is not an error. You can create a root commit from there just fine.

I am totally shocked by the fact that it will create the pointee file. I am a little curious about the design here. I know when creating a new repo, HEAD is a dangling symbolic ref. When we do first commit, it will create the pointee file. But HEAD is a special file, it should be treated differently. Why we treat the other symrefs the same way.

Quoted 10 lines

If there is anything that needs improvement in your example, it is that "checkout branch-3" should be taught to either (1) not warn about dangling symbolic link and just give the error, which is in line with how "git checkout HEAD" in a freshly created repository behaves, or (2) just like unborn 'master' pointed at by 'HEAD' is perfectly happy to be checked out, allow the unborn 'branch-3' to be pointed at by 'HEAD', and arrange the first commit (which will be a root commit) you create in that state to be pointed by the ref 'branch-3' points at.

Should we really improve this? From my perspective, a user will get more information when this warn message shows up. The "error: pathspec 'branch-3' did not match any file(s) known to git" is less informative.

Maybe the pro git users do not use git command to write to the git file system. However, this is a bad way to do this. We always want to the user to use the command to handle with git file system.

Instead, we should warn about the user when using

  $ git branch branch-3 master

It will create a new pointee here, I suppose we should warn the user here except HEAD ref. The user could get the dangling symref by deleting the pointee ref. For example:

  $ git symbolic-ref refs/heads/main master
  $ git checkout -b master-copy
  $ git branch --delete master

Should we warn this for user in git-fsck(1)?

Quoted 3 lines

So from all of these reasons, I do not think missing target should be treated as any error worthy event. Not even warning.

Yes, if git-branch implicitly creates the pointee ref file. We should not handle such case. But I think giving a warning here except HEAD ref is worthful.

Quoted 5 lines

On the other hand, the target of the symref in "path" must be checked, even if it does not currently exist, for its validity, the same way an existing ref gets checked (lives inside refs/, passes check-ref-format, etc.).

Yes!

Quoted 4 lines
Quoted 3 lines

I intentionally ignored the "escape" situation. Actually, the path could be either absolute or relative. It may be a little complicated. I will find a way to support this in the next version.

Quoted 11 lines

Yes, if this wants to claim to be part of "FSCK", it should catch all the errors the regular runtime would complain about, and "escape" thing is one of the first things that you need to get right. Whatever refs.c:read_ref_internal() for S_ISLNK(st.st_mode) case takes as legit should be considered legit, the "fallback - read through the symlink as if it were a non-symlink" case probably wants to be warned. refs.c:resolve_ref_unsafe(), which is used at the low level from object-name.c:get_oid_basic() via refs.c:repo_dwim_ref(), has further checks to see if a refname that a symref resolves to is valid and the runtime sanity relies on these checks.

Thanks for your help! I will look at these codes to implement the "escape" check.

Quoted 2 lines

Thanks.

#29 lore raw

Re: [GSoC][PATCH v2 7/7] fsck: add ref content check for files backend

Quoted 6 lines · jialuo she

Instead, we should warn about the user when using

  $ git branch branch-3 master

It will create a new pointee here, I suppose we should warn the user here except HEAD ref.

The user created branch-3 as a symlink, fully expecting that the pointee will be updated.

By the way, a seemingly normal branch looking symref is very handy and that is what my "maint" branch looks like. Right now it is pointing at maint-2.45 (because 2.45.0 was the last feature release)

        $ git symbolic-ref refs/heads/maint
        refs/heads/maint-2.45

and I would imagine many other users are using the trick to maintain multiple maintenance tracks. They would be upset if their next "Now 2.46.0 release is done, let's start the 2.46 maintenance track" mantra, which would look like so:

	$ git symbolic-ref refs/heads/maint refs/heads/maint-2.46
	$ git branch maint v2.46.0

starts to give an unnecessary warning.

author
#31 lore raw

v3 cover ref consistency check infra setup

In reply to shejialuo

This is the third version. Mainly focuses on the following problems:

1. Add "escape" check for symrefs and symbolic links.
2. Resolve the reviews.
3. Fix some errors in version 2.

shejialuo (7):
fsck: add refs check interfaces to interface with fsck error levels
refs: set up ref consistency check infrastructure builtin/refs: add verify subcommand
builtin/fsck: add git-refs verify child process
files-backend: add unified interface for refs scanning
fsck: add ref name check for files backend
fsck: add ref content check for files backend

 Documentation/fsck-msgids.txt |  12 ++
 Documentation/git-refs.txt    |  13 ++
 builtin/fsck.c                |  17 +++
 builtin/refs.c                |  45 ++++++
 fsck.c                        |  69 +++++++++
 fsck.h                        |  49 ++++++-
 refs.c                        |   7 +-
 refs.h                        |   8 ++
 refs/debug.c                  |  11 ++
 refs/files-backend.c          | 255 +++++++++++++++++++++++++++++++++-
 refs/packed-backend.c         |   8 ++
 refs/refs-internal.h          |  11 +-
 refs/reftable-backend.c       |   8 ++
 t/t0602-reffiles-fsck.sh      | 211 ++++++++++++++++++++++++++++
 14 files changed, 716 insertions(+), 8 deletions(-)
 create mode 100755 t/t0602-reffiles-fsck.sh

Range-diff against v2:
1: a65de15dc7 ! 1: b8d20c4be9 fsck: add refs check interfaces to interface with fsck error levels

    @@ fsck.h: int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o,
     +struct fsck_refs_options {
     +	enum fsck_msg_type *msg_type;
     +	unsigned strict:1;
    ++
     +	fsck_refs_error error_func;
     +	unsigned verbose:1;
     +};
    @@ fsck.h: int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o,
     +}
     +
      struct fsck_options {
    ++	/*
    ++	 * Reorder the fields to allow `fsck_ref_options` to use
    ++	 * the interfaces using `struct fsck_options`.
    ++	 */
     +	enum fsck_msg_type *msg_type;
     +	unsigned strict:1;
    ++
      	fsck_walk_func walk;
      	fsck_error error_func;
     -	unsigned strict:1;
2:  a87149069a ! 2:  c7ed95d6b4 refs: set up ref consistency check infrastructure
    @@ refs/debug.c: static int debug_reflog_expire(struct ref_store *ref_store, const
     +static int debug_fsck(struct ref_store *ref_store,
     +		      struct fsck_refs_options *o)
     +{
    -+	trace_printf_key(&trace_refs, "fsck\n");
    -+	return 0;
    ++	struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store;
    ++	int res = drefs->refs->be->fsck(drefs->refs, o);
    ++	trace_printf_key(&trace_refs, "fsck: %d\n", res);
    ++	return res;
     +}
     +
      struct ref_storage_be refs_be_debug = {

3: e1f1b77527 ! 3: 373f470fdc builtin/refs: add verify subcommand
@@ Documentation/git-refs.txt: include::ref-storage-format.txt[]
can be used to double check that the migration works as expected before performing the actual migration.

++The following options are specific to 'git refs verify':
++
+--strict::
+ Enable more strict checking, every WARN severity for the Fsck Messages
+ be seen as ERROR.

4:  2b0ee79e78 = 4:  4f8ebea573 builtin/fsck: add `git-refs verify` child process
5:  6d23eaa1d1 ! 5:  6062b3b453 files-backend: add unified interface for refs scanning
    @@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
      		      struct fsck_refs_options *o)
      {
     @@ refs/files-backend.c: static int files_fsck(struct ref_store *ref_store,
    + 	struct files_ref_store *refs =
      		files_downcast(ref_store, REF_STORE_READ, "fsck");
      
    - 	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o);
    -+	ret = files_fsck_refs(ref_store, o);
    +-	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o);
    ++	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o)
    ++	    | files_fsck_refs(ref_store, o);
     +
      	return ret;
      }
6:  8c55ecdd88 = 6:  87a22f4c7f fsck: add ref name check for files backend
7:  aba5f50940 ! 7:  b945f61ef1 fsck: add ref content check for files backend
    @@ Commit message
         in the files backend. The new functionality ensures that symrefs, real
         symbolic link and regular refs are validated correctly.
     
    -    Add a new function "files_fsck_symref" to check whether the symrefs and
    -    symbolic link points to a valid object and a new function
    -    "files_fsck_refs_content" handles both regular refs and symbolic refs.
    +    In order to check the trailing content of the regular refs, add a new
    +    parameter `trailing` to `parse_loose_ref_contents`.
     
    -    In order to check the trailing content, add a new parameter
    -    "trailing" to "parse_loose_ref_contents" function.
    +    For symrefs, `parse_loose_ref_contents` will set the "referent".
    +    However, symbolic link could be either absolute or relative. Use
    +    "strbuf_add_real_path" to read the symbolic link and convert the
    +    relative path to absolute path. Then use "skip_prefix" to make it align
    +    with symref "referent".
    +
    +    Thus, the symrefs and symbolic links could share the same interface. Add
    +    a new function "files_fsck_symref_target" which aims at checking the
    +    following things:
    +
    +    1. whether the pointee is under the `refs/` directory.
    +    2. whether the pointee name is correct.
    +    3. whether the pointee path is a wrong type in filesystem.
     
         Last, add the following FSCK MESSAGEs:
     
    -    1. "badRefSha(ERROR)": A ref has a bad sha.
    -    2. "danglingSymre(WARN)": Found a dangling symref.
    +    1. "badRefContent(ERROR)": A ref has a bad content
    +    2. "badSymrefPointee(ERROR)": The pointee of a symref is bad.
         3. "trailingRefContent(WARN)": A ref content has trailing contents.
     
         Mentored-by: Patrick Steinhardt <[email protected]>
    @@ Commit message
     
      ## Documentation/fsck-msgids.txt ##
     @@
    + `badParentSha1`::
    + 	(ERROR) A commit object has a bad parent sha1.
    + 
    ++`badRefContent`::
    ++	(ERROR) A ref has a bad content.
    ++
      `badRefName`::
      	(ERROR) A ref has a bad name.
      
    -+`badRefSha`::
    -+	(ERROR) A ref has a bad sha.
    ++`badSymrefPointee`::
    ++	(ERROR) The pointee of a symref is bad.
     +
      `badTagName`::
      	(INFO) A tag has an invalid format.
      
    -@@
    - `badType`::
    - 	(ERROR) Found an invalid object type.
    - 
    -+`danglingSymref`::
    -+	(WARN) Found a dangling symref.
    -+
    - `duplicateEntries`::
    - 	(ERROR) A tree contains duplicate file entries.
    - 
     @@
      `symlinkTargetMissing`::
      	(ERROR) Unable to read symbolic link target's blob.
    @@ fsck.h: enum fsck_msg_type {
      	FUNC(BAD_OBJECT_SHA1, ERROR) \
      	FUNC(BAD_PARENT_SHA1, ERROR) \
      	FUNC(BAD_REF_NAME, ERROR) \
    -+	FUNC(BAD_REF_SHA, ERROR) \
    ++	FUNC(BAD_REF_CONTENT, ERROR) \
    ++	FUNC(BAD_SYMREF_POINTEE, ERROR) \
      	FUNC(BAD_TIMEZONE, ERROR) \
      	FUNC(BAD_TREE, ERROR) \
      	FUNC(BAD_TREE_SHA1, ERROR) \
     @@ fsck.h: enum fsck_msg_type {
    - 	FUNC(SYMLINK_TARGET_BLOB, ERROR) \
    - 	/* warnings */ \
    - 	FUNC(EMPTY_NAME, WARN) \
    -+	FUNC(DANGLING_SYMREF, WARN) \
    - 	FUNC(FULL_PATHNAME, WARN) \
    - 	FUNC(HAS_DOT, WARN) \
      	FUNC(HAS_DOTDOT, WARN) \
      	FUNC(HAS_DOTGIT, WARN) \
      	FUNC(NULL_SHA1, WARN) \
    @@ refs.c: static int refs_read_special_head(struct ref_store *ref_store,
      	strbuf_release(&full_path);
     
      ## refs/files-backend.c ##
    +@@
    + #include "../git-compat-util.h"
    ++#include "../abspath.h"
    + #include "../copy.h"
    + #include "../environment.h"
    + #include "../gettext.h"
     @@ refs/files-backend.c: static int read_ref_internal(struct ref_store *ref_store, const char *refname,
      	strbuf_rtrim(&sb_contents);
      	buf = sb_contents.buf;
    @@ refs/files-backend.c: static int files_read_symbolic_ref(struct ref_store *ref_s
      int parse_loose_ref_contents(const char *buf, struct object_id *oid,
      			     struct strbuf *referent, unsigned int *type,
     -			     int *failure_errno)
    -+			     int *failure_errno, unsigned int *trailing)
    ++			     int *failure_errno, const char **trailing)
      {
      	const char *p;
      	if (skip_prefix(buf, "ref:", &buf)) {
    @@ refs/files-backend.c: int parse_loose_ref_contents(const char *buf, struct objec
      		return -1;
      	}
     +
    -+	if (trailing && (*p != '\0' && *p != '\n'))
    -+		*trailing = 1;
    ++	if (trailing)
    ++		*trailing = p;
     +
      	return 0;
      }
    @@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
      	return ret;
      }
      
    -+static int files_fsck_symref(struct fsck_refs_options *o,
    -+			     struct strbuf *refname,
    -+			     struct strbuf *path)
    ++/*
    ++ * Check the symref "pointee_name" and "pointee_path". The caller should
    ++ * make sure that "pointee_path" is absolute. For symbolic ref, "pointee_name"
    ++ * would be the content after "refs:". For symblic link, "pointee_name" would
    ++ * be the relative path agaignst "gitdir".
    ++ */
    ++static int files_fsck_symref_target(struct fsck_refs_options *o,
    ++				    const char *refname,
    ++				    const char *pointee_name,
    ++				    const char *pointee_path)
     +{
    ++	const char *p = NULL;
     +	struct stat st;
     +	int ret = 0;
     +
    -+	if (lstat(path->buf, &st) < 0) {
    -+		ret = fsck_refs_report(o, refname->buf,
    -+				       FSCK_MSG_DANGLING_SYMREF,
    -+				       "point to non-existent ref");
    ++	if (!skip_prefix(pointee_name, "refs/", &p)) {
    ++
    ++		ret = fsck_refs_report(o, refname,
    ++				       FSCK_MSG_BAD_SYMREF_POINTEE,
    ++				       "point to target out of refs hierarchy");
     +		goto out;
     +	}
     +
    ++	if (check_refname_format(pointee_name, 0)) {
    ++		ret = fsck_refs_report(o, refname,
    ++				       FSCK_MSG_BAD_SYMREF_POINTEE,
    ++				       "point to invalid refname");
    ++	}
    ++
    ++	if (lstat(pointee_path, &st) < 0)
    ++		goto out;
    ++
     +	if (!S_ISREG(st.st_mode) && !S_ISLNK(st.st_mode)) {
    -+		ret = fsck_refs_report(o, refname->buf,
    -+				       FSCK_MSG_DANGLING_SYMREF,
    -+				       "point to invalid object");
    ++		ret = fsck_refs_report(o, refname,
    ++				       FSCK_MSG_BAD_SYMREF_POINTEE,
    ++				       "point to invalid target");
     +		goto out;
     +	}
     +out:
    @@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
     +				   const char *refs_check_dir,
     +				   struct dir_iterator *iter)
     +{
    -+	struct strbuf path = STRBUF_INIT,
    -+		      refname = STRBUF_INIT,
    ++	struct strbuf pointee_path = STRBUF_INIT,
     +		      ref_content = STRBUF_INIT,
    -+		      referent = STRBUF_INIT;
    -+	unsigned int trailing = 0;
    ++		      abs_gitdir = STRBUF_INIT,
    ++		      referent = STRBUF_INIT,
    ++		      refname = STRBUF_INIT;
    ++	const char *trailing = NULL;
     +	int failure_errno = 0;
     +	unsigned int type = 0;
     +	struct object_id oid;
     +	int ret = 0;
     +
    -+	strbuf_addbuf(&path, &iter->path);
     +	strbuf_addf(&refname, "%s/%s", refs_check_dir, iter->relative_path);
     +
     +	/*
    @@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
     +	 * of the destination object.
     +	 */
     +	if (S_ISLNK(iter->st.st_mode)) {
    -+		strbuf_strip_file_from_path(&path);
    -+		ret = strbuf_readlink(&ref_content,
    -+				      iter->path.buf, iter->st.st_size);
    -+		if (ret < 0) {
    -+			ret = error_errno(_("could not read link '%s'"),
    -+					  iter->path.buf);
    ++		const char *pointee_name = NULL;
    ++
    ++		strbuf_add_real_path(&pointee_path, iter->path.buf);
    ++
    ++		strbuf_add_absolute_path(&abs_gitdir, gitdir);
    ++		strbuf_normalize_path(&abs_gitdir);
    ++		if (!is_dir_sep(abs_gitdir.buf[abs_gitdir.len - 1]))
    ++			strbuf_addch(&abs_gitdir, '/');
    ++
    ++		if (!skip_prefix(pointee_path.buf,
    ++				 abs_gitdir.buf, &pointee_name)) {
    ++			ret = fsck_refs_report(o, refname.buf,
    ++					       FSCK_MSG_BAD_SYMREF_POINTEE,
    ++					       "point to target outside gitdir");
     +			goto clean;
     +		}
    -+		strbuf_addbuf(&path, &ref_content);
    -+		strbuf_reset(&ref_content);
     +
    -+		ret = files_fsck_symref(o, &refname, &path);
    ++		ret = files_fsck_symref_target(o, refname.buf, pointee_name,
    ++					       pointee_path.buf);
     +		goto clean;
     +	}
     +
    -+	if (strbuf_read_file(&ref_content, path.buf, 0) < 0) {
    ++	if (strbuf_read_file(&ref_content, iter->path.buf, 0) < 0) {
     +		ret = error_errno(_("%s/%s: unable to read the ref"),
     +				  refs_check_dir, iter->relative_path);
     +		goto clean;
    @@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
     +				     &referent, &type,
     +				     &failure_errno, &trailing)) {
     +		ret = fsck_refs_report(o, refname.buf,
    -+				       FSCK_MSG_BAD_REF_SHA,
    ++				       FSCK_MSG_BAD_REF_CONTENT,
     +				       "invalid ref content");
     +		goto clean;
     +	}
    @@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
     +	 * connectivity.
     +	 */
     +	if (referent.len && (type & REF_ISSYMREF)) {
    -+		strbuf_reset(&path);
    -+		strbuf_addf(&path, "%s/%s", gitdir, referent.buf);
    -+
    -+		if (check_refname_format(referent.buf, 0)) {
    -+			ret = fsck_refs_report(o, refname.buf,
    -+					       FSCK_MSG_DANGLING_SYMREF,
    -+					       "point to invalid refname");
    -+			goto clean;
    -+		}
    ++		strbuf_addf(&pointee_path, "%s/%s", gitdir, referent.buf);
    ++		strbuf_rtrim(&referent);
     +
    -+		ret = files_fsck_symref(o, &refname, &path);
    ++		ret = files_fsck_symref_target(o, refname.buf, referent.buf,
    ++					       pointee_path.buf);
     +		goto clean;
     +	} else {
     +		/*
     +		 * Only regular refs could have a trailing garbage. Should
     +		 * be reported as a warning.
     +		 */
    -+		if (trailing) {
    ++		if (trailing && (*trailing != '\0' && *trailing != '\n')) {
     +			ret = fsck_refs_report(o, refname.buf,
     +					       FSCK_MSG_TRAILING_REF_CONTENT,
     +					       "trailing garbage in ref");
    @@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
     +	}
     +
     +clean:
    -+	strbuf_release(&path);
    ++	strbuf_release(&abs_gitdir);
    ++	strbuf_release(&pointee_path);
     +	strbuf_release(&refname);
     +	strbuf_release(&ref_content);
     +	strbuf_release(&referent);
    @@ refs/refs-internal.h: struct ref_store {
      /*
       * Parse contents of a loose ref file. *failure_errno maybe be set to EINVAL for
     - * invalid contents.
    -+ * invalid contents. Also trailing is set to 1 when there is any bytes after the
    -+ * hex.
    ++ * invalid contents. Also *trailing is set to the first character after the
    ++ * refname or NULL if the referent is not empty.
       */
      int parse_loose_ref_contents(const char *buf, struct object_id *oid,
      			     struct strbuf *referent, unsigned int *type,
     -			     int *failure_errno);
    -+			     int *failure_errno, unsigned int *trailing);
    ++			     int *failure_errno, const char **trailing);
      
      /*
       * Fill in the generic part of refs and add it to our collection of
    @@ t/t0602-reffiles-fsck.sh: test_expect_success 'ref name check should be adapted
     +		printf "xfsazqfxcadas" > $tag_dir_prefix/tag-2-bad &&
     +		test_must_fail git refs verify 2>err &&
     +		cat >expect <<-EOF &&
    -+		error: refs/tags/tag-2-bad: badRefSha: invalid ref content
    ++		error: refs/tags/tag-2-bad: badRefContent: invalid ref content
     +		EOF
     +		rm $tag_dir_prefix/tag-2-bad &&
     +		test_cmp expect err
    @@ t/t0602-reffiles-fsck.sh: test_expect_success 'ref name check should be adapted
     +		printf "xfsazqfxcadas" > $branch_dir_prefix/a/b/branch-2-bad &&
     +		test_must_fail git refs verify 2>err &&
     +		cat >expect <<-EOF &&
    -+		error: refs/heads/a/b/branch-2-bad: badRefSha: invalid ref content
    ++		error: refs/heads/a/b/branch-2-bad: badRefContent: invalid ref content
     +		EOF
     +		rm $branch_dir_prefix/a/b/branch-2-bad &&
     +		test_cmp expect err
    @@ t/t0602-reffiles-fsck.sh: test_expect_success 'ref name check should be adapted
     +	) &&
     +	(
     +		cd repo &&
    -+		printf "ref: refs/heads/branch-3" > $branch_dir_prefix/branch-2-bad &&
    -+		git refs verify 2>err &&
    ++		printf "ref: refs/heads/.branch" > $branch_dir_prefix/branch-2-bad &&
    ++		test_must_fail git refs verify 2>err &&
     +		cat >expect <<-EOF &&
    -+		warning: refs/heads/branch-2-bad: danglingSymref: point to non-existent ref
    ++		error: refs/heads/branch-2-bad: badSymrefPointee: point to invalid refname
     +		EOF
     +		rm $branch_dir_prefix/branch-2-bad &&
     +		test_cmp expect err
     +	) &&
     +	(
     +		cd repo &&
    -+		printf "ref: refs/heads/.branch" > $branch_dir_prefix/branch-2-bad &&
    -+		git refs verify 2>err &&
    ++		printf "ref: refs/heads" > $branch_dir_prefix/branch-2-bad &&
    ++		test_must_fail git refs verify 2>err &&
     +		cat >expect <<-EOF &&
    -+		warning: refs/heads/branch-2-bad: danglingSymref: point to invalid refname
    ++		error: refs/heads/branch-2-bad: badSymrefPointee: point to invalid target
     +		EOF
     +		rm $branch_dir_prefix/branch-2-bad &&
     +		test_cmp expect err
     +	) &&
     +	(
     +		cd repo &&
    -+		printf "ref: refs/heads" > $branch_dir_prefix/branch-2-bad &&
    -+		git refs verify 2>err &&
    ++		printf "ref: logs/maint-v2.45" > $branch_dir_prefix/branch-2-bad &&
    ++		test_must_fail git refs verify 2>err &&
     +		cat >expect <<-EOF &&
    -+		warning: refs/heads/branch-2-bad: danglingSymref: point to invalid object
    ++		error: refs/heads/branch-2-bad: badSymrefPointee: point to target out of refs hierarchy
     +		EOF
     +		rm $branch_dir_prefix/branch-2-bad &&
     +		test_cmp expect err
Signature · 1 line
2.45.2
#32 lore raw
Quoted 6 lines · shejialuo

This is the third version. Mainly focuses on the following problems:

1. Add "escape" check for symrefs and symbolic links.
2. Resolve the reviews.
3. Fix some errors in version 2.

Quoted 8 lines

shejialuo (7):
fsck: add refs check interfaces to interface with fsck error levels
refs: set up ref consistency check infrastructure builtin/refs: add verify subcommand
builtin/fsck: add git-refs verify child process
files-backend: add unified interface for refs scanning
fsck: add ref name check for files backend
fsck: add ref content check for files backend

What commit is this series based on? It does not seem to apply cleanly on top of the same base as I queued an earlier iteration of this series (f931bfb2 (Merge branch 'ps/ref-storage-migration' into sj/ref-fsck, 2024-06-13)), or the current tip of 'master' that already merges that prerequisite topic. The early 6 applies on top of 'master' fine, but the last one. I wiggled the last one in (the patch to the fsck-msgids.txt documentation expects stale entries that were already removed in 2.45.2), but if you are basing your patches on an unusual commit, please describe it clearly here in the cover letter so that others can locate (or recreate as needed) it.

author
#33 lore raw
Quoted 18 lines · Junio C Hamano
Quoted 6 lines · shejialuo

This is the third version. Mainly focuses on the following problems:

1. Add "escape" check for symrefs and symbolic links.
2. Resolve the reviews.
3. Fix some errors in version 2.

Quoted 8 lines

shejialuo (7):
fsck: add refs check interfaces to interface with fsck error levels
refs: set up ref consistency check infrastructure builtin/refs: add verify subcommand
builtin/fsck: add git-refs verify child process
files-backend: add unified interface for refs scanning
fsck: add ref name check for files backend
fsck: add ref content check for files backend

Quoted 10 lines

What commit is this series based on? It does not seem to apply cleanly on top of the same base as I queued an earlier iteration of this series (f931bfb2 (Merge branch 'ps/ref-storage-migration' into sj/ref-fsck, 2024-06-13)), or the current tip of 'master' that already merges that prerequisite topic. The early 6 applies on top of 'master' fine, but the last one. I wiggled the last one in (the patch to the fsck-msgids.txt documentation expects stale entries that were already removed in 2.45.2), but if you are basing your patches on an unusual commit, please describe it clearly here in the cover letter so that others can locate (or recreate as needed) it.

Well, I simply does "git checkout -b js-refs-fsck-v3" for the current branch "js-refs-fsck". And I use "git rebase" to change the code in this branch. So the base commit should be unchanged. However, maybe Patrick has changed the code and I forgot to update upstream.

And I have set up the base using the following commands:

  $ git remote add junio https://github.com/gister/git.git
  $ git fetch junio
  $ git branch js-refs-fsck junio/ps/ref-storage-migration

I am sorry for the confusion. Next version, I will merge the master code and send the patch. I am sorry to make you so inconvenient.

author
#34 lore raw

PATCH v3 1/7 fsck: add refs check interfaces to interface with fsck error levels

In reply to shejialuo

The git-fsck(1) focuses on object database consistency check. It relies on the "fsck_options" to interact with fsck error levels. However, "fsck_options" aims at checking the object database which makes it unsuitable to change the semantics of it. Instead, create "fsck_refs_options" structure to handle refs consistency check.

The "git_fsck_config" sets up the "msg_type" and "skiplist" member of the "fsck_options". For refs, we just need the "msg_type". In order to allow setting up more refs-specific options easily later, add a separate function "git_fsck_refs_config" to initialize the refs-specific options.

Move the "msg_type" and "strict" member to the top of the "fsck_options" which allows us to convert "fsck_refs_options *" to "fsck_options *" to reuse the interfaces provided by "fsck.h" without changing the original code.

The static function "report" provided by "fsck.c" aims at reporting the problems related to object database which cannot be reused for refs. Provide "fsck_refs_report" function to integrate the fsck error levels into reference consistency check.

---
 fsck.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 fsck.h | 45 ++++++++++++++++++++++++++++++++++++--
 2 files changed, 112 insertions(+), 2 deletions(-)
fsck.c +69 -0
index 8ef962199f..13528c646e 100644
@@ -1249,6 +1249,20 @@ int fsck_buffer(const struct object_id *oid, enum object_type type,
 		      type);
 }
 
+int fsck_refs_error_function(struct fsck_refs_options *o UNUSED,
+			     const char *name,
+			     enum fsck_msg_type msg_type,
+			     enum fsck_msg_id msg_id UNUSED,
+			     const char *message)
+{
+	if (msg_type == FSCK_WARN) {
+		warning("%s: %s", name, message);
+		return 0;
+	}
+	error("%s: %s", name, message);
+	return 1;
+}
+
 int fsck_error_function(struct fsck_options *o,
 			const struct object_id *oid,
 			enum object_type object_type UNUSED,
@@ -1323,6 +1337,61 @@ int fsck_finish(struct fsck_options *options)
 	return ret;
 }
 
+int fsck_refs_report(struct fsck_refs_options *o,
+		     const char *name,
+		     enum fsck_msg_id msg_id,
+		     const char *fmt, ...)
+{
+	va_list ap;
+	struct strbuf sb = STRBUF_INIT;
+	enum fsck_msg_type msg_type =
+		fsck_msg_type(msg_id, (struct fsck_options*)o);
+	int ret = 0;
+
+	if (msg_type == FSCK_IGNORE)
+		return 0;
+
+	if (msg_type == FSCK_FATAL)
+		msg_type = FSCK_ERROR;
+	else if (msg_type == FSCK_INFO)
+		msg_type = FSCK_WARN;
+
+	prepare_msg_ids();
+	strbuf_addf(&sb, "%s: ", msg_id_info[msg_id].camelcased);
+
+	va_start(ap, fmt);
+	strbuf_vaddf(&sb, fmt, ap);
+	ret = o->error_func(o, name, msg_type, msg_id, sb.buf);
+	strbuf_release(&sb);
+	va_end(ap);
+
+	return ret;
+}
+
+int git_fsck_refs_config(const char *var, const char *value,
+			 const struct config_context *ctx, void *cb)
+{
+	struct fsck_refs_options *options = cb;
+	const char *msg_id;
+
+	/*
+	 * We don't check the value of fsck.skiplist here, because it
+	 * is specific to object database, not reference database.
+	 */
+	if (strcmp(var, "fsck.skiplist") == 0) {
+		return 0;
+	}
+
+	if (skip_prefix(var, "fsck.", &msg_id)) {
+		if (!value)
+			return config_error_nonbool(var);
+		fsck_set_msg_type((struct fsck_options*)options, msg_id, value);
+		return 0;
+	}
+
+	return git_default_config(var, value, ctx, cb);
+}
+
 int git_fsck_config(const char *var, const char *value,
 		    const struct config_context *ctx, void *cb)
 {
fsck.h +43 -2

78 lines not on this page · raw message

Signature · 1 line
2.45.2
#35 lore raw

Re: [GSoC][PATCH v3 1/7] fsck: add refs check interfaces to interface with fsck error levels

shejialuo <[email protected]> writes:

[snip]

fsck.h int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o, · line 170

Quoted 5 lines

struct fsck_options {

fsck.h +4 -0
+	/*
+	 * Reorder the fields to allow `fsck_ref_options` to use
+	 * the interfaces using `struct fsck_options`.
+	 */

Why is this added? It makes sense to have it in the commit message because it talks about the change, but why make it persistent in the code?

[snip]

author
#36 lore raw

Re: [GSoC][PATCH v3 1/7] fsck: add refs check interfaces to interface with fsck error levels

Quoted 4 lines · Karthik Nayak

shejialuo <[email protected]> writes:

[snip]

Quoted 10 lines

fsck.h int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o, · line 170

Quoted 5 lines

struct fsck_options {

fsck.h +4 -0
+	/*
+	 * Reorder the fields to allow `fsck_ref_options` to use
+	 * the interfaces using `struct fsck_options`.
+	 */

Why is this added? It makes sense to have it in the commit message because it talks about the change, but why make it persistent in the code?

I explicitly add this comments due to the following reasons:

1. If someone needs to change the fsck_options, without this comment, he might be just add some new fields at top. Although the change will fail the tests here, I think we should mention this in code.
2. My later intention is that we should extract these common fields out of the fsck_options and fsck_ref_options.

Quoted 1 line

[snip]

#37 lore raw

Re: [GSoC][PATCH v3 1/7] fsck: add refs check interfaces to interface with fsck error levels

Quoted 16 lines · shejialuo
Quoted 4 lines · Karthik Nayak

shejialuo <[email protected]> writes:

[snip]

Quoted 10 lines

fsck.h int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o, · line 170

Quoted 5 lines

struct fsck_options {

fsck.h +4 -0
+	/*
+	 * Reorder the fields to allow `fsck_ref_options` to use
+	 * the interfaces using `struct fsck_options`.
+	 */

Why is this added? It makes sense to have it in the commit message because it talks about the change, but why make it persistent in the code?

Quoted 5 lines

I explicitly add this comments due to the following reasons:

1. If someone needs to change the fsck_options, without this comment, he might be just add some new fields at top. Although the change will fail the tests here, I think we should mention this in code.

Do you mean you plan to take advantage of the fact that early members of two structures are the same? IOW, if there is a function that takes a pointer to smaller fsck_refs_options, you plan to pass a pointer to fsck_options from some callers, e.g.

    extern void func(struct fsck_refs_options *);
    void a_caller(struct fsck_options *o)
    {
	func((struct fsck_options *)o);
	...

If that is the case, then ...

Do not do that.

Your data structure design is broken. Instead you would do this:

	struct fsck_options {
		struct fsck_refs_options refs;
		... other members ...
	};
	void a_caller(struct fsck_options *o)
	{
		func(&o->refs);
		...
author
#38 lore raw

Re: [GSoC][PATCH v3 1/7] fsck: add refs check interfaces to interface with fsck error levels

Quoted 30 lines · Junio C Hamano
Quoted 16 lines · shejialuo
Quoted 10 lines · Karthik Nayak

shejialuo <[email protected]> writes:

[snip]

fsck.h int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o, · line 170

Quoted 5 lines

struct fsck_options {

fsck.h +4 -0
+  /*
+   * Reorder the fields to allow `fsck_ref_options` to use
+   * the interfaces using `struct fsck_options`.
+   */
Quoted 4 lines

Why is this added? It makes sense to have it in the commit message because it talks about the change, but why make it persistent in the code?

Quoted 5 lines

I explicitly add this comments due to the following reasons:

1. If someone needs to change the fsck_options, without this comment, he might be just add some new fields at top. Although the change will fail the tests here, I think we should mention this in code.

Do you mean you plan to take advantage of the fact that early members of two structures are the same? IOW, if there is a function that takes a pointer to smaller fsck_refs_options, you plan to pass a pointer to fsck_options from some callers, e.g.

Quoted 6 lines
    extern void func(struct fsck_refs_options *);
    void a_caller(struct fsck_options *o)
    {
        func((struct fsck_options *)o);
        ...

I do not want to convert "struct fsck_options*" to "struct fsck_refs_options*". Instead, I want to convert "struct fsck_refs_options*" to "struct fsck_options*" to reuse the functions which use the "struct fsck_options*" parameter.

Like the commit message said:

Move the "msg_type" and "strict" member to the top of the "fsck_options" which allows us to convert "fsck_refs_options *" to "fsck_options *" to reuse the interfaces provided by "fsck.h" without changing the original code.

It may seem we should add some fields into fsck_options, but I don't think it's a good idea. I will elaborate my design here:

The git-fsck(1) is highly relevant with the object database. I don't want to add some new fields into "fsck_options" due to the following reason:

The fields in fsck_options are all related to objects except "fsck_msg_type" and "strict". Adding some ref-related fields into "fsck_options" will break the semantics.

Actually, it may be perfect that I may abstract some interfaces here, however, it would be too complicated for this patch, many functions use "fsck_options *" as the parameter. I think we may do this later.

author
#39 lore raw

Re: [GSoC][PATCH v3 1/7] fsck: add refs check interfaces to interface with fsck error levels

In reply to Junio C Hamano
Quoted 41 lines · Junio C Hamano
Quoted 16 lines · shejialuo
Quoted 4 lines · Karthik Nayak

shejialuo <[email protected]> writes:

[snip]

Quoted 10 lines

fsck.h int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o, · line 170

Quoted 5 lines

struct fsck_options {

fsck.h +4 -0
+	/*
+	 * Reorder the fields to allow `fsck_ref_options` to use
+	 * the interfaces using `struct fsck_options`.
+	 */

Why is this added? It makes sense to have it in the commit message because it talks about the change, but why make it persistent in the code?

Quoted 5 lines

I explicitly add this comments due to the following reasons:

1. If someone needs to change the fsck_options, without this comment, he might be just add some new fields at top. Although the change will fail the tests here, I think we should mention this in code.

Do you mean you plan to take advantage of the fact that early members of two structures are the same? IOW, if there is a function that takes a pointer to smaller fsck_refs_options, you plan to pass a pointer to fsck_options from some callers, e.g.

    extern void func(struct fsck_refs_options *);
    void a_caller(struct fsck_options *o)
    {
	func((struct fsck_options *)o);
	...

If that is the case, then ...

Do not do that.

Your data structure design is broken. Instead you would do this:

Quoted 8 lines
	struct fsck_options {
		struct fsck_refs_options refs;
		... other members ...
	};
	void a_caller(struct fsck_options *o)
	{
		func(&o->refs);
		...

Well, I totally agree with this. It's bad to convert the pointer type. I will change the code in this patch to make it OK.

Thanks for your advice.

author
#40 lore raw

PATCH v3 2/7 refs: set up ref consistency check infrastructure

In reply to shejialuo

The interfaces defined in the ref_storage_be are carefully structured in semantic. It's organized as the five parts:

1. The name and the initialization interfaces.
2. The ref transaction interfaces.
3. The ref internal interfaces (pack, rename and copy).
4. The ref filesystem interfaces.
5. The reflog related interfaces.

To keep consistent with the git-fsck(1), add a new interface named "fsck_refs_fn" to the end of "ref_storage_be". This semantic cannot be grouped into any above five categories. Explicitly add blank line to make it different from others.

---
 refs.c                  |  5 +++++
 refs.h                  |  8 ++++++++
 refs/debug.c            | 11 +++++++++++
 refs/files-backend.c    | 15 ++++++++++++++-
 refs/packed-backend.c   |  8 ++++++++
 refs/refs-internal.h    |  6 ++++++
 refs/reftable-backend.c |  8 ++++++++
 7 files changed, 60 insertions(+), 1 deletion(-)
refs.c +5 -0
index f7c7765d23..0922439275 100644
@@ -316,6 +316,11 @@ int check_refname_format(const char *refname, int flags)
 	return check_or_sanitize_refname(refname, flags, NULL);
 }
 
+int refs_fsck(struct ref_store *refs, struct fsck_refs_options *o)
+{
+	return refs->be->fsck(refs, o);
+}
+
 void sanitize_refname_component(const char *refname, struct strbuf *out)
 {
 	if (check_or_sanitize_refname(refname, REFNAME_ALLOW_ONELEVEL, out))
refs.h +8 -0
index 76d25df4de..5a042695f5 100644
@@ -3,6 +3,7 @@
 
 #include "commit.h"
 
+struct fsck_refs_options;
 struct object_id;
 struct ref_store;
 struct repository;
@@ -547,6 +548,13 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat
  */
 int check_refname_format(const char *refname, int flags);
 
+/*
+ * Check the reference database for consistency. Return 0 if refs and
+ * reflogs are consistent, and non-zero otherwise. The errors will be
+ * written to stderr.
+ */
+int refs_fsck(struct ref_store *refs, struct fsck_refs_options *o);
+
 /*
  * Apply the rules from check_refname_format, but mutate the result until it
  * is acceptable, and place the result in "out".
refs/debug.c +11 -0
index 547d9245b9..6306c25fd3 100644
@@ -419,6 +419,15 @@ static int debug_reflog_expire(struct ref_store *ref_store, const char *refname,
 	return res;
 }
 
+static int debug_fsck(struct ref_store *ref_store,
+		      struct fsck_refs_options *o)
+{
+	struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store;
+	int res = drefs->refs->be->fsck(drefs->refs, o);
+	trace_printf_key(&trace_refs, "fsck: %d\n", res);
+	return res;
+}
+
 struct ref_storage_be refs_be_debug = {
 	.name = "debug",
 	.init = NULL,
@@ -451,4 +460,6 @@ struct ref_storage_be refs_be_debug = {
 	.create_reflog = debug_create_reflog,
 	.delete_reflog = debug_delete_reflog,
 	.reflog_expire = debug_reflog_expire,
+
+	.fsck = debug_fsck,
 };
refs/files-backend.c +14 -1
index cb752d32b6..e965345ad8 100644
@@ -3402,6 +3402,17 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,
 	return ret;
 }
 
+static int files_fsck(struct ref_store *ref_store,
+		      struct fsck_refs_options *o)
+{
+	int ret;
+	struct files_ref_store *refs =
+		files_downcast(ref_store, REF_STORE_READ, "fsck");
+
+	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o);
+	return ret;
+}
+
 struct ref_storage_be refs_be_files = {
 	.name = "files",
 	.init = files_ref_store_init,
@@ -3428,5 +3439,7 @@ struct ref_storage_be refs_be_files = {
 	.reflog_exists = files_reflog_exists,
 	.create_reflog = files_create_reflog,
 	.delete_reflog = files_delete_reflog,
-	.reflog_expire = files_reflog_expire
+	.reflog_expire = files_reflog_expire,
+
+	.fsck = files_fsck,
 };
refs/packed-backend.c +8 -0
index c4c1e36aa2..db152053f8 100644
@@ -1733,6 +1733,12 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_s
 	return empty_ref_iterator_begin();
 }
 
+static int packed_fsck(struct ref_store *ref_store,
+		       struct fsck_refs_options *o)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_packed = {
 	.name = "packed",
 	.init = packed_ref_store_init,
@@ -1760,4 +1766,6 @@ struct ref_storage_be refs_be_packed = {
 	.create_reflog = NULL,
 	.delete_reflog = NULL,
 	.reflog_expire = NULL,
+
+	.fsck = packed_fsck,
 };
refs/refs-internal.h +6 -0
index cbcb6f9c36..8f42f21e77 100644
@@ -4,6 +4,7 @@
 #include "refs.h"
 #include "iterator.h"
 
+struct fsck_refs_options;
 struct ref_transaction;
 
 /*
@@ -650,6 +651,9 @@ typedef int read_raw_ref_fn(struct ref_store *ref_store, const char *refname,
 typedef int read_symbolic_ref_fn(struct ref_store *ref_store, const char *refname,
 				 struct strbuf *referent);
 
+typedef int fsck_fn(struct ref_store *ref_store,
+		    struct fsck_refs_options *o);
+
 struct ref_storage_be {
 	const char *name;
 	ref_store_init_fn *init;
@@ -677,6 +681,8 @@ struct ref_storage_be {
 	create_reflog_fn *create_reflog;
 	delete_reflog_fn *delete_reflog;
 	reflog_expire_fn *reflog_expire;
+
+	fsck_fn *fsck;
 };
 
 extern struct ref_storage_be refs_be_files;
refs/reftable-backend.c +8 -0
index e555be4671..7f606faa9e 100644
@@ -2242,6 +2242,12 @@ static int reftable_be_reflog_expire(struct ref_store *ref_store,
 	return ret;
 }
 
+static int reftable_be_fsck(struct ref_store *ref_store,
+			    struct fsck_refs_options *o)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_reftable = {
 	.name = "reftable",
 	.init = reftable_be_init,
@@ -2269,4 +2275,6 @@ struct ref_storage_be refs_be_reftable = {
 	.create_reflog = reftable_be_create_reflog,
 	.delete_reflog = reftable_be_delete_reflog,
 	.reflog_expire = reftable_be_reflog_expire,
+
+	.fsck = reftable_be_fsck,
 };
Signature · 1 line
2.45.2
author
#41 lore raw

PATCH v3 3/7 builtin/refs: add verify subcommand

In reply to shejialuo

Introduce a new subcommand "verify" in git-refs(1) to allow the user to check the reference database consistency.

---
 Documentation/git-refs.txt | 13 +++++++++++
 builtin/refs.c             | 45 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+)
Documentation/git-refs.txt +13 -0
index 5b99e04385..16b518ecce 100644
@@ -10,6 +10,7 @@ SYNOPSIS
 --------
 [verse]
 'git refs migrate' --ref-format=<format> [--dry-run]
+'git refs verify' [--strict] [--verbose]
 
 DESCRIPTION
 -----------
@@ -22,6 +23,9 @@ COMMANDS
 migrate::
 	Migrate ref store between different formats.
 
+verify::
+	Verify reference database consistency.
+
 OPTIONS
 -------
 
@@ -39,6 +43,15 @@ include::ref-storage-format.txt[]
 	can be used to double check that the migration works as expected before
 	performing the actual migration.
 
+The following options are specific to 'git refs verify':
+
+--strict::
+	Enable more strict checking, every WARN severity for the `Fsck Messages`
+	be seen as ERROR.
+
+--verbose::
+	When verifying the reference database consistency, be chatty.
+
 KNOWN LIMITATIONS
 -----------------
 
builtin/refs.c +45 -0

70 lines not on this page · raw message

Signature · 1 line
2.45.2
#42 lore raw

Re: [PATCH v3 3/7] builtin/refs: add verify subcommand

Quoted 34 lines · shejialuo

Introduce a new subcommand "verify" in git-refs(1) to allow the user to check the reference database consistency.

---
 Documentation/git-refs.txt | 13 +++++++++++
 builtin/refs.c             | 45 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+)
Documentation/git-refs.txt +4 -0
index 5b99e04385..16b518ecce 100644
@@ -10,6 +10,7 @@ SYNOPSIS
 --------
 [verse]
 'git refs migrate' --ref-format=<format> [--dry-run]
+'git refs verify' [--strict] [--verbose]
 
 DESCRIPTION
 -----------
@@ -22,6 +23,9 @@ COMMANDS
 migrate::
 	Migrate ref store between different formats.
 
+verify::
+	Verify reference database consistency.
+
 OPTIONS
 -------
 

Documentation/git-refs.txt include::ref-storage-format.txt[] · line 51

Quoted 9 lines
Documentation/git-refs.txt +6 -0
@@ -39,6 +43,15 @@ include::ref-storage-format.txt[]
 	can be used to double check that the migration works as expected before
 	performing the actual migration.
 
+The following options are specific to 'git refs verify':
+
+--strict::
+	Enable more strict checking, every WARN severity for the `Fsck Messages`
+	be seen as ERROR.
+

Nit: It would be nice to redirect users to the fsck documentation to understand severity levels.

[snip]

author
#43 lore raw

Re: [PATCH v3 3/7] builtin/refs: add verify subcommand

Quoted 39 lines · Karthik Nayak
Quoted 37 lines · shejialuo

Introduce a new subcommand "verify" in git-refs(1) to allow the user to check the reference database consistency.

---
 Documentation/git-refs.txt | 13 +++++++++++
 builtin/refs.c             | 45 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+)
Documentation/git-refs.txt +4 -0
index 5b99e04385..16b518ecce 100644
@@ -10,6 +10,7 @@ SYNOPSIS
 --------
 [verse]
 'git refs migrate' --ref-format=<format> [--dry-run]
+'git refs verify' [--strict] [--verbose]
 
 DESCRIPTION
 -----------
@@ -22,6 +23,9 @@ COMMANDS
 migrate::
 	Migrate ref store between different formats.
 
+verify::
+	Verify reference database consistency.
+
 OPTIONS
 -------
 
@@ -39,6 +43,15 @@ include::ref-storage-format.txt[]
 	can be used to double check that the migration works as expected before
 	performing the actual migration.
 
Quoted 10 lines

Documentation/git-refs.txt include::ref-storage-format.txt[] · line 51

Quoted 6 lines
Documentation/git-refs.txt +6 -0
+The following options are specific to 'git refs verify':
+
+--strict::
+	Enable more strict checking, every WARN severity for the `Fsck Messages`
+	be seen as ERROR.
+

Nit: It would be nice to redirect users to the fsck documentation to understand severity levels.

I agree. Will improve this in the next version.

Quoted 1 line

[snip]

author
#44 lore raw

PATCH v3 4/7 builtin/fsck: add `git-refs verify` child process

In reply to shejialuo

Introduce a new function "fsck_refs" that initializes and runs a child process to execute the "git-refs verify" command.

---
 builtin/fsck.c | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
builtin/fsck.c +17 -0
index d13a226c2e..10d73f534f 100644
@@ -896,6 +896,21 @@ static int check_pack_rev_indexes(struct repository *r, int show_progress)
 	return res;
 }
 
+static void fsck_refs(void)
+{
+	struct child_process refs_verify = CHILD_PROCESS_INIT;
+	child_process_init(&refs_verify);
+	refs_verify.git_cmd = 1;
+	strvec_pushl(&refs_verify.args, "refs", "verify", NULL);
+	if (verbose)
+		strvec_push(&refs_verify.args, "--verbose");
+	if (check_strict)
+		strvec_push(&refs_verify.args, "--strict");
+
+	if (run_command(&refs_verify))
+		errors_found |= ERROR_REFS;
+}
+
 static char const * const fsck_usage[] = {
 	N_("git fsck [--tags] [--root] [--unreachable] [--cache] [--no-reflogs]\n"
 	   "         [--[no-]full] [--strict] [--verbose] [--lost-found]\n"
@@ -1065,6 +1080,8 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
 
 	check_connectivity();
 
+	fsck_refs();
+
 	if (the_repository->settings.core_commit_graph) {
 		struct child_process commit_graph_verify = CHILD_PROCESS_INIT;
 
Signature · 1 line
2.45.2
author
#45 lore raw

PATCH v3 5/7 files-backend: add unified interface for refs scanning

In reply to shejialuo

For refs and reflogs, we need to scan its corresponding directories to check every regular file or symbolic link which shares the same pattern. Introduce a unified interface for scanning directories for files-backend.

---
 refs/files-backend.c | 77 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 76 insertions(+), 1 deletion(-)
refs/files-backend.c +76 -1
index e965345ad8..74242dbba4 100644
@@ -4,6 +4,7 @@
 #include "../gettext.h"
 #include "../hash.h"
 #include "../hex.h"
+#include "../fsck.h"
 #include "../refs.h"
 #include "refs-internal.h"
 #include "ref-cache.h"
@@ -3402,6 +3403,78 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,
 	return ret;
 }
 
+/*
+ * For refs and reflogs, they share a unified interface when scanning
+ * the whole directory. This function is used as the callback for each
+ * regular file or symlink in the directory.
+ */
+typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o,
+				  const char *gitdir,
+				  const char *refs_check_dir,
+				  struct dir_iterator *iter);
+
+static int files_fsck_refs_dir(struct ref_store *ref_store,
+			       struct fsck_refs_options *o,
+			       const char *refs_check_dir,
+			       files_fsck_refs_fn *fsck_refs_fns)
+{
+	const char *gitdir = ref_store->gitdir;
+	struct strbuf sb = STRBUF_INIT;
+	struct dir_iterator *iter;
+	int iter_status;
+	int ret = 0;
+
+	strbuf_addf(&sb, "%s/%s", gitdir, refs_check_dir);
+
+	iter = dir_iterator_begin(sb.buf, 0);
+
+	if (!iter) {
+		ret = error_errno("cannot open directory %s", sb.buf);
+		goto out;
+	}
+
+	while ((iter_status = dir_iterator_advance(iter)) == ITER_OK) {
+		if (S_ISDIR(iter->st.st_mode)) {
+			continue;
+		} else if (S_ISREG(iter->st.st_mode) ||
+			   S_ISLNK(iter->st.st_mode)) {
+			if (o->verbose)
+				fprintf_ln(stderr, "Checking %s/%s",
+					   refs_check_dir, iter->relative_path);
+			for (size_t i = 0; fsck_refs_fns[i]; i++) {
+				if (fsck_refs_fns[i](o, gitdir, refs_check_dir, iter))
+					ret = -1;
+			}
+		} else {
+			ret = error(_("unexpected file type for '%s'"),
+				    iter->basename);
+		}
+	}
+
+	if (iter_status != ITER_DONE)
+		ret = error(_("failed to iterate over '%s'"), sb.buf);
+
+out:
+	strbuf_release(&sb);
+	return ret;
+}
+
+static int files_fsck_refs(struct ref_store *ref_store,
+			   struct fsck_refs_options *o)
+{
+	int ret;
+	files_fsck_refs_fn fsck_refs_fns[]= {
+		NULL
+	};
+
+	if (o->verbose)
+		fprintf_ln(stderr, "Checking references consistency");
+
+	ret = files_fsck_refs_dir(ref_store, o, "refs", fsck_refs_fns);
+
+	return ret;
+}
+
 static int files_fsck(struct ref_store *ref_store,
 		      struct fsck_refs_options *o)
 {
@@ -3409,7 +3482,9 @@ static int files_fsck(struct ref_store *ref_store,
 	struct files_ref_store *refs =
 		files_downcast(ref_store, REF_STORE_READ, "fsck");
 
-	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o);
+	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o)
+	    | files_fsck_refs(ref_store, o);
+
 	return ret;
 }
 
Signature · 1 line
2.45.2
author
#46 lore raw

PATCH v3 6/7 fsck: add ref name check for files backend

In reply to shejialuo

The git-fsck(1) only implicitly checks the reference, it does not fully check refs with bad format name such as standalone "@" and name ending with ".lock".

In order to provide such checks, add a new fsck message id "badRefName" with default ERROR type. Use existing "check_refname_format" to explicit check the ref name. And add a new unit test to verify the functionality.

---
 Documentation/fsck-msgids.txt |   3 +
 fsck.h                        |   1 +
 refs/files-backend.c          |  20 +++++++
 t/t0602-reffiles-fsck.sh      | 101 ++++++++++++++++++++++++++++++++++
 4 files changed, 125 insertions(+)
 create mode 100755 t/t0602-reffiles-fsck.sh
Documentation/fsck-msgids.txt +3 -0
index 5edc06c658..cc85c897cc 100644
@@ -19,6 +19,9 @@
 `badParentSha1`::
 	(ERROR) A commit object has a bad parent sha1.
 
+`badRefName`::
+	(ERROR) A ref has a bad name.
+
 `badTagName`::
 	(INFO) A tag has an invalid format.
 
fsck.h +1 -0
index 6a38ac4a16..6b3e68b873 100644
@@ -31,6 +31,7 @@ enum fsck_msg_type {
 	FUNC(BAD_NAME, ERROR) \
 	FUNC(BAD_OBJECT_SHA1, ERROR) \
 	FUNC(BAD_PARENT_SHA1, ERROR) \
+	FUNC(BAD_REF_NAME, ERROR) \
 	FUNC(BAD_TIMEZONE, ERROR) \
 	FUNC(BAD_TREE, ERROR) \
 	FUNC(BAD_TREE_SHA1, ERROR) \
refs/files-backend.c +20 -0
index 74242dbba4..3f0b565a13 100644
@@ -3413,6 +3413,25 @@ typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o,
 				  const char *refs_check_dir,
 				  struct dir_iterator *iter);
 
+static int files_fsck_refs_name(struct fsck_refs_options *o,
+				const char *gitdir UNUSED,
+				const char *refs_check_dir,
+				struct dir_iterator *iter)
+{
+	struct strbuf sb = STRBUF_INIT;
+	int ret = 0;
+
+	if (check_refname_format(iter->basename, REFNAME_ALLOW_ONELEVEL)) {
+		strbuf_addf(&sb, "%s/%s", refs_check_dir, iter->relative_path);
+		ret = fsck_refs_report(o, sb.buf,
+				       FSCK_MSG_BAD_REF_NAME,
+				       "invalid refname format");
+	}
+
+	strbuf_release(&sb);
+	return ret;
+}
+
 static int files_fsck_refs_dir(struct ref_store *ref_store,
 			       struct fsck_refs_options *o,
 			       const char *refs_check_dir,
@@ -3464,6 +3483,7 @@ static int files_fsck_refs(struct ref_store *ref_store,
 {
 	int ret;
 	files_fsck_refs_fn fsck_refs_fns[]= {
+		files_fsck_refs_name,
 		NULL
 	};
 
t/t0602-reffiles-fsck.sh new file +101 -0

101 lines not on this page · raw message

Signature · 1 line
2.45.2
author
#47 lore raw

PATCH v3 7/7 fsck: add ref content check for files backend

In reply to shejialuo

Enhance the git-fsck(1) command by adding a check for reference content in the files backend. The new functionality ensures that symrefs, real symbolic link and regular refs are validated correctly.

In order to check the trailing content of the regular refs, add a new parameter trailing to parse_loose_ref_contents.

For symrefs, parse_loose_ref_contents will set the "referent". However, symbolic link could be either absolute or relative. Use "strbuf_add_real_path" to read the symbolic link and convert the relative path to absolute path. Then use "skip_prefix" to make it align with symref "referent".

Thus, the symrefs and symbolic links could share the same interface. Add a new function "files_fsck_symref_target" which aims at checking the following things:

1. whether the pointee is under the refs/ directory.
2. whether the pointee name is correct.
3. whether the pointee path is a wrong type in filesystem.

Last, add the following FSCK MESSAGEs:

1. "badRefContent(ERROR)": A ref has a bad content
2. "badSymrefPointee(ERROR)": The pointee of a symref is bad.
3. "trailingRefContent(WARN)": A ref content has trailing contents.

---
 Documentation/fsck-msgids.txt |   9 +++
 fsck.h                        |   3 +
 refs.c                        |   2 +-
 refs/files-backend.c          | 145 +++++++++++++++++++++++++++++++++-
 refs/refs-internal.h          |   5 +-
 t/t0602-reffiles-fsck.sh      | 110 ++++++++++++++++++++++++++
 6 files changed, 269 insertions(+), 5 deletions(-)
Documentation/fsck-msgids.txt +9 -0
index cc85c897cc..ebe2de2bdc 100644
@@ -19,9 +19,15 @@
 `badParentSha1`::
 	(ERROR) A commit object has a bad parent sha1.
 
+`badRefContent`::
+	(ERROR) A ref has a bad content.
+
 `badRefName`::
 	(ERROR) A ref has a bad name.
 
+`badSymrefPointee`::
+	(ERROR) The pointee of a symref is bad.
+
 `badTagName`::
 	(INFO) A tag has an invalid format.
 
@@ -179,6 +185,9 @@
 `symlinkTargetMissing`::
 	(ERROR) Unable to read symbolic link target's blob.
 
+`trailingRefContent`::
+	(WARN) A ref content has trailing contents.
+
 `treeNotSorted`::
 	(ERROR) A tree is not properly sorted.
 
fsck.h +3 -0
index 6b3e68b873..215b1466d2 100644
@@ -32,6 +32,8 @@ enum fsck_msg_type {
 	FUNC(BAD_OBJECT_SHA1, ERROR) \
 	FUNC(BAD_PARENT_SHA1, ERROR) \
 	FUNC(BAD_REF_NAME, ERROR) \
+	FUNC(BAD_REF_CONTENT, ERROR) \
+	FUNC(BAD_SYMREF_POINTEE, ERROR) \
 	FUNC(BAD_TIMEZONE, ERROR) \
 	FUNC(BAD_TREE, ERROR) \
 	FUNC(BAD_TREE_SHA1, ERROR) \
@@ -74,6 +76,7 @@ enum fsck_msg_type {
 	FUNC(HAS_DOTDOT, WARN) \
 	FUNC(HAS_DOTGIT, WARN) \
 	FUNC(NULL_SHA1, WARN) \
+	FUNC(TRAILING_REF_CONTENT, WARN) \
 	FUNC(ZERO_PADDED_FILEMODE, WARN) \
 	FUNC(NUL_IN_COMMIT, WARN) \
 	FUNC(LARGE_PATHNAME, WARN) \
refs.c +1 -1
index 0922439275..1325f83269 100644
@@ -1744,7 +1744,7 @@ static int refs_read_special_head(struct ref_store *ref_store,
 	}
 
 	result = parse_loose_ref_contents(content.buf, oid, referent, type,
-					  failure_errno);
+					  failure_errno, NULL);
 
 done:
 	strbuf_release(&full_path);
refs/files-backend.c +143 -2

179 lines not on this page · raw message

refs/refs-internal.h +3 -2
index 8f42f21e77..72f0945ada 100644
@@ -709,11 +709,12 @@ struct ref_store {
 
 /*
  * Parse contents of a loose ref file. *failure_errno maybe be set to EINVAL for
- * invalid contents.
+ * invalid contents. Also *trailing is set to the first character after the
+ * refname or NULL if the referent is not empty.
  */
 int parse_loose_ref_contents(const char *buf, struct object_id *oid,
 			     struct strbuf *referent, unsigned int *type,
-			     int *failure_errno);
+			     int *failure_errno, const char **trailing);
 
 /*
  * Fill in the generic part of refs and add it to our collection of
t/t0602-reffiles-fsck.sh +110 -0

114 lines not on this page · raw message

Signature · 1 line
2.45.2
author
#48 lore raw

v4 cover ref consistency check infra setup

In reply to shejialuo

Hi All:

This version mainly focuses on solving following reviews:

1. As Junio said, it's a bad idea to convert the "fsck_refs_options *" to "fsck_options *" or convert the "fsck_options *" to "fsck_refs_options *". So this version, I changed a lot for the first patch. I refactor the "fsck_options *" which contains refs-specific, objs-specific and common settings. This designs allows us to use combination to make the code much more clean.
2. As Karthik said, I should enhance documents for "git refs verify". This version does this.

And Junio found that the commit the previous series based on is strange. In order to make our life easier, I change the based on commit. Now it is based on the top master. And I will always keep the code align with the master to avoid strange things.

66ac6e4bcd (The fourteenth batch, 2024-06-17)

Thanks

shejialuo (7):
fsck: add refs check interfaces to interact with fsck error levels
refs: set up ref consistency check infrastructure builtin/refs: add verify subcommand
builtin/fsck: add git-refs verify child process
files-backend: add unified interface for refs scanning
fsck: add ref name check for files backend
fsck: add ref content check for files backend

 Documentation/fsck-msgids.txt |  12 ++
 Documentation/git-refs.txt    |  13 ++
 builtin/fsck.c                |  25 +++-
 builtin/index-pack.c          |   2 +-
 builtin/mktag.c               |   5 +-
 builtin/refs.c                |  45 ++++++
 builtin/unpack-objects.c      |   2 +-
 fetch-pack.c                  |   8 +-
 fsck.c                        | 113 +++++++++++----
 fsck.h                        |  83 ++++++++---
 object-file.c                 |   2 +-
 refs.c                        |   7 +-
 refs.h                        |   8 ++
 refs/debug.c                  |  11 ++
 refs/files-backend.c          | 255 +++++++++++++++++++++++++++++++++-
 refs/packed-backend.c         |   8 ++
 refs/refs-internal.h          |  11 +-
 refs/reftable-backend.c       |   8 ++
 t/t0602-reffiles-fsck.sh      | 211 ++++++++++++++++++++++++++++
 19 files changed, 765 insertions(+), 64 deletions(-)
 create mode 100755 t/t0602-reffiles-fsck.sh

Range-diff against v3:

1:  b8d20c4be9 < -:  ---------- fsck: add refs check interfaces to interface with fsck error levels
-:  ---------- > 1:  2d2c1f2cd3 fsck: add refs check interfaces to interact with fsck error levels
2:  c7ed95d6b4 ! 2:  4bacffe44e refs: set up ref consistency check infrastructure
    @@ refs.c: int check_refname_format(const char *refname, int flags)
      	return check_or_sanitize_refname(refname, flags, NULL);
      }
      
    -+int refs_fsck(struct ref_store *refs, struct fsck_refs_options *o)
    ++int refs_fsck(struct ref_store *refs, struct fsck_options *o)
     +{
     +	return refs->be->fsck(refs, o);
     +}
    @@ refs.h
      
      #include "commit.h"
      
    -+struct fsck_refs_options;
    ++struct fsck_options;
      struct object_id;
      struct ref_store;
      struct repository;
    @@ refs.h: int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void
     + * reflogs are consistent, and non-zero otherwise. The errors will be
     + * written to stderr.
     + */
    -+int refs_fsck(struct ref_store *refs, struct fsck_refs_options *o);
    ++int refs_fsck(struct ref_store *refs, struct fsck_options *o);
     +
      /*
       * Apply the rules from check_refname_format, but mutate the result until it
    @@ refs/debug.c: static int debug_reflog_expire(struct ref_store *ref_store, const
      }
      
     +static int debug_fsck(struct ref_store *ref_store,
    -+		      struct fsck_refs_options *o)
    ++		      struct fsck_options *o)
     +{
     +	struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store;
     +	int res = drefs->refs->be->fsck(drefs->refs, o);
    @@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
      }
      
     +static int files_fsck(struct ref_store *ref_store,
    -+		      struct fsck_refs_options *o)
    ++		      struct fsck_options *o)
     +{
     +	int ret;
     +	struct files_ref_store *refs =
    @@ refs/packed-backend.c: static struct ref_iterator *packed_reflog_iterator_begin(
      }
      
     +static int packed_fsck(struct ref_store *ref_store,
    -+		       struct fsck_refs_options *o)
    ++		       struct fsck_options *o)
     +{
     +	return 0;
     +}
    @@ refs/refs-internal.h
      #include "refs.h"
      #include "iterator.h"
      
    -+struct fsck_refs_options;
    ++struct fsck_options;
      struct ref_transaction;
      
      /*
    @@ refs/refs-internal.h: typedef int read_raw_ref_fn(struct ref_store *ref_store, c
      				 struct strbuf *referent);
      
     +typedef int fsck_fn(struct ref_store *ref_store,
    -+		    struct fsck_refs_options *o);
    ++		    struct fsck_options *o);
     +
      struct ref_storage_be {
      	const char *name;
    @@ refs/reftable-backend.c: static int reftable_be_reflog_expire(struct ref_store *
      }
      
     +static int reftable_be_fsck(struct ref_store *ref_store,
    -+			    struct fsck_refs_options *o)
    ++			    struct fsck_options *o)
     +{
     +	return 0;
     +}
3:  373f470fdc ! 3:  f00acab4d1 builtin/refs: add verify subcommand
    @@ Documentation/git-refs.txt: include::ref-storage-format.txt[]
     +
     +--strict::
     +	Enable more strict checking, every WARN severity for the `Fsck Messages`
    -+	be seen as ERROR.
    ++	be seen as ERROR. See linkgit:git-fsck[1].
     +
     +--verbose::
     +	When verifying the reference database consistency, be chatty.
    @@ builtin/refs.c: static int cmd_refs_migrate(int argc, const char **argv, const c
     +	};
     +	int ret = 0;
     +	unsigned int verbose = 0, strict = 0;
    -+	struct fsck_refs_options fsck_refs_options = FSCK_REFS_OPTIONS_DEFAULT;
    ++	struct fsck_options fsck_options = FSCK_OPTIONS_DEFAULT;
     +	struct option options[] = {
     +		OPT__VERBOSE(&verbose, N_("be verbose")),
     +		OPT_BOOL(0, "strict", &strict, N_("enable strict checking")),
    @@ builtin/refs.c: static int cmd_refs_migrate(int argc, const char **argv, const c
     +		usage(_("too many arguments"));
     +
     +	if (verbose)
    -+		fsck_refs_options.verbose = 1;
    ++		fsck_options.verbose = 1;
     +	if (strict)
    -+		fsck_refs_options.strict = 1;
    ++		fsck_options.strict = 1;
     +
    -+	git_config(git_fsck_refs_config, &fsck_refs_options);
    ++	git_config(git_fsck_config, &fsck_options);
     +	prepare_repo_settings(the_repository);
     +
    -+	ret = refs_fsck(get_main_ref_store(the_repository), &fsck_refs_options);
    ++	ret = refs_fsck(get_main_ref_store(the_repository), &fsck_options);
     +
     +	/*
    -+	 * Explicitly free the allocated array. This is necessary because
    -+	 * this program is executed as child process of git-fsck(1) and the
    -+	 * allocated array may not freed when git-fsck(1) aborts somewhere.
    ++	 * Explicitly free the allocated array and object skiplist set. Because
    ++	 * we reuse `git_fsck_config` here. It will still set the skiplist.
     +	 */
    -+	free(fsck_refs_options.msg_type);
    ++	free(fsck_options.msg_type);
    ++	oidset_clear(&fsck_options.objs_options.skiplist);
     +	return ret;
     +}
     +
4:  4f8ebea573 = 4:  6494f5b0c2 builtin/fsck: add `git-refs verify` child process
5:  6062b3b453 ! 5:  2c064f4f71 files-backend: add unified interface for refs scanning

@@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
+ * the whole directory. This function is used as the callback for each
+ * regular file or symlink in the directory.
+ */
-+typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o, ++typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
+ const char *gitdir,
+ const char *refs_check_dir,
+ struct dir_iterator *iter);
+
+static int files_fsck_refs_dir(struct ref_store *ref_store,
-+ struct fsck_refs_options *o,
++ struct fsck_options *o,
+ const char *refs_check_dir,
+ files_fsck_refs_fn *fsck_refs_fns)
+{
@@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
+}
+
+static int files_fsck_refs(struct ref_store *ref_store,
-+ struct fsck_refs_options *o)
++ struct fsck_options *o)
+{
+ int ret;
+ files_fsck_refs_fn fsck_refs_fns[]= {
@@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
+}
+
static int files_fsck(struct ref_store *ref_store,
- struct fsck_refs_options *o)
+ struct fsck_options *o)
{
@@ refs/files-backend.c: static int files_fsck(struct ref_store *ref_store,
struct files_ref_store *refs =
6: 87a22f4c7f ! 6: 88b54b9e1a fsck: add ref name check for files backend
@@ fsck.h: enum fsck_msg_type {
FUNC(BAD_TREE_SHA1, ERROR) \

## refs/files-backend.c ##
-@@ refs/files-backend.c: typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o, +@@ refs/files-backend.c: typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
const char *refs_check_dir,
struct dir_iterator *iter);

-+static int files_fsck_refs_name(struct fsck_refs_options *o, ++static int files_fsck_refs_name(struct fsck_options *o,
+ const char *gitdir UNUSED,
+ const char *refs_check_dir,
+ struct dir_iterator *iter)
@@ refs/files-backend.c: typedef int (*files_fsck_refs_fn)(struct fsck_refs_options
+}
+
static int files_fsck_refs_dir(struct ref_store *ref_store,
- struct fsck_refs_options *o,
+ struct fsck_options *o,
const char *refs_check_dir,
@@ refs/files-backend.c: static int files_fsck_refs(struct ref_store *ref_store,
{
7: b945f61ef1 ! 7: 8309a4746a fsck: add ref content check for files backend
@@ refs/files-backend.c: int parse_loose_ref_contents(const char *buf, struct objec
return 0;
}

-@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *o, +@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
return ret;
}

@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
+ * would be the content after "refs:". For symblic link, "pointee_name" would
+ * be the relative path agaignst "gitdir".
+ */
-+static int files_fsck_symref_target(struct fsck_refs_options *o, ++static int files_fsck_symref_target(struct fsck_options *o,
+ const char *refname,
+ const char *pointee_name,
+ const char *pointee_path)
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
+ return ret;
+}
+
-+static int files_fsck_refs_content(struct fsck_refs_options *o, ++static int files_fsck_refs_content(struct fsck_options *o,
+ const char *gitdir,
+ const char *refs_check_dir,
+ struct dir_iterator *iter)
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
+}
+
static int files_fsck_refs_dir(struct ref_store *ref_store,
- struct fsck_refs_options *o,
+ struct fsck_options *o,
const char *refs_check_dir,
@@ refs/files-backend.c: static int files_fsck_refs(struct ref_store *ref_store,
int ret;

Signature · 1 line
2.45.2
author
#49 lore raw

PATCH v4 1/7 fsck: add refs check interfaces to interact with fsck error levels

The git-fsck(1) focuses on object database consistency check. It relies on the "fsck_options" to interact with fsck error levels. However, "fsck_options" aims at checking the object database which contains a lot of fields only related to object database.

In order to add ref operations, create a new struct named "fsck_refs_options" and a new struct named "fsck_objs_options". Remove object-related fields from "fsck_options" to "fsck_objs_options". Change the "fsck_options" with three parts of members:

1. The "fsck_refs_options".
2. The "fsck_objs_options".
3. The common settings both for refs and objects. Because we leave common settings in "fsck_options". The setup process could be fully reused without any code changing.

Also add related macros to align with the current code. Because we remove some fields from "fsck_options" to "fsck_objs_options". Change the influenced code to use the "fsck_options.objs_options" instead of using "fsck_options" itself.

The static function "report" provided by "fsck.c" aims at reporting the problems related to object database which cannot be reused for refs. Provide "fsck_refs_report" function to integrate the fsck error levels into reference consistency check.

---
 builtin/fsck.c           |   8 +--
 builtin/index-pack.c     |   2 +-
 builtin/mktag.c          |   5 +-
 builtin/unpack-objects.c |   2 +-
 fetch-pack.c             |   8 +--
 fsck.c                   | 113 +++++++++++++++++++++++++++++----------
 fsck.h                   |  79 ++++++++++++++++++++-------
 object-file.c            |   2 +-
 8 files changed, 161 insertions(+), 58 deletions(-)
builtin/fsck.c +4 -4
index d13a226c2e..a5b82e228c 100644
@@ -233,7 +233,7 @@ static void mark_unreachable_referents(const struct object_id *oid)
 			object_as_type(obj, type, 0);
 	}
 
-	options.walk = mark_used;
+	options.objs_options.walk = mark_used;
 	fsck_walk(obj, NULL, &options);
 	if (obj->type == OBJ_TREE)
 		free_tree_buffer((struct tree *)obj);
@@ -936,9 +936,9 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
 
 	argc = parse_options(argc, argv, prefix, fsck_opts, fsck_usage, 0);
 
-	fsck_walk_options.walk = mark_object;
-	fsck_obj_options.walk = mark_used;
-	fsck_obj_options.error_func = fsck_error_func;
+	fsck_walk_options.objs_options.walk = mark_object;
+	fsck_obj_options.objs_options.walk = mark_used;
+	fsck_obj_options.objs_options.error_func = fsck_error_func;
 	if (check_strict)
 		fsck_obj_options.strict = 1;
 
builtin/index-pack.c +1 -1
index 856428fef9..8824d7b8a9 100644
@@ -1746,7 +1746,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
 		usage(index_pack_usage);
 
 	disable_replace_refs();
-	fsck_options.walk = mark_link;
+	fsck_options.objs_options.walk = mark_link;
 
 	reset_pack_idx_option(&opts);
 	opts.flags |= WRITE_REV;
builtin/mktag.c +3 -2
index 4767f1a97e..e88475f009 100644
@@ -91,8 +91,9 @@ int cmd_mktag(int argc, const char **argv, const char *prefix)
 	if (strbuf_read(&buf, 0, 0) < 0)
 		die_errno(_("could not read from stdin"));
 
-	fsck_options.error_func = mktag_fsck_error_func;
-	fsck_set_msg_type_from_ids(&fsck_options, FSCK_MSG_EXTRA_HEADER_ENTRY,
+	fsck_options.objs_options.error_func = mktag_fsck_error_func;
+	fsck_set_msg_type_from_ids(&fsck_options,
+				   FSCK_MSG_EXTRA_HEADER_ENTRY,
 				   FSCK_WARN);
 	/* config might set fsck.extraHeaderEntry=* again */
 	git_config(git_fsck_config, &fsck_options);
builtin/unpack-objects.c +1 -1
index f1c85a00ae..368810a2a1 100644
@@ -239,7 +239,7 @@ static int check_object(struct object *obj, enum object_type type,
 		die("Whoops! Cannot find object '%s'", oid_to_hex(&obj->oid));
 	if (fsck_object(obj, obj_buf->buffer, obj_buf->size, &fsck_options))
 		die("fsck error in packed object");
-	fsck_options.walk = check_object;
+	fsck_options.objs_options.walk = check_object;
 	if (fsck_walk(obj, NULL, &fsck_options))
 		die("Error on reachable objects of %s", oid_to_hex(&obj->oid));
 	write_cached_object(obj, obj_buf);
fetch-pack.c +5 -3
index eba9e420ea..148f9bd371 100644
@@ -1220,7 +1220,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 	} else
 		alternate_shallow_file = NULL;
 	if (get_pack(args, fd, pack_lockfiles, NULL, sought, nr_sought,
-		     &fsck_options.gitmodules_found))
+		     &fsck_options.objs_options.gitmodules_found))
 		die(_("git fetch-pack: fetch failed."));
 	if (fsck_finish(&fsck_options))
 		die("fsck failed");
@@ -1780,7 +1780,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
 
 			if (get_pack(args, fd, pack_lockfiles,
 				     packfile_uris.nr ? &index_pack_args : NULL,
-				     sought, nr_sought, &fsck_options.gitmodules_found))
+				     sought, nr_sought,
+				     &fsck_options.objs_options.gitmodules_found))
 				die(_("git fetch-pack: fetch failed."));
 			do_check_stateless_delimiter(args->stateless_rpc, &reader);
 
@@ -1823,7 +1824,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
 
 		packname[the_hash_algo->hexsz] = '\0';
 
-		parse_gitmodules_oids(cmd.out, &fsck_options.gitmodules_found);
+		parse_gitmodules_oids(cmd.out,
+				      &fsck_options.objs_options.gitmodules_found);
 
 		close(cmd.out);
 
fsck.c +85 -28

275 lines not on this page · raw message

fsck.h +61 -18

123 lines not on this page · raw message

object-file.c +1 -1
index d3cf4b8b2e..b027d70725 100644
@@ -2510,7 +2510,7 @@ static int index_mem(struct index_state *istate,
 		struct fsck_options opts = FSCK_OPTIONS_DEFAULT;
 
 		opts.strict = 1;
-		opts.error_func = hash_format_check_report;
+		opts.objs_options.error_func = hash_format_check_report;
 		if (fsck_buffer(null_oid(), type, buf, size, &opts))
 			die(_("refusing to create malformed object"));
 		fsck_finish(&opts);
Signature · 1 line
2.45.2
#50 lore raw

Re: [GSoC][PATCH v4 1/7] fsck: add refs check interfaces to interact with fsck error levels

Quoted 26 lines · shejialuo

The git-fsck(1) focuses on object database consistency check. It relies on the "fsck_options" to interact with fsck error levels. However, "fsck_options" aims at checking the object database which contains a lot of fields only related to object database.

In order to add ref operations, create a new struct named "fsck_refs_options" and a new struct named "fsck_objs_options". Remove object-related fields from "fsck_options" to "fsck_objs_options". Change the "fsck_options" with three parts of members:

1. The "fsck_refs_options".
2. The "fsck_objs_options".
3. The common settings both for refs and objects. Because we leave common settings in "fsck_options". The setup process could be fully reused without any code changing.

Also add related macros to align with the current code. Because we remove some fields from "fsck_options" to "fsck_objs_options". Change the influenced code to use the "fsck_options.objs_options" instead of using "fsck_options" itself.

The static function "report" provided by "fsck.c" aims at reporting the problems related to object database which cannot be reused for refs. Provide "fsck_refs_report" function to integrate the fsck error levels into reference consistency check.

Quoted 4 lines

---

This seems to be doing too many things at once, making the result a lot harder to review than necessary. At this point, nobody checks refs and reports problems with refs, so fsck_refs_report() has no callers and it is impossible to tell if the function signature of it, iow, the set of parameters it receives, is sufficient, for example.

Stepping back a bit, it is true that (1) all existing checks are about "objects", and (2) all checks we want to implement around "objects" and "refs" can be split cleanly into these two categories?

I am wondering if there are checks and reports that would benefit from having access to both objects and refs (e.g. when checking a ref, you may want to see both what the name of the ref is and what object the ref points at), in which case, being forced to implement such a check-and-report as "object" or "ref" that has access to only different subset of information may turn out to be too limiting.

Yes, I am OK with having substructure in fsck_options, but I am doubting if it is a good idea to have a separate fsck_refs_report() that can only take "name" that is different from fsck.c::report().

For example, how would we ensure that refs/heads/foo is allowed to point at a commit object and nothing else, and how would we report a violation when we find that ref/heads/foo is pointing at a tag, i.e., "refs/heads/foo points at f665776185ad074b236c00751d666da7d1977dbe which is a tag". The fsck_refs_report() function is not equipped to do that; neither is .refs_options.error_func() that only takes "name".

fsck.c static int fsck_blobs(struct oidset *blobs_found, struct oidset *blobs_done, · line 1322

Quoted 10 lines
fsck.c +9 -0
+int fsck_refs_report(struct fsck_options *o,
+		     const char *name,
+		     enum fsck_msg_id msg_id,
+		     const char *fmt, ...)
...
+	va_start(ap, fmt);
+	strbuf_vaddf(&sb, fmt, ap);
+	ret = o->refs_options.error_func(o, name, msg_type, msg_id, sb.buf);
+	strbuf_release(&sb);
+	va_end(ap);

Perhaps the code and data structure of the entire series may be capable of supporting such a check-and-report, but the primary point I am making is that among what [1/7] adds, we cannot sanely judge if these "refs" related additions are sensible by looking at [1/7].

Thanks.

author
#51 lore raw

Re: [GSoC][PATCH v4 1/7] fsck: add refs check interfaces to interact with fsck error levels

Quoted 1 line · Junio C Hamano

shejialuo <[email protected]> writes:

[snip]

Quoted 7 lines

This seems to be doing too many things at once, making the result a lot harder to review than necessary. At this point, nobody checks refs and reports problems with refs, so fsck_refs_report() has no callers and it is impossible to tell if the function signature of it, iow, the set of parameters it receives, is sufficient, for example.

I will split this commit into multiple commits in the next version.

Quoted 4 lines

Stepping back a bit, it is true that (1) all existing checks are about "objects", and (2) all checks we want to implement around "objects" and "refs" can be split cleanly into these two categories?

I am sure that "objects" and "refs" checks cannot be split cleanly into two categories. Let me elaborate more about this. The git-fsck(1) has already implicitly checked refs. When executing git-fsck(1) with no arguments, it will call "builtin/fsck.c::get_default_heads()". It will traverse every raw ref and use "fsck_handle_ref" to check the refs.

	struct obj *obj;
	obj = parse_object(the_repository, oid);
	if (!obj) {
		...
		errors_found |= ERROR_REACHABLE;
		return 0;
	}

	if (obj->type != OBJ_COMMIT && is_branch(refname)) {
		...
		errors_found |= ERROR_REFS;
	}
	obj->flags |= USED;
	mark_object_reachable(obj);
	return 0;

git-fsck(1) firstly scans the object directory to check every loose object. If everything is OK, it will set "obj->flags" like the following:

	obj->flags &= ~(REACHABLE | SEEN);
	obj->flags |= HAS_OBJ;

When an object is marked with "UNREACHABLE" and "UNUSED", it will be dangling.

When checking objects, git-fsck(1) still checks the refs. But from my understanding, the main motivation here is mark the "USED" flag because we want to tell whether an object is dangling and BTY we check refs. So, we cannot really classify into these two categories. But we can classify into these two categories from action view. When checking refs, we look at the refs options; when checking objects, we look at the objects options. Although we may implicitly check some other things, this should be OK.

So, I think "fsck_options" is only about options. Because the nature of the connectivity between refs and commit objects, we cannot make the boundary so clear.

Quoted 7 lines

I am wondering if there are checks and reports that would benefit from having access to both objects and refs (e.g. when checking a ref, you may want to see both what the name of the ref is and what object the ref points at), in which case, being forced to implement such a check-and-report as "object" or "ref" that has access to only different subset of information may turn out to be too limiting.

Should "check-and-report" does this? I am doubt. There are some cases we do look at both object and ref. But "report" should not do this. The flow git-fsck(1) does is like the following:

	if (some_thing_bad) {
		ret = report(..., message);
	}

Here, "check" will provide dedicated message to report. So If we really both look at both object and ref, the caller should be responsible for that. Like the message "refs/heads/foo points at f665776185ad074b236c00751d666da7d1977dbe which is a tag". The caller should produce such message. It will call some APIs to get the information and pass it to the "report" function.

Quoted 4 lines

Yes, I am OK with having substructure in fsck_options, but I am doubting if it is a good idea to have a separate fsck_refs_report() that can only take "name" that is different from fsck.c::report().

Quoted 8 lines

For example, how would we ensure that refs/heads/foo is allowed to point at a commit object and nothing else, and how would we report a violation when we find that ref/heads/foo is pointing at a tag, i.e., "refs/heads/foo points at f665776185ad074b236c00751d666da7d1977dbe which is a tag". The fsck_refs_report() function is not equipped to do that; neither is .refs_options.error_func() that only takes "name".

Yes, I am not satisfied with current design either. Actually, there are many redundant code in new "fsck_refs_report" function. Let's look at "fsck.c::report()" closely to see what it does.

	static int report(struct fsck_options *options,
				const struct object_id *oid, enum object_type object_type,
				enum fsck_msg_id msg_id, const char *fmt, ...)
	{
		va_list ap;
		struct strbuf sb = STRBUF_INIT;
		enum fsck_msg_type msg_type = fsck_msg_type(msg_id, options);
		int result;

		if (msg_type == FSCK_IGNORE)
			return 0;

		if (object_on_skiplist(options, oid))
			return 0;

		if (msg_type == FSCK_FATAL)
			msg_type = FSCK_ERROR;
		else if (msg_type == FSCK_INFO)
			msg_type = FSCK_WARN;

		prepare_msg_ids();
		strbuf_addf(&sb, "%s: ", msg_id_info[msg_id].camelcased);

		va_start(ap, fmt);
		strbuf_vaddf(&sb, fmt, ap);
		result = options->objs_options.error_func(options, oid, object_type,
								msg_type, msg_id, sb.buf);
		strbuf_release(&sb);
		va_end(ap);

		return result;
	}

It will get the current message "msg_type" which would be configured and it will convert the "FSCK_FATAL" to "FSCK_ERROR", and "FSCK_INFO" to "FSCK_WARN". From above code, we can easily know what would the error (warn) message be the following:

'object: fsck_describe_object(o, oid): FSCK_MESSAGE: message'.

The "fsck_describe_object" aims to provide meaningful message about the oid. And "FSCK_MESSAGE" is "msg_id_info[msg_id]".The last field "message" is the caller-provided message.

So the hardest part to reuse this function is that it receives the parameters related to object such as oid and object_type. Actually, the "fsck_refs_report" could generate such information "refs/heads/foo points at f665776185ad074b236c00751d666da7d1977dbe which is a tag" like the following (I use string literal for simplicity here):

	if (some_thing_bad) {
		...
		ret = fsck_refs_report(o, "refs/heads/foo", FSCK_BAD_REF_CONTENT,
			"points at f665776185ad074b236c00751d666da7d1977dbe which is a tag")
	}

It will generate the following output in the current design:

	error: refs/heads/foo: badRefContent: points at
	f665776185ad074b236c00751d666da7d1977dbe which is a tag

So my idea is that we just reuse the "report" function here. The bad part for current "report" function is that it is highly coupled with the object check. However, my idea is just like the "fsck_refs_report".

<name>: <FSCK_MESSAGE_STRING>: <message>

Thus, we may unify the report interface both for refs and objects. And we don't define two differtn interface:

1. "fsck_refs_error"
2. "fsck_objs_error"

Also, we just need to use "fsck_refs_report" and "fsck_objs_report" to wrap the "report" function here to avoid redundance.

Quoted 7 lines
Quoted 7 lines
fsck.c +6 -0
+int fsck_refs_report(struct fsck_options *o,
+		     const char *name,
+		     enum fsck_msg_id msg_id,
+		     const char *fmt, ...)
...
+	va_start(ap, fmt);
+	strbuf_vaddf(&sb, fmt, ap);
Quoted 9 lines

fsck.c static int fsck_blobs(struct oidset *blobs_found, struct oidset *blobs_done, · line 1322

Quoted 3 lines
fsck.c +3 -0
+	ret = o->refs_options.error_func(o, name, msg_type, msg_id, sb.buf);
+	strbuf_release(&sb);
+	va_end(ap);

Perhaps the code and data structure of the entire series may be capable of supporting such a check-and-report, but the primary point I am making is that among what [1/7] adds, we cannot sanely judge if these "refs" related additions are sensible by looking at [1/7].

From my above reply, I guess the current design gives the caller extensibility to provide customize message. All we do is make code more clean without redundance.

Quoted 1 line

Thanks.

Thanks,
Jialuo

author
#52 lore raw

PATCH v4 2/7 refs: set up ref consistency check infrastructure

In reply to shejialuo

The interfaces defined in the ref_storage_be are carefully structured in semantic. It's organized as the five parts:

1. The name and the initialization interfaces.
2. The ref transaction interfaces.
3. The ref internal interfaces (pack, rename and copy).
4. The ref filesystem interfaces.
5. The reflog related interfaces.

To keep consistent with the git-fsck(1), add a new interface named "fsck_refs_fn" to the end of "ref_storage_be". This semantic cannot be grouped into any above five categories. Explicitly add blank line to make it different from others.

---
 refs.c                  |  5 +++++
 refs.h                  |  8 ++++++++
 refs/debug.c            | 11 +++++++++++
 refs/files-backend.c    | 15 ++++++++++++++-
 refs/packed-backend.c   |  8 ++++++++
 refs/refs-internal.h    |  6 ++++++
 refs/reftable-backend.c |  8 ++++++++
 7 files changed, 60 insertions(+), 1 deletion(-)
refs.c +5 -0
index 0c1e45f5a9..cb3b8ec36a 100644
@@ -316,6 +316,11 @@ int check_refname_format(const char *refname, int flags)
 	return check_or_sanitize_refname(refname, flags, NULL);
 }
 
+int refs_fsck(struct ref_store *refs, struct fsck_options *o)
+{
+	return refs->be->fsck(refs, o);
+}
+
 void sanitize_refname_component(const char *refname, struct strbuf *out)
 {
 	if (check_or_sanitize_refname(refname, REFNAME_ALLOW_ONELEVEL, out))
refs.h +8 -0
index 756039a051..93fa23c7ee 100644
@@ -3,6 +3,7 @@
 
 #include "commit.h"
 
+struct fsck_options;
 struct object_id;
 struct ref_store;
 struct repository;
@@ -547,6 +548,13 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat
  */
 int check_refname_format(const char *refname, int flags);
 
+/*
+ * Check the reference database for consistency. Return 0 if refs and
+ * reflogs are consistent, and non-zero otherwise. The errors will be
+ * written to stderr.
+ */
+int refs_fsck(struct ref_store *refs, struct fsck_options *o);
+
 /*
  * Apply the rules from check_refname_format, but mutate the result until it
  * is acceptable, and place the result in "out".
refs/debug.c +11 -0
index 547d9245b9..45e2e784a0 100644
@@ -419,6 +419,15 @@ static int debug_reflog_expire(struct ref_store *ref_store, const char *refname,
 	return res;
 }
 
+static int debug_fsck(struct ref_store *ref_store,
+		      struct fsck_options *o)
+{
+	struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store;
+	int res = drefs->refs->be->fsck(drefs->refs, o);
+	trace_printf_key(&trace_refs, "fsck: %d\n", res);
+	return res;
+}
+
 struct ref_storage_be refs_be_debug = {
 	.name = "debug",
 	.init = NULL,
@@ -451,4 +460,6 @@ struct ref_storage_be refs_be_debug = {
 	.create_reflog = debug_create_reflog,
 	.delete_reflog = debug_delete_reflog,
 	.reflog_expire = debug_reflog_expire,
+
+	.fsck = debug_fsck,
 };
refs/files-backend.c +14 -1
index 4519b46171..ba9e57d1e9 100644
@@ -3405,6 +3405,17 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,
 	return ret;
 }
 
+static int files_fsck(struct ref_store *ref_store,
+		      struct fsck_options *o)
+{
+	int ret;
+	struct files_ref_store *refs =
+		files_downcast(ref_store, REF_STORE_READ, "fsck");
+
+	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o);
+	return ret;
+}
+
 struct ref_storage_be refs_be_files = {
 	.name = "files",
 	.init = files_ref_store_init,
@@ -3431,5 +3442,7 @@ struct ref_storage_be refs_be_files = {
 	.reflog_exists = files_reflog_exists,
 	.create_reflog = files_create_reflog,
 	.delete_reflog = files_delete_reflog,
-	.reflog_expire = files_reflog_expire
+	.reflog_expire = files_reflog_expire,
+
+	.fsck = files_fsck,
 };
refs/packed-backend.c +8 -0
index c4c1e36aa2..ad3e8fb1d1 100644
@@ -1733,6 +1733,12 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_s
 	return empty_ref_iterator_begin();
 }
 
+static int packed_fsck(struct ref_store *ref_store,
+		       struct fsck_options *o)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_packed = {
 	.name = "packed",
 	.init = packed_ref_store_init,
@@ -1760,4 +1766,6 @@ struct ref_storage_be refs_be_packed = {
 	.create_reflog = NULL,
 	.delete_reflog = NULL,
 	.reflog_expire = NULL,
+
+	.fsck = packed_fsck,
 };
refs/refs-internal.h +6 -0
index cbcb6f9c36..280acb7f9e 100644
@@ -4,6 +4,7 @@
 #include "refs.h"
 #include "iterator.h"
 
+struct fsck_options;
 struct ref_transaction;
 
 /*
@@ -650,6 +651,9 @@ typedef int read_raw_ref_fn(struct ref_store *ref_store, const char *refname,
 typedef int read_symbolic_ref_fn(struct ref_store *ref_store, const char *refname,
 				 struct strbuf *referent);
 
+typedef int fsck_fn(struct ref_store *ref_store,
+		    struct fsck_options *o);
+
 struct ref_storage_be {
 	const char *name;
 	ref_store_init_fn *init;
@@ -677,6 +681,8 @@ struct ref_storage_be {
 	create_reflog_fn *create_reflog;
 	delete_reflog_fn *delete_reflog;
 	reflog_expire_fn *reflog_expire;
+
+	fsck_fn *fsck;
 };
 
 extern struct ref_storage_be refs_be_files;
refs/reftable-backend.c +8 -0
index 3a75d63eb5..2b2630d47a 100644
@@ -2281,6 +2281,12 @@ static int reftable_be_reflog_expire(struct ref_store *ref_store,
 	return ret;
 }
 
+static int reftable_be_fsck(struct ref_store *ref_store,
+			    struct fsck_options *o)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_reftable = {
 	.name = "reftable",
 	.init = reftable_be_init,
@@ -2308,4 +2314,6 @@ struct ref_storage_be refs_be_reftable = {
 	.create_reflog = reftable_be_create_reflog,
 	.delete_reflog = reftable_be_delete_reflog,
 	.reflog_expire = reftable_be_reflog_expire,
+
+	.fsck = reftable_be_fsck,
 };
Signature · 1 line
2.45.2
author
#53 lore raw

PATCH v4 3/7 builtin/refs: add verify subcommand

In reply to shejialuo

Introduce a new subcommand "verify" in git-refs(1) to allow the user to check the reference database consistency.

---
 Documentation/git-refs.txt | 13 +++++++++++
 builtin/refs.c             | 45 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+)
Documentation/git-refs.txt +13 -0
index 5b99e04385..1244a85b64 100644
@@ -10,6 +10,7 @@ SYNOPSIS
 --------
 [verse]
 'git refs migrate' --ref-format=<format> [--dry-run]
+'git refs verify' [--strict] [--verbose]
 
 DESCRIPTION
 -----------
@@ -22,6 +23,9 @@ COMMANDS
 migrate::
 	Migrate ref store between different formats.
 
+verify::
+	Verify reference database consistency.
+
 OPTIONS
 -------
 
@@ -39,6 +43,15 @@ include::ref-storage-format.txt[]
 	can be used to double check that the migration works as expected before
 	performing the actual migration.
 
+The following options are specific to 'git refs verify':
+
+--strict::
+	Enable more strict checking, every WARN severity for the `Fsck Messages`
+	be seen as ERROR. See linkgit:git-fsck[1].
+
+--verbose::
+	When verifying the reference database consistency, be chatty.
+
 KNOWN LIMITATIONS
 -----------------
 
builtin/refs.c +45 -0

70 lines not on this page · raw message

Signature · 1 line
2.45.2
author
#54 lore raw

PATCH v4 4/7 builtin/fsck: add `git-refs verify` child process

In reply to shejialuo

Introduce a new function "fsck_refs" that initializes and runs a child process to execute the "git-refs verify" command.

---
 builtin/fsck.c | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
builtin/fsck.c +17 -0
index a5b82e228c..cd988ff167 100644
@@ -896,6 +896,21 @@ static int check_pack_rev_indexes(struct repository *r, int show_progress)
 	return res;
 }
 
+static void fsck_refs(void)
+{
+	struct child_process refs_verify = CHILD_PROCESS_INIT;
+	child_process_init(&refs_verify);
+	refs_verify.git_cmd = 1;
+	strvec_pushl(&refs_verify.args, "refs", "verify", NULL);
+	if (verbose)
+		strvec_push(&refs_verify.args, "--verbose");
+	if (check_strict)
+		strvec_push(&refs_verify.args, "--strict");
+
+	if (run_command(&refs_verify))
+		errors_found |= ERROR_REFS;
+}
+
 static char const * const fsck_usage[] = {
 	N_("git fsck [--tags] [--root] [--unreachable] [--cache] [--no-reflogs]\n"
 	   "         [--[no-]full] [--strict] [--verbose] [--lost-found]\n"
@@ -1065,6 +1080,8 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
 
 	check_connectivity();
 
+	fsck_refs();
+
 	if (the_repository->settings.core_commit_graph) {
 		struct child_process commit_graph_verify = CHILD_PROCESS_INIT;
 
Signature · 1 line
2.45.2
author
#55 lore raw

PATCH v4 5/7 files-backend: add unified interface for refs scanning

In reply to shejialuo

For refs and reflogs, we need to scan its corresponding directories to check every regular file or symbolic link which shares the same pattern. Introduce a unified interface for scanning directories for files-backend.

---
 refs/files-backend.c | 77 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 76 insertions(+), 1 deletion(-)
refs/files-backend.c +76 -1
index ba9e57d1e9..5156530774 100644
@@ -4,6 +4,7 @@
 #include "../gettext.h"
 #include "../hash.h"
 #include "../hex.h"
+#include "../fsck.h"
 #include "../refs.h"
 #include "refs-internal.h"
 #include "ref-cache.h"
@@ -3405,6 +3406,78 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,
 	return ret;
 }
 
+/*
+ * For refs and reflogs, they share a unified interface when scanning
+ * the whole directory. This function is used as the callback for each
+ * regular file or symlink in the directory.
+ */
+typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
+				  const char *gitdir,
+				  const char *refs_check_dir,
+				  struct dir_iterator *iter);
+
+static int files_fsck_refs_dir(struct ref_store *ref_store,
+			       struct fsck_options *o,
+			       const char *refs_check_dir,
+			       files_fsck_refs_fn *fsck_refs_fns)
+{
+	const char *gitdir = ref_store->gitdir;
+	struct strbuf sb = STRBUF_INIT;
+	struct dir_iterator *iter;
+	int iter_status;
+	int ret = 0;
+
+	strbuf_addf(&sb, "%s/%s", gitdir, refs_check_dir);
+
+	iter = dir_iterator_begin(sb.buf, 0);
+
+	if (!iter) {
+		ret = error_errno("cannot open directory %s", sb.buf);
+		goto out;
+	}
+
+	while ((iter_status = dir_iterator_advance(iter)) == ITER_OK) {
+		if (S_ISDIR(iter->st.st_mode)) {
+			continue;
+		} else if (S_ISREG(iter->st.st_mode) ||
+			   S_ISLNK(iter->st.st_mode)) {
+			if (o->verbose)
+				fprintf_ln(stderr, "Checking %s/%s",
+					   refs_check_dir, iter->relative_path);
+			for (size_t i = 0; fsck_refs_fns[i]; i++) {
+				if (fsck_refs_fns[i](o, gitdir, refs_check_dir, iter))
+					ret = -1;
+			}
+		} else {
+			ret = error(_("unexpected file type for '%s'"),
+				    iter->basename);
+		}
+	}
+
+	if (iter_status != ITER_DONE)
+		ret = error(_("failed to iterate over '%s'"), sb.buf);
+
+out:
+	strbuf_release(&sb);
+	return ret;
+}
+
+static int files_fsck_refs(struct ref_store *ref_store,
+			   struct fsck_options *o)
+{
+	int ret;
+	files_fsck_refs_fn fsck_refs_fns[]= {
+		NULL
+	};
+
+	if (o->verbose)
+		fprintf_ln(stderr, "Checking references consistency");
+
+	ret = files_fsck_refs_dir(ref_store, o, "refs", fsck_refs_fns);
+
+	return ret;
+}
+
 static int files_fsck(struct ref_store *ref_store,
 		      struct fsck_options *o)
 {
@@ -3412,7 +3485,9 @@ static int files_fsck(struct ref_store *ref_store,
 	struct files_ref_store *refs =
 		files_downcast(ref_store, REF_STORE_READ, "fsck");
 
-	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o);
+	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o)
+	    | files_fsck_refs(ref_store, o);
+
 	return ret;
 }
 
Signature · 1 line
2.45.2
author
#56 lore raw

PATCH v4 6/7 fsck: add ref name check for files backend

In reply to shejialuo

The git-fsck(1) only implicitly checks the reference, it does not fully check refs with bad format name such as standalone "@" and name ending with ".lock".

In order to provide such checks, add a new fsck message id "badRefName" with default ERROR type. Use existing "check_refname_format" to explicit check the ref name. And add a new unit test to verify the functionality.

---
 Documentation/fsck-msgids.txt |   3 +
 fsck.h                        |   1 +
 refs/files-backend.c          |  20 +++++++
 t/t0602-reffiles-fsck.sh      | 101 ++++++++++++++++++++++++++++++++++
 4 files changed, 125 insertions(+)
 create mode 100755 t/t0602-reffiles-fsck.sh
Documentation/fsck-msgids.txt +3 -0
index f643585a34..dab4012246 100644
@@ -19,6 +19,9 @@
 `badParentSha1`::
 	(ERROR) A commit object has a bad parent sha1.
 
+`badRefName`::
+	(ERROR) A ref has a bad name.
+
 `badTagName`::
 	(INFO) A tag has an invalid format.
 
fsck.h +1 -0
index 0391dffbb0..f26dec2ea4 100644
@@ -31,6 +31,7 @@ enum fsck_msg_type {
 	FUNC(BAD_NAME, ERROR) \
 	FUNC(BAD_OBJECT_SHA1, ERROR) \
 	FUNC(BAD_PARENT_SHA1, ERROR) \
+	FUNC(BAD_REF_NAME, ERROR) \
 	FUNC(BAD_TIMEZONE, ERROR) \
 	FUNC(BAD_TREE, ERROR) \
 	FUNC(BAD_TREE_SHA1, ERROR) \
refs/files-backend.c +20 -0
index 5156530774..5bc233b524 100644
@@ -3416,6 +3416,25 @@ typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
 				  const char *refs_check_dir,
 				  struct dir_iterator *iter);
 
+static int files_fsck_refs_name(struct fsck_options *o,
+				const char *gitdir UNUSED,
+				const char *refs_check_dir,
+				struct dir_iterator *iter)
+{
+	struct strbuf sb = STRBUF_INIT;
+	int ret = 0;
+
+	if (check_refname_format(iter->basename, REFNAME_ALLOW_ONELEVEL)) {
+		strbuf_addf(&sb, "%s/%s", refs_check_dir, iter->relative_path);
+		ret = fsck_refs_report(o, sb.buf,
+				       FSCK_MSG_BAD_REF_NAME,
+				       "invalid refname format");
+	}
+
+	strbuf_release(&sb);
+	return ret;
+}
+
 static int files_fsck_refs_dir(struct ref_store *ref_store,
 			       struct fsck_options *o,
 			       const char *refs_check_dir,
@@ -3467,6 +3486,7 @@ static int files_fsck_refs(struct ref_store *ref_store,
 {
 	int ret;
 	files_fsck_refs_fn fsck_refs_fns[]= {
+		files_fsck_refs_name,
 		NULL
 	};
 
t/t0602-reffiles-fsck.sh new file +101 -0

101 lines not on this page · raw message

Signature · 1 line
2.45.2
author
#57 lore raw

PATCH v4 7/7 fsck: add ref content check for files backend

In reply to shejialuo

Enhance the git-fsck(1) command by adding a check for reference content in the files backend. The new functionality ensures that symrefs, real symbolic link and regular refs are validated correctly.

In order to check the trailing content of the regular refs, add a new parameter trailing to parse_loose_ref_contents.

For symrefs, parse_loose_ref_contents will set the "referent". However, symbolic link could be either absolute or relative. Use "strbuf_add_real_path" to read the symbolic link and convert the relative path to absolute path. Then use "skip_prefix" to make it align with symref "referent".

Thus, the symrefs and symbolic links could share the same interface. Add a new function "files_fsck_symref_target" which aims at checking the following things:

1. whether the pointee is under the refs/ directory.
2. whether the pointee name is correct.
3. whether the pointee path is a wrong type in filesystem.

Last, add the following FSCK MESSAGEs:

1. "badRefContent(ERROR)": A ref has a bad content
2. "badSymrefPointee(ERROR)": The pointee of a symref is bad.
3. "trailingRefContent(WARN)": A ref content has trailing contents.

---
 Documentation/fsck-msgids.txt |   9 +++
 fsck.h                        |   3 +
 refs.c                        |   2 +-
 refs/files-backend.c          | 145 +++++++++++++++++++++++++++++++++-
 refs/refs-internal.h          |   5 +-
 t/t0602-reffiles-fsck.sh      | 110 ++++++++++++++++++++++++++
 6 files changed, 269 insertions(+), 5 deletions(-)
Documentation/fsck-msgids.txt +9 -0
index dab4012246..b1630a478b 100644
@@ -19,9 +19,15 @@
 `badParentSha1`::
 	(ERROR) A commit object has a bad parent sha1.
 
+`badRefContent`::
+	(ERROR) A ref has a bad content.
+
 `badRefName`::
 	(ERROR) A ref has a bad name.
 
+`badSymrefPointee`::
+	(ERROR) The pointee of a symref is bad.
+
 `badTagName`::
 	(INFO) A tag has an invalid format.
 
@@ -167,6 +173,9 @@
 `nullSha1`::
 	(WARN) Tree contains entries pointing to a null sha1.
 
+`trailingRefContent`::
+	(WARN) A ref content has trailing contents.
+
 `treeNotSorted`::
 	(ERROR) A tree is not properly sorted.
 
fsck.h +3 -0
index f26dec2ea4..8afee05f20 100644
@@ -32,6 +32,8 @@ enum fsck_msg_type {
 	FUNC(BAD_OBJECT_SHA1, ERROR) \
 	FUNC(BAD_PARENT_SHA1, ERROR) \
 	FUNC(BAD_REF_NAME, ERROR) \
+	FUNC(BAD_REF_CONTENT, ERROR) \
+	FUNC(BAD_SYMREF_POINTEE, ERROR) \
 	FUNC(BAD_TIMEZONE, ERROR) \
 	FUNC(BAD_TREE, ERROR) \
 	FUNC(BAD_TREE_SHA1, ERROR) \
@@ -72,6 +74,7 @@ enum fsck_msg_type {
 	FUNC(HAS_DOTDOT, WARN) \
 	FUNC(HAS_DOTGIT, WARN) \
 	FUNC(NULL_SHA1, WARN) \
+	FUNC(TRAILING_REF_CONTENT, WARN) \
 	FUNC(ZERO_PADDED_FILEMODE, WARN) \
 	FUNC(NUL_IN_COMMIT, WARN) \
 	FUNC(LARGE_PATHNAME, WARN) \
refs.c +1 -1
index cb3b8ec36a..5fc34d39b1 100644
@@ -1744,7 +1744,7 @@ static int refs_read_special_head(struct ref_store *ref_store,
 	}
 
 	result = parse_loose_ref_contents(content.buf, oid, referent, type,
-					  failure_errno);
+					  failure_errno, NULL);
 
 done:
 	strbuf_release(&full_path);
refs/files-backend.c +143 -2

179 lines not on this page · raw message

refs/refs-internal.h +3 -2
index 280acb7f9e..1126c6102a 100644
@@ -709,11 +709,12 @@ struct ref_store {
 
 /*
  * Parse contents of a loose ref file. *failure_errno maybe be set to EINVAL for
- * invalid contents.
+ * invalid contents. Also *trailing is set to the first character after the
+ * refname or NULL if the referent is not empty.
  */
 int parse_loose_ref_contents(const char *buf, struct object_id *oid,
 			     struct strbuf *referent, unsigned int *type,
-			     int *failure_errno);
+			     int *failure_errno, const char **trailing);
 
 /*
  * Fill in the generic part of refs and add it to our collection of
t/t0602-reffiles-fsck.sh +110 -0

114 lines not on this page · raw message

Signature · 1 line
2.45.2
author
#58 lore raw

v5 cover ref consistency check infra setup

In reply to shejialuo

Hi All, this version have been changed a lot compared with the v4. The mainly changed part is how to integrate the ref checks into the current git-fsck(1) which concentrates on the object database check.

In the previous version, I design the following structures:

  struct fsck_options {
    struct fsck_refs_options refs_options;
    struct fsck_objs_options objs_options;
    enum fsck_msg_type *msg_type;
    unsigned strict:1,
             verbose:1;
  };

Although Junio said It's OK that therea are sub-structures in "fsck_options", I decide not to use the above design. This is because in the current implementation, we either check refs or objects. The above design is a combination. It will give us the feeling that "fsck_options" is used to check both refs and objects.

So I decide to create two new structs "fsck_objects_options" and "fsck_refs_options". So we need to think about how to design this. In my implementation, I totally change the "fsck_options" to be a general options which "fsck_objects_options" and "fsck_refs_options" will incorporate.

However, there are some options related to configs such as "enum fsck_msg_type *msg_type" and "skiplist". I change them into the "fskc.c" as the static variable for singleton pattern.

And then extract the common options into "fsck_options". Thus we could reuse some interfaces when chekcing refs or objects.

At last, solve the review from Junio. Junio hopes that we could use a unified interface to report fsck error messages. This version solves this.

BTY, this patch is based on the commit:

1e1586e4ed (The sixteenth batch, 2024-06-24)

Thanks all

shejialuo (12):
fsck: rename "fsck_options" to "fsck_objects_options"
fsck: use "fsck_configs" to set up configs
fsck: abstract common options for reusing
fsck: add "fsck_refs_options" struct
fsck: add a unified interface for reporting fsck messages
fsck: add "fsck_refs_options" initialization macros
refs: set up ref consistency check infrastructure builtin/refs: add verify subcommand builtin/fsck: add git-refs verify child process
files-backend: add unified interface for refs scanning
fsck: add ref name check for files backend
fsck: add ref content check for files backend

 Documentation/fsck-msgids.txt |  12 ++
 Documentation/git-refs.txt    |  13 ++
 builtin/fsck.c                |  38 +++--
 builtin/index-pack.c          |  18 +--
 builtin/mktag.c               |  15 +-
 builtin/refs.c                |  36 +++++
 builtin/unpack-objects.c      |  14 +-
 fetch-pack.c                  |  12 +-
 fsck.c                        | 197 +++++++++++++++++++-------
 fsck.h                        | 122 ++++++++++++----
 object-file.c                 |  20 +--
 refs.c                        |   7 +-
 refs.h                        |   8 ++
 refs/debug.c                  |  11 ++
 refs/files-backend.c          | 255 +++++++++++++++++++++++++++++++++-
 refs/packed-backend.c         |   8 ++
 refs/refs-internal.h          |  11 +-
 refs/reftable-backend.c       |   8 ++
 t/t0602-reffiles-fsck.sh      | 211 ++++++++++++++++++++++++++++
 19 files changed, 889 insertions(+), 127 deletions(-)
 create mode 100755 t/t0602-reffiles-fsck.sh

Range-diff against v4:

 1:  2d2c1f2cd3 <  -:  ---------- fsck: add refs check interfaces to interact with fsck error levels
 -:  ---------- >  1:  d83b5797cb fsck: rename "fsck_options" to "fsck_objects_options"
 -:  ---------- >  2:  88000b092a fsck: use "fsck_configs" to set up configs
 -:  ---------- >  3:  65093bae64 fsck: abstract common options for reusing
 -:  ---------- >  4:  403750da30 fsck: add "fsck_refs_options" struct
 -:  ---------- >  5:  8dea5654d5 fsck: add a unified interface for reporting fsck messages
 -:  ---------- >  6:  8fb62b54c6 fsck: add "fsck_refs_options" initialization macros
 2:  4bacffe44e !  7:  2ecd1f5407 refs: set up ref consistency check infrastructure
    @@ refs.c: int check_refname_format(const char *refname, int flags)
      	return check_or_sanitize_refname(refname, flags, NULL);
      }
      
    -+int refs_fsck(struct ref_store *refs, struct fsck_options *o)
    ++int refs_fsck(struct ref_store *refs, struct fsck_refs_options *o)
     +{
     +	return refs->be->fsck(refs, o);
     +}
    @@ refs.h
      
      #include "commit.h"
      
    -+struct fsck_options;
    ++struct fsck_refs_options;
      struct object_id;
      struct ref_store;
      struct repository;
    @@ refs.h: int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void
     + * reflogs are consistent, and non-zero otherwise. The errors will be
     + * written to stderr.
     + */
    -+int refs_fsck(struct ref_store *refs, struct fsck_options *o);
    ++int refs_fsck(struct ref_store *refs, struct fsck_refs_options *o);
     +
      /*
       * Apply the rules from check_refname_format, but mutate the result until it
    @@ refs/debug.c: static int debug_reflog_expire(struct ref_store *ref_store, const
      }
      
     +static int debug_fsck(struct ref_store *ref_store,
    -+		      struct fsck_options *o)
    ++		      struct fsck_refs_options *o)
     +{
     +	struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store;
     +	int res = drefs->refs->be->fsck(drefs->refs, o);
    @@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
      }
      
     +static int files_fsck(struct ref_store *ref_store,
    -+		      struct fsck_options *o)
    ++		      struct fsck_refs_options *o)
     +{
     +	int ret;
     +	struct files_ref_store *refs =
    @@ refs/packed-backend.c: static struct ref_iterator *packed_reflog_iterator_begin(
      }
      
     +static int packed_fsck(struct ref_store *ref_store,
    -+		       struct fsck_options *o)
    ++		       struct fsck_refs_options *o)
     +{
     +	return 0;
     +}
    @@ refs/refs-internal.h: typedef int read_raw_ref_fn(struct ref_store *ref_store, c
      				 struct strbuf *referent);
      
     +typedef int fsck_fn(struct ref_store *ref_store,
    -+		    struct fsck_options *o);
    ++		    struct fsck_refs_options *o);
     +
      struct ref_storage_be {
      	const char *name;
    @@ refs/reftable-backend.c: static int reftable_be_reflog_expire(struct ref_store *
      }
      
     +static int reftable_be_fsck(struct ref_store *ref_store,
    -+			    struct fsck_options *o)
    ++			    struct fsck_refs_options *o)
     +{
     +	return 0;
     +}
 3:  f00acab4d1 !  8:  534a3d00af builtin/refs: add verify subcommand
    @@ builtin/refs.c: static int cmd_refs_migrate(int argc, const char **argv, const c
      
     +static int cmd_refs_verify(int argc, const char **argv, const char *prefix)
     +{
    ++	struct fsck_refs_options fsck_refs_options = FSCK_REFS_OPTIONS_DEFAULT;
     +	const char * const verify_usage[] = {
     +		REFS_VERIFY_USAGE,
     +		NULL,
     +	};
    -+	int ret = 0;
     +	unsigned int verbose = 0, strict = 0;
    -+	struct fsck_options fsck_options = FSCK_OPTIONS_DEFAULT;
     +	struct option options[] = {
     +		OPT__VERBOSE(&verbose, N_("be verbose")),
     +		OPT_BOOL(0, "strict", &strict, N_("enable strict checking")),
    @@ builtin/refs.c: static int cmd_refs_migrate(int argc, const char **argv, const c
     +		usage(_("too many arguments"));
     +
     +	if (verbose)
    -+		fsck_options.verbose = 1;
    ++		fsck_refs_options.fsck_options.verbose = 1;
     +	if (strict)
    -+		fsck_options.strict = 1;
    ++		fsck_refs_options.fsck_options.strict = 1;
     +
    -+	git_config(git_fsck_config, &fsck_options);
    ++	git_config(git_fsck_config, &fsck_refs_options.fsck_options);
     +	prepare_repo_settings(the_repository);
     +
    -+	ret = refs_fsck(get_main_ref_store(the_repository), &fsck_options);
    -+
    -+	/*
    -+	 * Explicitly free the allocated array and object skiplist set. Because
    -+	 * we reuse `git_fsck_config` here. It will still set the skiplist.
    -+	 */
    -+	free(fsck_options.msg_type);
    -+	oidset_clear(&fsck_options.objs_options.skiplist);
    -+	return ret;
    ++	return refs_fsck(get_main_ref_store(the_repository), &fsck_refs_options);
     +}
     +
      int cmd_refs(int argc, const char **argv, const char *prefix)
 4:  6494f5b0c2 =  9:  054c8cea89 builtin/fsck: add `git-refs verify` child process
 5:  2c064f4f71 ! 10:  7f87c6a26e files-backend: add unified interface for refs scanning
    @@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
     + * the whole directory. This function is used as the callback for each
     + * regular file or symlink in the directory.
     + */
    -+typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
    ++typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o,
     +				  const char *gitdir,
     +				  const char *refs_check_dir,
     +				  struct dir_iterator *iter);
     +
     +static int files_fsck_refs_dir(struct ref_store *ref_store,
    -+			       struct fsck_options *o,
    ++			       struct fsck_refs_options *o,
     +			       const char *refs_check_dir,
     +			       files_fsck_refs_fn *fsck_refs_fns)
     +{
    @@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
     +			continue;
     +		} else if (S_ISREG(iter->st.st_mode) ||
     +			   S_ISLNK(iter->st.st_mode)) {
    -+			if (o->verbose)
    ++			if (o->fsck_options.verbose)
     +				fprintf_ln(stderr, "Checking %s/%s",
     +					   refs_check_dir, iter->relative_path);
     +			for (size_t i = 0; fsck_refs_fns[i]; i++) {
    @@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
     +}
     +
     +static int files_fsck_refs(struct ref_store *ref_store,
    -+			   struct fsck_options *o)
    ++			   struct fsck_refs_options *o)
     +{
     +	int ret;
     +	files_fsck_refs_fn fsck_refs_fns[]= {
     +		NULL
     +	};
     +
    -+	if (o->verbose)
    ++	if (o->fsck_options.verbose)
     +		fprintf_ln(stderr, "Checking references consistency");
     +
     +	ret = files_fsck_refs_dir(ref_store, o, "refs", fsck_refs_fns);
    @@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
     +}
     +
      static int files_fsck(struct ref_store *ref_store,
    - 		      struct fsck_options *o)
    + 		      struct fsck_refs_options *o)
      {
     @@ refs/files-backend.c: static int files_fsck(struct ref_store *ref_store,
      	struct files_ref_store *refs =
 6:  88b54b9e1a ! 11:  7d78014e5f fsck: add ref name check for files backend
    @@ fsck.h: enum fsck_msg_type {
      	FUNC(BAD_TREE_SHA1, ERROR) \
     
      ## refs/files-backend.c ##
    -@@ refs/files-backend.c: typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
    +@@ refs/files-backend.c: typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o,
      				  const char *refs_check_dir,
      				  struct dir_iterator *iter);
      
    -+static int files_fsck_refs_name(struct fsck_options *o,
    ++static int files_fsck_refs_name(struct fsck_refs_options *o,
     +				const char *gitdir UNUSED,
     +				const char *refs_check_dir,
     +				struct dir_iterator *iter)
    @@ refs/files-backend.c: typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
     +
     +	if (check_refname_format(iter->basename, REFNAME_ALLOW_ONELEVEL)) {
     +		strbuf_addf(&sb, "%s/%s", refs_check_dir, iter->relative_path);
    -+		ret = fsck_refs_report(o, sb.buf,
    ++		ret = fsck_refs_report(o, NULL, sb.buf,
     +				       FSCK_MSG_BAD_REF_NAME,
     +				       "invalid refname format");
     +	}
    @@ refs/files-backend.c: typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
     +}
     +
      static int files_fsck_refs_dir(struct ref_store *ref_store,
    - 			       struct fsck_options *o,
    + 			       struct fsck_refs_options *o,
      			       const char *refs_check_dir,
     @@ refs/files-backend.c: static int files_fsck_refs(struct ref_store *ref_store,
      {
 7:  8309a4746a ! 12:  16209a73ed fsck: add ref content check for files backend
    @@ refs/files-backend.c: int parse_loose_ref_contents(const char *buf, struct objec
      	return 0;
      }
      
    -@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
    +@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *o,
      	return ret;
      }
      
    @@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
     + * would be the content after "refs:". For symblic link, "pointee_name" would
     + * be the relative path agaignst "gitdir".
     + */
    -+static int files_fsck_symref_target(struct fsck_options *o,
    ++static int files_fsck_symref_target(struct fsck_refs_options *o,
     +				    const char *refname,
     +				    const char *pointee_name,
     +				    const char *pointee_path)
    @@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
     +
     +	if (!skip_prefix(pointee_name, "refs/", &p)) {
     +
    -+		ret = fsck_refs_report(o, refname,
    ++		ret = fsck_refs_report(o, NULL, refname,
     +				       FSCK_MSG_BAD_SYMREF_POINTEE,
     +				       "point to target out of refs hierarchy");
     +		goto out;
     +	}
     +
     +	if (check_refname_format(pointee_name, 0)) {
    -+		ret = fsck_refs_report(o, refname,
    ++		ret = fsck_refs_report(o, NULL, refname,
     +				       FSCK_MSG_BAD_SYMREF_POINTEE,
     +				       "point to invalid refname");
     +	}
    @@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
     +		goto out;
     +
     +	if (!S_ISREG(st.st_mode) && !S_ISLNK(st.st_mode)) {
    -+		ret = fsck_refs_report(o, refname,
    ++		ret = fsck_refs_report(o, NULL, refname,
     +				       FSCK_MSG_BAD_SYMREF_POINTEE,
     +				       "point to invalid target");
     +		goto out;
    @@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
     +	return ret;
     +}
     +
    -+static int files_fsck_refs_content(struct fsck_options *o,
    ++static int files_fsck_refs_content(struct fsck_refs_options *o,
     +				   const char *gitdir,
     +				   const char *refs_check_dir,
     +				   struct dir_iterator *iter)
    @@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
     +
     +		if (!skip_prefix(pointee_path.buf,
     +				 abs_gitdir.buf, &pointee_name)) {
    -+			ret = fsck_refs_report(o, refname.buf,
    ++			ret = fsck_refs_report(o, NULL, refname.buf,
     +					       FSCK_MSG_BAD_SYMREF_POINTEE,
     +					       "point to target outside gitdir");
     +			goto clean;
    @@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
     +	if (parse_loose_ref_contents(ref_content.buf, &oid,
     +				     &referent, &type,
     +				     &failure_errno, &trailing)) {
    -+		ret = fsck_refs_report(o, refname.buf,
    ++		ret = fsck_refs_report(o, NULL, refname.buf,
     +				       FSCK_MSG_BAD_REF_CONTENT,
     +				       "invalid ref content");
     +		goto clean;
    @@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
     +		 * be reported as a warning.
     +		 */
     +		if (trailing && (*trailing != '\0' && *trailing != '\n')) {
    -+			ret = fsck_refs_report(o, refname.buf,
    ++			ret = fsck_refs_report(o, NULL, refname.buf,
     +					       FSCK_MSG_TRAILING_REF_CONTENT,
     +					       "trailing garbage in ref");
     +			goto clean;
    @@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
     +}
     +
      static int files_fsck_refs_dir(struct ref_store *ref_store,
    - 			       struct fsck_options *o,
    + 			       struct fsck_refs_options *o,
      			       const char *refs_check_dir,
     @@ refs/files-backend.c: static int files_fsck_refs(struct ref_store *ref_store,
      	int ret;
Signature · 1 line
2.45.2
author
#59 lore raw

PATCH v5 1/12 fsck: rename "fsck_options" to "fsck_objects_options"

The git-fsck(1) focuses on object database consistency check. It relies on the "fsck_options" to interact with fsck error levels. However, "fsck_options" aims at checking the object database which contains a lot of fields only related to object database.

In order to provide git-fsck(1) with refs consistency check. Rename "fsck_options" to "fsck_objects_options" to explicitly indicate this structure is used to check the object database.

---
 builtin/fsck.c           | 13 ++++----
 builtin/index-pack.c     | 18 +++++------
 builtin/mktag.c          | 13 ++++----
 builtin/unpack-objects.c | 14 ++++-----
 fetch-pack.c             | 12 ++++----
 fsck.c                   | 64 ++++++++++++++++++++++------------------
 fsck.h                   | 43 ++++++++++++++-------------
 object-file.c            |  4 +--
 8 files changed, 95 insertions(+), 86 deletions(-)
builtin/fsck.c +7 -6
index d13a226c2e..ec3220880d 100644
@@ -42,8 +42,8 @@ static int check_full = 1;
 static int connectivity_only;
 static int check_strict;
 static int keep_cache_objects;
-static struct fsck_options fsck_walk_options = FSCK_OPTIONS_DEFAULT;
-static struct fsck_options fsck_obj_options = FSCK_OPTIONS_DEFAULT;
+static struct fsck_objects_options fsck_walk_options = FSCK_OBJECTS_OPTIONS_DEFAULT;
+static struct fsck_objects_options fsck_obj_options = FSCK_OBJECTS_OPTIONS_DEFAULT;
 static int errors_found;
 static int write_lost_and_found;
 static int verbose;
@@ -89,7 +89,7 @@ static int objerror(struct object *obj, const char *err)
 	return -1;
 }
 
-static int fsck_error_func(struct fsck_options *o UNUSED,
+static int fsck_error_func(struct fsck_objects_options *o UNUSED,
 			   const struct object_id *oid,
 			   enum object_type object_type,
 			   enum fsck_msg_type msg_type,
@@ -118,7 +118,7 @@ static int fsck_error_func(struct fsck_options *o UNUSED,
 static struct object_array pending;
 
 static int mark_object(struct object *obj, enum object_type type,
-		       void *data, struct fsck_options *options UNUSED)
+		       void *data, struct fsck_objects_options *options UNUSED)
 {
 	struct object *parent = data;
 
@@ -204,7 +204,8 @@ static int traverse_reachable(void)
 }
 
 static int mark_used(struct object *obj, enum object_type type UNUSED,
-		     void *data UNUSED, struct fsck_options *options UNUSED)
+		     void *data UNUSED,
+		     struct fsck_objects_options *options UNUSED)
 {
 	if (!obj)
 		return 1;
@@ -214,7 +215,7 @@ static int mark_used(struct object *obj, enum object_type type UNUSED,
 
 static void mark_unreachable_referents(const struct object_id *oid)
 {
-	struct fsck_options options = FSCK_OPTIONS_DEFAULT;
+	struct fsck_objects_options options = FSCK_OBJECTS_OPTIONS_DEFAULT;
 	struct object *obj = lookup_object(the_repository, oid);
 
 	if (!obj || !(obj->flags & HAS_OBJ))
builtin/index-pack.c +9 -9
index 856428fef9..08ebeedfd3 100644
@@ -127,7 +127,7 @@ static int nr_threads;
 static int from_stdin;
 static int strict;
 static int do_fsck_object;
-static struct fsck_options fsck_options = FSCK_OPTIONS_MISSING_GITMODULES;
+static struct fsck_objects_options fsck_objects_options = FSCK_OBJECTS_OPTIONS_MISSING_GITMODULES;
 static int verbose;
 static const char *progress_title;
 static int show_resolving_progress;
@@ -220,7 +220,7 @@ static void cleanup_thread(void)
 
 static int mark_link(struct object *obj, enum object_type type,
 		     void *data UNUSED,
-		     struct fsck_options *options UNUSED)
+		     struct fsck_objects_options *options UNUSED)
 {
 	if (!obj)
 		return -1;
@@ -852,7 +852,7 @@ static void sha1_object(const void *data, struct object_entry *obj_entry,
 			else
 				die(_("invalid blob object %s"), oid_to_hex(oid));
 			if (do_fsck_object &&
-			    fsck_object(&blob->object, (void *)data, size, &fsck_options))
+			    fsck_object(&blob->object, (void *)data, size, &fsck_objects_options))
 				die(_("fsck error in packed object"));
 		} else {
 			struct object *obj;
@@ -871,9 +871,9 @@ static void sha1_object(const void *data, struct object_entry *obj_entry,
 			if (!obj)
 				die(_("invalid %s"), type_name(type));
 			if (do_fsck_object &&
-			    fsck_object(obj, buf, size, &fsck_options))
+			    fsck_object(obj, buf, size, &fsck_objects_options))
 				die(_("fsck error in packed object"));
-			if (strict && fsck_walk(obj, NULL, &fsck_options))
+			if (strict && fsck_walk(obj, NULL, &fsck_objects_options))
 				die(_("Not all child objects of %s are reachable"), oid_to_hex(&obj->oid));
 
 			if (obj->type == OBJ_TREE) {
@@ -1746,7 +1746,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
 		usage(index_pack_usage);
 
 	disable_replace_refs();
-	fsck_options.walk = mark_link;
+	fsck_objects_options.walk = mark_link;
 
 	reset_pack_idx_option(&opts);
 	opts.flags |= WRITE_REV;
@@ -1770,13 +1770,13 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
 			} else if (skip_to_optional_arg(arg, "--strict", &arg)) {
 				strict = 1;
 				do_fsck_object = 1;
-				fsck_set_msg_types(&fsck_options, arg);
+				fsck_set_msg_types(&fsck_objects_options, arg);
 			} else if (!strcmp(arg, "--check-self-contained-and-connected")) {
 				strict = 1;
 				check_self_contained_and_connected = 1;
 			} else if (skip_to_optional_arg(arg, "--fsck-objects", &arg)) {
 				do_fsck_object = 1;
-				fsck_set_msg_types(&fsck_options, arg);
+				fsck_set_msg_types(&fsck_objects_options, arg);
 			} else if (!strcmp(arg, "--verify")) {
 				verify = 1;
 			} else if (!strcmp(arg, "--verify-stat")) {
@@ -1943,7 +1943,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
 	else
 		close(input_fd);
 
-	if (do_fsck_object && fsck_finish(&fsck_options))
+	if (do_fsck_object && fsck_finish(&fsck_objects_options))
 		die(_("fsck error in pack objects"));
 
 	free(opts.anomaly);
builtin/mktag.c +7 -6
index 4767f1a97e..be2abc71d8 100644
@@ -15,9 +15,9 @@ static char const * const builtin_mktag_usage[] = {
 };
 static int option_strict = 1;
 
-static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
+static struct fsck_objects_options fsck_objects_options = FSCK_OBJECTS_OPTIONS_STRICT;
 
-static int mktag_fsck_error_func(struct fsck_options *o UNUSED,
+static int mktag_fsck_error_func(struct fsck_objects_options *o UNUSED,
 				 const struct object_id *oid UNUSED,
 				 enum object_type object_type UNUSED,
 				 enum fsck_msg_type msg_type,
@@ -91,12 +91,13 @@ int cmd_mktag(int argc, const char **argv, const char *prefix)
 	if (strbuf_read(&buf, 0, 0) < 0)
 		die_errno(_("could not read from stdin"));
 
-	fsck_options.error_func = mktag_fsck_error_func;
-	fsck_set_msg_type_from_ids(&fsck_options, FSCK_MSG_EXTRA_HEADER_ENTRY,
+	fsck_objects_options.error_func = mktag_fsck_error_func;
+	fsck_set_msg_type_from_ids(&fsck_objects_options,
+				   FSCK_MSG_EXTRA_HEADER_ENTRY,
 				   FSCK_WARN);
 	/* config might set fsck.extraHeaderEntry=* again */
-	git_config(git_fsck_config, &fsck_options);
-	if (fsck_tag_standalone(NULL, buf.buf, buf.len, &fsck_options,
+	git_config(git_fsck_config, &fsck_objects_options);
+	if (fsck_tag_standalone(NULL, buf.buf, buf.len, &fsck_objects_options,
 				&tagged_oid, &tagged_type))
 		die(_("tag on stdin did not pass our strict fsck check"));
 
builtin/unpack-objects.c +7 -7
index f1c85a00ae..c59e330db9 100644
@@ -25,7 +25,7 @@ static unsigned int offset, len;
 static off_t consumed_bytes;
 static off_t max_input_size;
 static git_hash_ctx ctx;
-static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
+static struct fsck_objects_options fsck_objects_options = FSCK_OBJECTS_OPTIONS_STRICT;
 static struct progress *progress;
 
 /*
@@ -212,7 +212,7 @@ static void write_cached_object(struct object *obj, struct obj_buffer *obj_buf)
  */
 static int check_object(struct object *obj, enum object_type type,
 			void *data UNUSED,
-			struct fsck_options *options UNUSED)
+			struct fsck_objects_options *options UNUSED)
 {
 	struct obj_buffer *obj_buf;
 
@@ -237,10 +237,10 @@ static int check_object(struct object *obj, enum object_type type,
 	obj_buf = lookup_object_buffer(obj);
 	if (!obj_buf)
 		die("Whoops! Cannot find object '%s'", oid_to_hex(&obj->oid));
-	if (fsck_object(obj, obj_buf->buffer, obj_buf->size, &fsck_options))
+	if (fsck_object(obj, obj_buf->buffer, obj_buf->size, &fsck_objects_options))
 		die("fsck error in packed object");
-	fsck_options.walk = check_object;
-	if (fsck_walk(obj, NULL, &fsck_options))
+	fsck_objects_options.walk = check_object;
+	if (fsck_walk(obj, NULL, &fsck_objects_options))
 		die("Error on reachable objects of %s", oid_to_hex(&obj->oid));
 	write_cached_object(obj, obj_buf);
 	return 0;
@@ -635,7 +635,7 @@ int cmd_unpack_objects(int argc, const char **argv, const char *prefix UNUSED)
 			}
 			if (skip_prefix(arg, "--strict=", &arg)) {
 				strict = 1;
-				fsck_set_msg_types(&fsck_options, arg);
+				fsck_set_msg_types(&fsck_objects_options, arg);
 				continue;
 			}
 			if (starts_with(arg, "--pack_header=")) {
@@ -671,7 +671,7 @@ int cmd_unpack_objects(int argc, const char **argv, const char *prefix UNUSED)
 	the_hash_algo->final_oid_fn(&oid, &tmp_ctx);
 	if (strict) {
 		write_rest();
-		if (fsck_finish(&fsck_options))
+		if (fsck_finish(&fsck_objects_options))
 			die(_("fsck error in pack objects"));
 	}
 	if (!hasheq(fill(the_hash_algo->rawsz), oid.hash))
fetch-pack.c +6 -6
index eba9e420ea..7d1e4a5087 100644
@@ -46,7 +46,7 @@ static int server_supports_filtering;
 static int advertise_sid;
 static struct shallow_lock shallow_lock;
 static const char *alternate_shallow_file;
-static struct fsck_options fsck_options = FSCK_OPTIONS_MISSING_GITMODULES;
+static struct fsck_objects_options fsck_objects_options = FSCK_OBJECTS_OPTIONS_MISSING_GITMODULES;
 static struct strbuf fsck_msg_types = STRBUF_INIT;
 static struct string_list uri_protocols = STRING_LIST_INIT_DUP;
 
@@ -1220,9 +1220,9 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 	} else
 		alternate_shallow_file = NULL;
 	if (get_pack(args, fd, pack_lockfiles, NULL, sought, nr_sought,
-		     &fsck_options.gitmodules_found))
+		     &fsck_objects_options.gitmodules_found))
 		die(_("git fetch-pack: fetch failed."));
-	if (fsck_finish(&fsck_options))
+	if (fsck_finish(&fsck_objects_options))
 		die("fsck failed");
 
  all_done:
@@ -1780,7 +1780,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
 
 			if (get_pack(args, fd, pack_lockfiles,
 				     packfile_uris.nr ? &index_pack_args : NULL,
-				     sought, nr_sought, &fsck_options.gitmodules_found))
+				     sought, nr_sought, &fsck_objects_options.gitmodules_found))
 				die(_("git fetch-pack: fetch failed."));
 			do_check_stateless_delimiter(args->stateless_rpc, &reader);
 
@@ -1823,7 +1823,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
 
 		packname[the_hash_algo->hexsz] = '\0';
 
-		parse_gitmodules_oids(cmd.out, &fsck_options.gitmodules_found);
+		parse_gitmodules_oids(cmd.out, &fsck_objects_options.gitmodules_found);
 
 		close(cmd.out);
 
@@ -1844,7 +1844,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
 	string_list_clear(&packfile_uris, 0);
 	strvec_clear(&index_pack_args);
 
-	if (fsck_finish(&fsck_options))
+	if (fsck_finish(&fsck_objects_options))
 		die("fsck failed");
 
 	if (negotiator)
fsck.c +35 -29

237 lines not on this page · raw message

fsck.h +22 -21

133 lines not on this page · raw message

object-file.c +2 -2
index d3cf4b8b2e..ec44ac3d82 100644
@@ -2472,7 +2472,7 @@ int repo_has_object_file(struct repository *r,
  * report the minimal fsck error here, and rely on the caller to
  * give more context.
  */
-static int hash_format_check_report(struct fsck_options *opts UNUSED,
+static int hash_format_check_report(struct fsck_objects_options *opts UNUSED,
 				     const struct object_id *oid UNUSED,
 				     enum object_type object_type UNUSED,
 				     enum fsck_msg_type msg_type UNUSED,
@@ -2507,7 +2507,7 @@ static int index_mem(struct index_state *istate,
 		}
 	}
 	if (flags & HASH_FORMAT_CHECK) {
-		struct fsck_options opts = FSCK_OPTIONS_DEFAULT;
+		struct fsck_objects_options opts = FSCK_OBJECTS_OPTIONS_DEFAULT;
 
 		opts.strict = 1;
 		opts.error_func = hash_format_check_report;
Signature · 1 line
2.45.2
#60 lore raw

Re: [GSoC][PATCH v5 01/12] fsck: rename "fsck_options" to "fsck_objects_options"

builtin/fsck.c static int objerror(struct object *obj, const char *err) · line 95

Quoted 5 lines · shejialuo
builtin/fsck.c +1 -1
-static int fsck_error_func(struct fsck_options *o UNUSED,
+static int fsck_error_func(struct fsck_objects_options *o UNUSED,
 			   const struct object_id *oid,
 			   enum object_type object_type,
 			   enum fsck_msg_type msg_type,

It is curious that the addition/renaming of fsck_objects_options is presumably to allow fsck_${xyzzy}_options to be added for different $xyzzy (the first one being "refs"), and this function is only about fsck_objects_options. What name would the corresponding error function, called by checkers that take fsck_${xyzzy}_options, be given? fsck_${xyzzy}_error_func()? Shouldn't this be then become fsck_objects_error_func() or something?

Having said that.

Do we really need such a parallel system between "objects" and other kinds of things being checked that you are introducing with this step? What benefit are we getting from this additional complexity?

I would have expected that adding ref-related new members that object consistency checkers has no interest in to the fsck_options structure would be sufficient for the purpose of this series. Or if we really wanted to prepare for more complex future, use of the "union of variants, switched with a tag" pattern to arrange the data this way:

	struct fsck_options {
		enum fsck_type {
			FSCK_OBJECTS,
			FSCK_REFS,
			...
		} t;
		union {
			struct fsck_objects_options objects;
			struct fsck_refs_options refs;
		} u;
	};

would still allow functions like fsck_error_func(), and fsck_set_msg_types(), etc. to work on the common "fsck_options".

I dunno.

author
#61 lore raw

Re: [GSoC][PATCH v5 01/12] fsck: rename "fsck_options" to "fsck_objects_options"

Quoted 16 lines · Junio C Hamano

builtin/fsck.c static int objerror(struct object *obj, const char *err) · line 95

Quoted 5 lines · shejialuo
builtin/fsck.c +1 -1
-static int fsck_error_func(struct fsck_options *o UNUSED,
+static int fsck_error_func(struct fsck_objects_options *o UNUSED,
 			   const struct object_id *oid,
 			   enum object_type object_type,
 			   enum fsck_msg_type msg_type,

It is curious that the addition/renaming of fsck_objects_options is presumably to allow fsck_${xyzzy}_options to be added for different $xyzzy (the first one being "refs"), and this function is only about fsck_objects_options. What name would the corresponding error function, called by checkers that take fsck_${xyzzy}_options, be given? fsck_${xyzzy}_error_func()? Shouldn't this be then become fsck_objects_error_func() or something?

Yes, it should be definitely changed here. Will improve in the next version.

Quoted 6 lines

Having said that.

Do we really need such a parallel system between "objects" and other kinds of things being checked that you are introducing with this step? What benefit are we getting from this additional complexity?

I am agree that the most simple way to handle for this series is add some ref-related new members. Thus, we can reuse existing code. However, it makes me feel so weird when implementing the code using this idea. For example,

  struct fsck_options {
    struct fsck_refs_options;
    ...
  }

When we create a new "fsck_options", it will be so misleading that the caller may think we will handle both refs and objects checks by using "fsck_options". So I just introduce this parallel system. When checking objects, caller should explicitly create "fsck_objects_options", when checking refs, caller should explicitly create "fsck_refs_options".

Because in semantics, we introduce a new check here. Combination means we will check the both. Although it is simple, but it will cause a lot of trouble in the future.

Quoted 19 lines

I would have expected that adding ref-related new members that object consistency checkers has no interest in to the fsck_options structure would be sufficient for the purpose of this series. Or if we really wanted to prepare for more complex future, use of the "union of variants, switched with a tag" pattern to arrange the data this way:

	struct fsck_options {
		enum fsck_type {
			FSCK_OBJECTS,
			FSCK_REFS,
			...
		} t;
		union {
			struct fsck_objects_options objects;
			struct fsck_refs_options refs;
		} u;
	};
Quoted 3 lines

would still allow functions like fsck_error_func(), and fsck_set_msg_types(), etc. to work on the common "fsck_options".

I agree that we could use this pattern, using union will make the semantics more clear.

Quoted 1 line

I dunno.

author
#62 lore raw

PATCH v5 2/12 fsck: use "fsck_configs" to set up configs

In reply to shejialuo

Some fields such as "msg_type" and "skiplist" in "fsck_objects_options" are not options, these fields are related to "git-config(1)" which are initialized using "git_fsck_config" function. Create a static variable named "fsck_configs" in "fsck.c" which aims at handling configs. Thus we don't need to reply on the "fsck_objects_options" to set up the fsck error message severity.

---
 fsck.c | 29 ++++++++++++++++++-----------
 fsck.h |  3 ---
 2 files changed, 18 insertions(+), 14 deletions(-)
fsck.c +18 -11
index c24a0f9fae..81b93f02fc 100644
@@ -24,6 +24,14 @@
 
 static ssize_t max_tree_entry_len = 4096;
 
+static struct {
+	enum fsck_msg_type *msg_type;
+	struct oidset oid_skiplist;
+} fsck_configs = {
+	.msg_type = NULL,
+	.oid_skiplist = OIDSET_INIT
+};
+
 #define STR(x) #x
 #define MSG_ID(id, msg_type) { STR(id), NULL, NULL, FSCK_##msg_type },
 static struct {
@@ -103,7 +111,7 @@ static enum fsck_msg_type fsck_msg_type(enum fsck_msg_id msg_id,
 {
 	assert(msg_id >= 0 && msg_id < FSCK_MSG_MAX);
 
-	if (!options->msg_type) {
+	if (!fsck_configs.msg_type) {
 		enum fsck_msg_type msg_type = msg_id_info[msg_id].msg_type;
 
 		if (options->strict && msg_type == FSCK_WARN)
@@ -111,7 +119,7 @@ static enum fsck_msg_type fsck_msg_type(enum fsck_msg_id msg_id,
 		return msg_type;
 	}
 
-	return options->msg_type[msg_id];
+	return fsck_configs.msg_type[msg_id];
 }
 
 static enum fsck_msg_type parse_msg_type(const char *str)
@@ -138,16 +146,16 @@ void fsck_set_msg_type_from_ids(struct fsck_objects_options *options,
 				enum fsck_msg_id msg_id,
 				enum fsck_msg_type msg_type)
 {
-	if (!options->msg_type) {
+	if (!fsck_configs.msg_type) {
 		int i;
 		enum fsck_msg_type *severity;
 		ALLOC_ARRAY(severity, FSCK_MSG_MAX);
 		for (i = 0; i < FSCK_MSG_MAX; i++)
 			severity[i] = fsck_msg_type(i, options);
-		options->msg_type = severity;
+		fsck_configs.msg_type = severity;
 	}
 
-	options->msg_type[msg_id] = msg_type;
+	fsck_configs.msg_type[msg_id] = msg_type;
 }
 
 void fsck_set_msg_type(struct fsck_objects_options *options,
@@ -203,7 +211,7 @@ void fsck_set_msg_types(struct fsck_objects_options *options, const char *values
 		if (!strcmp(buf, "skiplist")) {
 			if (equal == len)
 				die("skiplist requires a path");
-			oidset_parse_file(&options->skiplist, buf + equal + 1);
+			oidset_parse_file(&fsck_configs.oid_skiplist, buf + equal + 1);
 			buf += len + 1;
 			continue;
 		}
@@ -217,10 +225,9 @@ void fsck_set_msg_types(struct fsck_objects_options *options, const char *values
 	free(to_free);
 }
 
-static int object_on_skiplist(struct fsck_objects_options *opts,
-			      const struct object_id *oid)
+static int object_on_skiplist(const struct object_id *oid)
 {
-	return opts && oid && oidset_contains(&opts->skiplist, oid);
+	return oid && oidset_contains(&fsck_configs.oid_skiplist, oid);
 }
 
 __attribute__((format (printf, 5, 6)))
@@ -236,7 +243,7 @@ static int report(struct fsck_objects_options *options,
 	if (msg_type == FSCK_IGNORE)
 		return 0;
 
-	if (object_on_skiplist(options, oid))
+	if (object_on_skiplist(oid))
 		return 0;
 
 	if (msg_type == FSCK_FATAL)
@@ -1109,7 +1116,7 @@ static int fsck_blob(const struct object_id *oid, const char *buf,
 {
 	int ret = 0;
 
-	if (object_on_skiplist(options, oid))
+	if (object_on_skiplist(oid))
 		return 0;
 
 	if (oidset_contains(&options->gitmodules_found, oid)) {
fsck.h +0 -3
index b64164db17..37deadc4bd 100644
@@ -135,8 +135,6 @@ struct fsck_objects_options {
 	fsck_walk_func walk;
 	fsck_error error_func;
 	unsigned strict:1;
-	enum fsck_msg_type *msg_type;
-	struct oidset skiplist;
 	struct oidset gitmodules_found;
 	struct oidset gitmodules_done;
 	struct oidset gitattributes_found;
@@ -145,7 +143,6 @@ struct fsck_objects_options {
 };
 
 #define FSCK_OBJECTS_OPTIONS_DEFAULT { \
-	.skiplist = OIDSET_INIT, \
 	.gitmodules_found = OIDSET_INIT, \
 	.gitmodules_done = OIDSET_INIT, \
 	.gitattributes_found = OIDSET_INIT, \
Signature · 1 line
2.45.2
#63 lore raw

Re: [GSoC][PATCH v5 02/12] fsck: use "fsck_configs" to set up configs

Quoted 6 lines · shejialuo

Some fields such as "msg_type" and "skiplist" in "fsck_objects_options" are not options, these fields are related to "git-config(1)" which are initialized using "git_fsck_config" function. Create a static variable named "fsck_configs" in "fsck.c" which aims at handling configs. Thus we don't need to reply on the "fsck_objects_options" to set up the fsck error message severity.

reply???

As configuration often is used to prepopulate options, I need a lot stonger justification to split these into a different structure than "'config' is a noun that is different from a noun 'option'".

If we intend to have many "option" instances but what these two members store will be the same across these "option" instances, for example, that would be a lot better reason why we may want to separate these two members out of it, but I have a suspicion that if we were to use the "union with tags" approach, these two would become members of the common part shared between "objects' and "refs", i.e. the overall data structure might look more like this:

	struct fsck_options {
		enum fsck_msg_type *msg_type;
		struct oidset oid_skiplist;
		enum fsck_what_check { O, R } tag;
		union {
			struct fsck_object_options o;
			struct fsck_ref_options r;
		} u;
	};

by moving these two members out of fsck_object_options and moving them to the shared part.

I dunno. It is unclear what the real reason is for these two things to be extracted out and made to appear totally independent from the "options" thing to begin with, and I am not sure if I agree with the reason when it is known.

author
#64 lore raw

Re: [GSoC][PATCH v5 02/12] fsck: use "fsck_configs" to set up configs

Quoted 11 lines · Junio C Hamano
Quoted 6 lines · shejialuo

Some fields such as "msg_type" and "skiplist" in "fsck_objects_options" are not options, these fields are related to "git-config(1)" which are initialized using "git_fsck_config" function. Create a static variable named "fsck_configs" in "fsck.c" which aims at handling configs. Thus we don't need to reply on the "fsck_objects_options" to set up the fsck error message severity.

reply???

Sorry, I often make mistake to type "rely" as "reply".

Quoted 4 lines

As configuration often is used to prepopulate options, I need a lot stonger justification to split these into a different structure than "'config' is a noun that is different from a noun 'option'".

Quoted 8 lines

If we intend to have many "option" instances but what these two members store will be the same across these "option" instances, for example, that would be a lot better reason why we may want to separate these two members out of it, but I have a suspicion that if we were to use the "union with tags" approach, these two would become members of the common part shared between "objects' and "refs", i.e. the overall data structure might look more like this:

Actually, I feel really wired for this part. Let me elaborate on this. "fsck.c::git_fsck_config()" is used to set up the configs. It will eventually call the "fsck.c::fsck_set_msg_type_from_ids" like the following:

  void fsck_set_msg_type_from_ids(struct fsck_options *options,
                                  enum fsck_msg_id msg_id,
                                  enum fsck_msg_type msg_type)
  {
    if (!options->msg_type) {
      int i;
      enum fsck_msg_type *severity.
      ALLOC_ARRAY(severity, FSCK_MSG_MAX);
      for (i = 0; i < FSCK_MSG_MAX; i++)
        severity[i] = fsck_msg_type(i, options);
      options->msg_type = severity;
    }
   options->msg_type[msg_id] = msg_type;
  }

In the current codebase, the caller will simply create a "fsck_options" and setup the fsck error message severity. However, let's see "builtin/fskc.c", it creates the following two "fsck_options" and it only uses

  static struct fsck_options fsck_walk_options = FSCK_OPTIONS_DEFAULT;
  static struct fsck_options fsck_obj_options = FSCK_OPTIONS_DEFAULT;

However, the code only uses "fsck_obj_options" to setup the configs. So it makes me feel so strange. So I just want to make it separation. Maybe a little wrong here.

Quoted 17 lines
	struct fsck_options {
		enum fsck_msg_type *msg_type;
		struct oidset oid_skiplist;
		enum fsck_what_check { O, R } tag;
		union {
			struct fsck_object_options o;
			struct fsck_ref_options r;
		} u;
	};

by moving these two members out of fsck_object_options and moving them to the shared part.

I dunno. It is unclear what the real reason is for these two things to be extracted out and made to appear totally independent from the "options" thing to begin with, and I am not sure if I agree with the reason when it is known.

author
#65 lore raw

PATCH v5 3/12 fsck: abstract common options for reusing

In reply to shejialuo

Create "fsck_options" struct to contain the general fields. Change "fsck_objects_options" to incorporate the general "fsck_options" and object-speicifc options.

The fsck message will use "strict" option to set the corresponding "msg_type". Rename the parameter name from "struct fsck_objects_options *" to "struct fsck_options *".

---
 builtin/fsck.c           |  6 +++---
 builtin/index-pack.c     |  4 ++--
 builtin/mktag.c          |  6 +++---
 builtin/unpack-objects.c |  2 +-
 fsck.c                   | 25 ++++++++++++++++---------
 fsck.h                   | 32 ++++++++++++++++++++++----------
 object-file.c            |  4 ++--
 7 files changed, 49 insertions(+), 30 deletions(-)
builtin/fsck.c +3 -3
index ec3220880d..c383125027 100644
@@ -939,9 +939,9 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
 
 	fsck_walk_options.walk = mark_object;
 	fsck_obj_options.walk = mark_used;
-	fsck_obj_options.error_func = fsck_error_func;
+	fsck_obj_options.fsck_options.error_func = fsck_error_func;
 	if (check_strict)
-		fsck_obj_options.strict = 1;
+		fsck_obj_options.fsck_options.strict = 1;
 
 	if (show_progress == -1)
 		show_progress = isatty(2);
@@ -956,7 +956,7 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
 	if (name_objects)
 		fsck_enable_object_names(&fsck_walk_options);
 
-	git_config(git_fsck_config, &fsck_obj_options);
+	git_config(git_fsck_config, &fsck_obj_options.fsck_options);
 	prepare_repo_settings(the_repository);
 
 	if (connectivity_only) {
builtin/index-pack.c +2 -2
index 08ebeedfd3..360106b0c8 100644
@@ -1770,13 +1770,13 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
 			} else if (skip_to_optional_arg(arg, "--strict", &arg)) {
 				strict = 1;
 				do_fsck_object = 1;
-				fsck_set_msg_types(&fsck_objects_options, arg);
+				fsck_set_msg_types(&fsck_objects_options.fsck_options, arg);
 			} else if (!strcmp(arg, "--check-self-contained-and-connected")) {
 				strict = 1;
 				check_self_contained_and_connected = 1;
 			} else if (skip_to_optional_arg(arg, "--fsck-objects", &arg)) {
 				do_fsck_object = 1;
-				fsck_set_msg_types(&fsck_objects_options, arg);
+				fsck_set_msg_types(&fsck_objects_options.fsck_options, arg);
 			} else if (!strcmp(arg, "--verify")) {
 				verify = 1;
 			} else if (!strcmp(arg, "--verify-stat")) {
builtin/mktag.c +3 -3
index be2abc71d8..76860f4c7c 100644
@@ -91,12 +91,12 @@ int cmd_mktag(int argc, const char **argv, const char *prefix)
 	if (strbuf_read(&buf, 0, 0) < 0)
 		die_errno(_("could not read from stdin"));
 
-	fsck_objects_options.error_func = mktag_fsck_error_func;
-	fsck_set_msg_type_from_ids(&fsck_objects_options,
+	fsck_objects_options.fsck_options.error_func = mktag_fsck_error_func;
+	fsck_set_msg_type_from_ids(&fsck_objects_options.fsck_options,
 				   FSCK_MSG_EXTRA_HEADER_ENTRY,
 				   FSCK_WARN);
 	/* config might set fsck.extraHeaderEntry=* again */
-	git_config(git_fsck_config, &fsck_objects_options);
+	git_config(git_fsck_config, &fsck_objects_options.fsck_options);
 	if (fsck_tag_standalone(NULL, buf.buf, buf.len, &fsck_objects_options,
 				&tagged_oid, &tagged_type))
 		die(_("tag on stdin did not pass our strict fsck check"));
builtin/unpack-objects.c +1 -1
index c59e330db9..d8d0b14018 100644
@@ -635,7 +635,7 @@ int cmd_unpack_objects(int argc, const char **argv, const char *prefix UNUSED)
 			}
 			if (skip_prefix(arg, "--strict=", &arg)) {
 				strict = 1;
-				fsck_set_msg_types(&fsck_objects_options, arg);
+				fsck_set_msg_types(&fsck_objects_options.fsck_options, arg);
 				continue;
 			}
 			if (starts_with(arg, "--pack_header=")) {
fsck.c +16 -9
index 81b93f02fc..7ac6e4587c 100644
@@ -107,7 +107,7 @@ void list_config_fsck_msg_ids(struct string_list *list, const char *prefix)
 }
 
 static enum fsck_msg_type fsck_msg_type(enum fsck_msg_id msg_id,
-					struct fsck_objects_options *options)
+					struct fsck_options *options)
 {
 	assert(msg_id >= 0 && msg_id < FSCK_MSG_MAX);
 
@@ -142,7 +142,7 @@ int is_valid_msg_type(const char *msg_id, const char *msg_type)
 	return 1;
 }
 
-void fsck_set_msg_type_from_ids(struct fsck_objects_options *options,
+void fsck_set_msg_type_from_ids(struct fsck_options *options,
 				enum fsck_msg_id msg_id,
 				enum fsck_msg_type msg_type)
 {
@@ -158,7 +158,7 @@ void fsck_set_msg_type_from_ids(struct fsck_objects_options *options,
 	fsck_configs.msg_type[msg_id] = msg_type;
 }
 
-void fsck_set_msg_type(struct fsck_objects_options *options,
+void fsck_set_msg_type(struct fsck_options *options,
 		       const char *msg_id_str, const char *msg_type_str)
 {
 	int msg_id = parse_msg_id(msg_id_str);
@@ -187,7 +187,7 @@ void fsck_set_msg_type(struct fsck_objects_options *options,
 	free(to_free);
 }
 
-void fsck_set_msg_types(struct fsck_objects_options *options, const char *values)
+void fsck_set_msg_types(struct fsck_options *options, const char *values)
 {
 	char *buf = xstrdup(values), *to_free = buf;
 	int done = 0;
@@ -237,9 +237,16 @@ static int report(struct fsck_objects_options *options,
 {
 	va_list ap;
 	struct strbuf sb = STRBUF_INIT;
-	enum fsck_msg_type msg_type = fsck_msg_type(msg_id, options);
+	struct fsck_options *fsck_options;
+	enum fsck_msg_type msg_type;
 	int result;
 
+	if (options)
+		fsck_options = &options->fsck_options;
+	else
+		BUG("fsck_options is not set");
+
+	msg_type = fsck_msg_type(msg_id, fsck_options);
 	if (msg_type == FSCK_IGNORE)
 		return 0;
 
@@ -256,8 +263,8 @@ static int report(struct fsck_objects_options *options,
 
 	va_start(ap, fmt);
 	strbuf_vaddf(&sb, fmt, ap);
-	result = options->error_func(options, oid, object_type,
-				     msg_type, msg_id, sb.buf);
+	result = fsck_options->error_func(options, oid, object_type,
+					  msg_type, msg_id, sb.buf);
 	strbuf_release(&sb);
 	va_end(ap);
 
@@ -711,7 +718,7 @@ static int fsck_tree(const struct object_id *tree_oid,
 		 * bits..
 		 */
 		case S_IFREG | 0664:
-			if (!options->strict)
+			if (!options->fsck_options.strict)
 				break;
 			/* fallthrough */
 		default:
@@ -1283,7 +1290,7 @@ int fsck_finish(struct fsck_objects_options *options)
 int git_fsck_config(const char *var, const char *value,
 		    const struct config_context *ctx, void *cb)
 {
-	struct fsck_objects_options *options = cb;
+	struct fsck_options *options = cb;
 	const char *msg_id;
 
 	if (strcmp(var, "fsck.skiplist") == 0) {
fsck.h +22 -10
index 37deadc4bd..e531b44a66 100644
@@ -92,15 +92,16 @@ enum fsck_msg_id {
 };
 #undef MSG_ID
 
+struct fsck_options;
 struct fsck_objects_options;
 struct object;
 
-void fsck_set_msg_type_from_ids(struct fsck_objects_options *options,
+void fsck_set_msg_type_from_ids(struct fsck_options *options,
 				enum fsck_msg_id msg_id,
 				enum fsck_msg_type msg_type);
-void fsck_set_msg_type(struct fsck_objects_options *options,
+void fsck_set_msg_type(struct fsck_options *options,
 		       const char *msg_id, const char *msg_type);
-void fsck_set_msg_types(struct fsck_objects_options *options, const char *values);
+void fsck_set_msg_types(struct fsck_options *options, const char *values);
 int is_valid_msg_type(const char *msg_id, const char *msg_type);
 
 /*
@@ -131,10 +132,15 @@ int fsck_error_cb_print_missing_gitmodules(struct fsck_objects_options *o,
 					   enum fsck_msg_id msg_id,
 					   const char *message);
 
+struct fsck_options {
+	fsck_error error_func;
+	unsigned verbose:1,
+		 strict:1;
+};
+
 struct fsck_objects_options {
+	struct fsck_options fsck_options;
 	fsck_walk_func walk;
-	fsck_error error_func;
-	unsigned strict:1;
 	struct oidset gitmodules_found;
 	struct oidset gitmodules_done;
 	struct oidset gitattributes_found;
@@ -143,27 +149,33 @@ struct fsck_objects_options {
 };
 
 #define FSCK_OBJECTS_OPTIONS_DEFAULT { \
+	.fsck_options = { \
+		.error_func = fsck_error_function, \
+	}, \
 	.gitmodules_found = OIDSET_INIT, \
 	.gitmodules_done = OIDSET_INIT, \
 	.gitattributes_found = OIDSET_INIT, \
 	.gitattributes_done = OIDSET_INIT, \
-	.error_func = fsck_error_function \
 }
 #define FSCK_OBJECTS_OPTIONS_STRICT { \
-	.strict = 1, \
+	.fsck_options = { \
+		.error_func = fsck_error_function, \
+		.strict = 1, \
+	}, \
 	.gitmodules_found = OIDSET_INIT, \
 	.gitmodules_done = OIDSET_INIT, \
 	.gitattributes_found = OIDSET_INIT, \
 	.gitattributes_done = OIDSET_INIT, \
-	.error_func = fsck_error_function, \
 }
 #define FSCK_OBJECTS_OPTIONS_MISSING_GITMODULES { \
-	.strict = 1, \
+	.fsck_options = { \
+		.error_func = fsck_error_cb_print_missing_gitmodules, \
+		.strict = 1, \
+	}, \
 	.gitmodules_found = OIDSET_INIT, \
 	.gitmodules_done = OIDSET_INIT, \
 	.gitattributes_found = OIDSET_INIT, \
 	.gitattributes_done = OIDSET_INIT, \
-	.error_func = fsck_error_cb_print_missing_gitmodules, \
 }
 
 /* descend in all linked child objects
object-file.c +2 -2
index ec44ac3d82..9eda05ee01 100644
@@ -2509,8 +2509,8 @@ static int index_mem(struct index_state *istate,
 	if (flags & HASH_FORMAT_CHECK) {
 		struct fsck_objects_options opts = FSCK_OBJECTS_OPTIONS_DEFAULT;
 
-		opts.strict = 1;
-		opts.error_func = hash_format_check_report;
+		opts.fsck_options.strict = 1;
+		opts.fsck_options.error_func = hash_format_check_report;
 		if (fsck_buffer(null_oid(), type, buf, size, &opts))
 			die(_("refusing to create malformed object"));
 		fsck_finish(&opts);
Signature · 1 line
2.45.2
author
#66 lore raw

PATCH v5 4/12 fsck: add "fsck_refs_options" struct

In reply to shejialuo

Add "fsck_refs_options" to support ref consistency checks.

---
 fsck.h | 4 ++++
 1 file changed, 4 insertions(+)
fsck.h +4 -0
index e531b44a66..a17fee30b4 100644
@@ -138,6 +138,10 @@ struct fsck_options {
 		 strict:1;
 };
 
+struct fsck_refs_options {
+	struct fsck_options fsck_options;
+};
+
 struct fsck_objects_options {
 	struct fsck_options fsck_options;
 	fsck_walk_func walk;
Signature · 1 line
2.45.2
author
#67 lore raw

PATCH v5 5/12 fsck: add a unified interface for reporting fsck messages

In reply to shejialuo

The static function "report" provided by "fsck.c" aims at reporting the problems related to object database which cannot be reused for refs. In order to provide a unified interface which can report either objects or refs, create a new function "fsck_report" by adding two parameters "refs_options" and "checked_ref_name" following the "report" prototype. However, instead of using "...", provide "va_list" to allow more flexibility.

The "vfsck_report" function will use "error_func" registered in "fsck_options" function to report customized messages. Change "error_func" prototype to align with the "vfsck_report".

Change "report" function to make it use "vfsck_report" to report objects-related messages. Add a new function called "fsck_refs_report" to use "vfsck_report" to report refs-related messages. Also, create a general function "fsck_report" for future use.

---
 builtin/fsck.c  |  4 ++-
 builtin/mktag.c |  4 ++-
 fsck.c          | 88 ++++++++++++++++++++++++++++++++++++++++---------
 fsck.h          | 39 +++++++++++++++++++---
 object-file.c   | 14 ++++----
 5 files changed, 122 insertions(+), 27 deletions(-)
builtin/fsck.c +3 -1
index c383125027..2a3b536c1b 100644
@@ -89,9 +89,11 @@ static int objerror(struct object *obj, const char *err)
 	return -1;
 }
 
-static int fsck_error_func(struct fsck_objects_options *o UNUSED,
+static int fsck_error_func(struct fsck_objects_options *objects_options UNUSED,
+			   struct fsck_refs_options *refs_options UNUSED,
 			   const struct object_id *oid,
 			   enum object_type object_type,
+			   const char *checked_ref_name UNUSED,
 			   enum fsck_msg_type msg_type,
 			   enum fsck_msg_id msg_id UNUSED,
 			   const char *message)
builtin/mktag.c +3 -1
index 76860f4c7c..0779a778e9 100644
@@ -17,9 +17,11 @@ static int option_strict = 1;
 
 static struct fsck_objects_options fsck_objects_options = FSCK_OBJECTS_OPTIONS_STRICT;
 
-static int mktag_fsck_error_func(struct fsck_objects_options *o UNUSED,
+static int mktag_fsck_error_func(struct fsck_objects_options *objects_options UNUSED,
+				 struct fsck_refs_options *refs_options UNUSED,
 				 const struct object_id *oid UNUSED,
 				 enum object_type object_type UNUSED,
+				 const char *checked_ref_name UNUSED,
 				 enum fsck_msg_type msg_type,
 				 enum fsck_msg_id msg_id UNUSED,
 				 const char *message)
fsck.c +73 -15

139 lines not on this page · raw message

fsck.h +35 -4
index a17fee30b4..70d5e78ae6 100644
@@ -93,6 +93,7 @@ enum fsck_msg_id {
 #undef MSG_ID
 
 struct fsck_options;
+struct fsck_refs_options;
 struct fsck_objects_options;
 struct object;
 
@@ -115,19 +116,27 @@ int is_valid_msg_type(const char *msg_id, const char *msg_type);
 typedef int (*fsck_walk_func)(struct object *obj, enum object_type object_type,
 			      void *data, struct fsck_objects_options *options);
 
-/* callback for fsck_object, type is FSCK_ERROR or FSCK_WARN */
-typedef int (*fsck_error)(struct fsck_objects_options *o,
+/*
+ * callback function for reporting errors when checking either objects or refs
+ */
+typedef int (*fsck_error)(struct fsck_objects_options *objects_options,
+			  struct fsck_refs_options *refs_options,
 			  const struct object_id *oid, enum object_type object_type,
+			  const char *checked_ref_name,
 			  enum fsck_msg_type msg_type, enum fsck_msg_id msg_id,
 			  const char *message);
 
-int fsck_error_function(struct fsck_objects_options *o,
+int fsck_error_function(struct fsck_objects_options *objects_options,
+			struct fsck_refs_options *refs_options,
 			const struct object_id *oid, enum object_type object_type,
+			const char *checked_ref_name,
 			enum fsck_msg_type msg_type, enum fsck_msg_id msg_id,
 			const char *message);
-int fsck_error_cb_print_missing_gitmodules(struct fsck_objects_options *o,
+int fsck_error_cb_print_missing_gitmodules(struct fsck_objects_options *objects_options,
+					   struct fsck_refs_options *refs_options,
 					   const struct object_id *oid,
 					   enum object_type object_type,
+					   const char *checked_ref_name,
 					   enum fsck_msg_type msg_type,
 					   enum fsck_msg_id msg_id,
 					   const char *message);
@@ -223,6 +232,28 @@ int fsck_tag_standalone(const struct object_id *oid, const char *buffer,
  */
 int fsck_finish(struct fsck_objects_options *options);
 
+/*
+ * Provide a unified interface for either fscking refs or objects.
+ * It will get the current msg error type and call the error_func callback
+ * which is registered in the "fsck_options" struct. For refs, the caller
+ * should pass NULL for "objs_options". For objects, the caller should pass
+ * NULL for "refs_options".
+ */
+__attribute__((format (printf, 7, 8)))
+int fsck_report(struct fsck_objects_options *objects_options,
+		struct fsck_refs_options *refs_options,
+		const struct object_id *oid,
+		enum object_type object_type,
+		const char *checked_ref_name,
+		enum fsck_msg_id msg_id, const char *fmt, ...);
+
+__attribute__((format (printf, 5, 6)))
+int fsck_refs_report(struct fsck_refs_options *refs_options,
+		     const struct object_id *oid,
+		     const char *checked_ref_name,
+		     enum fsck_msg_id msg_id,
+		     const char *fmt, ...);
+
 /*
  * Subsystem for storing human-readable names for each object.
  *
object-file.c +8 -6
index 9eda05ee01..5cb9117fc4 100644
@@ -2472,12 +2472,14 @@ int repo_has_object_file(struct repository *r,
  * report the minimal fsck error here, and rely on the caller to
  * give more context.
  */
-static int hash_format_check_report(struct fsck_objects_options *opts UNUSED,
-				     const struct object_id *oid UNUSED,
-				     enum object_type object_type UNUSED,
-				     enum fsck_msg_type msg_type UNUSED,
-				     enum fsck_msg_id msg_id UNUSED,
-				     const char *message)
+static int hash_format_check_report(struct fsck_objects_options *objects_options UNUSED,
+				    struct fsck_refs_options *refs_options UNUSED,
+				    const struct object_id *oid UNUSED,
+				    enum object_type object_type UNUSED,
+				    const char *chekced_ref_name UNUSED,
+				    enum fsck_msg_type msg_type UNUSED,
+				    enum fsck_msg_id msg_id UNUSED,
+				    const char *message)
 {
 	error(_("object fails fsck: %s"), message);
 	return 1;
Signature · 1 line
2.45.2
author
#68 lore raw

PATCH v5 6/12 fsck: add "fsck_refs_options" initialization macros

In reply to shejialuo

Add "FSCK_REFS_OPTIONS_DEFAULT" and "FSCK_REFS_OPTIONS_STRICT" macros to create the "fsck_refs_options" easily. Add refs-specific "error_func" callback "fsck_refs_error_function".

"fsck_refs_error_function" will use the "oid" parameter. When the caller passed the oid, it will use "oid_to_hex" to get the corresponding hex value to report to the user.

---
 fsck.c | 23 +++++++++++++++++++++++
 fsck.h | 19 +++++++++++++++++++
 2 files changed, 42 insertions(+)
fsck.c +23 -0
index 5184d17736..4869566d19 100644
@@ -1285,6 +1285,29 @@ int fsck_error_function(struct fsck_objects_options *objects_options,
 	return 1;
 }
 
+int fsck_refs_error_function(struct fsck_objects_options *objects_options UNUSED,
+			     struct fsck_refs_options *refs_options,
+			     const struct object_id *oid,
+			     enum object_type object_type UNUSED,
+			     const char *checked_ref_name,
+			     enum fsck_msg_type msg_type,
+			     enum fsck_msg_id msg_id UNUSED,
+			     const char *message)
+{
+	static struct strbuf sb = STRBUF_INIT;
+
+	strbuf_addstr(&sb, checked_ref_name);
+	if (oid)
+		strbuf_addf(&sb, " -> (%s)", oid_to_hex(oid));
+
+	if (msg_type == FSCK_WARN) {
+		warning("%s: %s", sb.buf, message);
+		return 0;
+	}
+	error("%s: %s", sb.buf, message);
+	return 1;
+}
+
 static int fsck_blobs(struct oidset *blobs_found, struct oidset *blobs_done,
 		      enum fsck_msg_id msg_missing, enum fsck_msg_id msg_type,
 		      struct fsck_objects_options *options,
fsck.h +19 -0
index 70d5e78ae6..e903845690 100644
@@ -140,6 +140,14 @@ int fsck_error_cb_print_missing_gitmodules(struct fsck_objects_options *objects_
 					   enum fsck_msg_type msg_type,
 					   enum fsck_msg_id msg_id,
 					   const char *message);
+int fsck_refs_error_function(struct fsck_objects_options *objects_options,
+			     struct fsck_refs_options *refs_options,
+			     const struct object_id *oid,
+			     enum object_type object_type,
+			     const char *checked_ref_name,
+			     enum fsck_msg_type msg_type,
+			     enum fsck_msg_id msg_id,
+			     const char *message);
 
 struct fsck_options {
 	fsck_error error_func;
@@ -150,6 +158,17 @@ struct fsck_options {
 struct fsck_refs_options {
 	struct fsck_options fsck_options;
 };
+#define FSCK_REFS_OPTIONS_DEFAULT { \
+	.fsck_options = { \
+		.error_func = fsck_refs_error_function, \
+	}, \
+}
+#define FSCK_REFS_OPTIONS_STRICT { \
+	.fsck_options = { \
+		.error_func = fsck_refs_error_function, \
+		.strict = 1, \
+	}, \
+}
 
 struct fsck_objects_options {
 	struct fsck_options fsck_options;
Signature · 1 line
2.45.2
author
#69 lore raw

PATCH v5 7/12 refs: set up ref consistency check infrastructure

In reply to shejialuo

The interfaces defined in the ref_storage_be are carefully structured in semantic. It's organized as the five parts:

1. The name and the initialization interfaces.
2. The ref transaction interfaces.
3. The ref internal interfaces (pack, rename and copy).
4. The ref filesystem interfaces.
5. The reflog related interfaces.

To keep consistent with the git-fsck(1), add a new interface named "fsck_refs_fn" to the end of "ref_storage_be". This semantic cannot be grouped into any above five categories. Explicitly add blank line to make it different from others.

---
 refs.c                  |  5 +++++
 refs.h                  |  8 ++++++++
 refs/debug.c            | 11 +++++++++++
 refs/files-backend.c    | 15 ++++++++++++++-
 refs/packed-backend.c   |  8 ++++++++
 refs/refs-internal.h    |  6 ++++++
 refs/reftable-backend.c |  8 ++++++++
 7 files changed, 60 insertions(+), 1 deletion(-)
refs.c +5 -0
index dd0d9c360f..646fdfaf22 100644
@@ -316,6 +316,11 @@ int check_refname_format(const char *refname, int flags)
 	return check_or_sanitize_refname(refname, flags, NULL);
 }
 
+int refs_fsck(struct ref_store *refs, struct fsck_refs_options *o)
+{
+	return refs->be->fsck(refs, o);
+}
+
 void sanitize_refname_component(const char *refname, struct strbuf *out)
 {
 	if (check_or_sanitize_refname(refname, REFNAME_ALLOW_ONELEVEL, out))
refs.h +8 -0
index df8cbf6124..a8f4deb108 100644
@@ -3,6 +3,7 @@
 
 #include "commit.h"
 
+struct fsck_refs_options;
 struct object_id;
 struct ref_store;
 struct repository;
@@ -547,6 +548,13 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat
  */
 int check_refname_format(const char *refname, int flags);
 
+/*
+ * Check the reference database for consistency. Return 0 if refs and
+ * reflogs are consistent, and non-zero otherwise. The errors will be
+ * written to stderr.
+ */
+int refs_fsck(struct ref_store *refs, struct fsck_refs_options *o);
+
 /*
  * Apply the rules from check_refname_format, but mutate the result until it
  * is acceptable, and place the result in "out".
refs/debug.c +11 -0
index 547d9245b9..6306c25fd3 100644
@@ -419,6 +419,15 @@ static int debug_reflog_expire(struct ref_store *ref_store, const char *refname,
 	return res;
 }
 
+static int debug_fsck(struct ref_store *ref_store,
+		      struct fsck_refs_options *o)
+{
+	struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store;
+	int res = drefs->refs->be->fsck(drefs->refs, o);
+	trace_printf_key(&trace_refs, "fsck: %d\n", res);
+	return res;
+}
+
 struct ref_storage_be refs_be_debug = {
 	.name = "debug",
 	.init = NULL,
@@ -451,4 +460,6 @@ struct ref_storage_be refs_be_debug = {
 	.create_reflog = debug_create_reflog,
 	.delete_reflog = debug_delete_reflog,
 	.reflog_expire = debug_reflog_expire,
+
+	.fsck = debug_fsck,
 };
refs/files-backend.c +14 -1
index 44c5c3b201..7257fffb77 100644
@@ -3406,6 +3406,17 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,
 	return ret;
 }
 
+static int files_fsck(struct ref_store *ref_store,
+		      struct fsck_refs_options *o)
+{
+	int ret;
+	struct files_ref_store *refs =
+		files_downcast(ref_store, REF_STORE_READ, "fsck");
+
+	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o);
+	return ret;
+}
+
 struct ref_storage_be refs_be_files = {
 	.name = "files",
 	.init = files_ref_store_init,
@@ -3432,5 +3443,7 @@ struct ref_storage_be refs_be_files = {
 	.reflog_exists = files_reflog_exists,
 	.create_reflog = files_create_reflog,
 	.delete_reflog = files_delete_reflog,
-	.reflog_expire = files_reflog_expire
+	.reflog_expire = files_reflog_expire,
+
+	.fsck = files_fsck,
 };
refs/packed-backend.c +8 -0
index c4c1e36aa2..db152053f8 100644
@@ -1733,6 +1733,12 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_s
 	return empty_ref_iterator_begin();
 }
 
+static int packed_fsck(struct ref_store *ref_store,
+		       struct fsck_refs_options *o)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_packed = {
 	.name = "packed",
 	.init = packed_ref_store_init,
@@ -1760,4 +1766,6 @@ struct ref_storage_be refs_be_packed = {
 	.create_reflog = NULL,
 	.delete_reflog = NULL,
 	.reflog_expire = NULL,
+
+	.fsck = packed_fsck,
 };
refs/refs-internal.h +6 -0
index fa975d69aa..ecc082baf8 100644
@@ -4,6 +4,7 @@
 #include "refs.h"
 #include "iterator.h"
 
+struct fsck_options;
 struct ref_transaction;
 
 /*
@@ -650,6 +651,9 @@ typedef int read_raw_ref_fn(struct ref_store *ref_store, const char *refname,
 typedef int read_symbolic_ref_fn(struct ref_store *ref_store, const char *refname,
 				 struct strbuf *referent);
 
+typedef int fsck_fn(struct ref_store *ref_store,
+		    struct fsck_refs_options *o);
+
 struct ref_storage_be {
 	const char *name;
 	ref_store_init_fn *init;
@@ -677,6 +681,8 @@ struct ref_storage_be {
 	create_reflog_fn *create_reflog;
 	delete_reflog_fn *delete_reflog;
 	reflog_expire_fn *reflog_expire;
+
+	fsck_fn *fsck;
 };
 
 extern struct ref_storage_be refs_be_files;
refs/reftable-backend.c +8 -0
index 7d872a32ac..2a358b40ca 100644
@@ -2292,6 +2292,12 @@ static int reftable_be_reflog_expire(struct ref_store *ref_store,
 	return ret;
 }
 
+static int reftable_be_fsck(struct ref_store *ref_store,
+			    struct fsck_refs_options *o)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_reftable = {
 	.name = "reftable",
 	.init = reftable_be_init,
@@ -2319,4 +2325,6 @@ struct ref_storage_be refs_be_reftable = {
 	.create_reflog = reftable_be_create_reflog,
 	.delete_reflog = reftable_be_delete_reflog,
 	.reflog_expire = reftable_be_reflog_expire,
+
+	.fsck = reftable_be_fsck,
 };
Signature · 1 line
2.45.2
author
#70 lore raw

PATCH v5 8/12 builtin/refs: add verify subcommand

In reply to shejialuo

Introduce a new subcommand "verify" in git-refs(1) to allow the user to check the reference database consistency.

---
 Documentation/git-refs.txt | 13 +++++++++++++
 builtin/refs.c             | 36 ++++++++++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+)
Documentation/git-refs.txt +13 -0
index 5b99e04385..1244a85b64 100644
@@ -10,6 +10,7 @@ SYNOPSIS
 --------
 [verse]
 'git refs migrate' --ref-format=<format> [--dry-run]
+'git refs verify' [--strict] [--verbose]
 
 DESCRIPTION
 -----------
@@ -22,6 +23,9 @@ COMMANDS
 migrate::
 	Migrate ref store between different formats.
 
+verify::
+	Verify reference database consistency.
+
 OPTIONS
 -------
 
@@ -39,6 +43,15 @@ include::ref-storage-format.txt[]
 	can be used to double check that the migration works as expected before
 	performing the actual migration.
 
+The following options are specific to 'git refs verify':
+
+--strict::
+	Enable more strict checking, every WARN severity for the `Fsck Messages`
+	be seen as ERROR. See linkgit:git-fsck[1].
+
+--verbose::
+	When verifying the reference database consistency, be chatty.
+
 KNOWN LIMITATIONS
 -----------------
 
builtin/refs.c +36 -0
index 46dcd150d4..511b0d0f8b 100644
@@ -1,4 +1,6 @@
 #include "builtin.h"
+#include "config.h"
+#include "fsck.h"
 #include "parse-options.h"
 #include "refs.h"
 #include "repository.h"
@@ -7,6 +9,9 @@
 #define REFS_MIGRATE_USAGE \
 	N_("git refs migrate --ref-format=<format> [--dry-run]")
 
+#define REFS_VERIFY_USAGE \
+	N_("git refs verify [--strict] [--verbose]")
+
 static int cmd_refs_migrate(int argc, const char **argv, const char *prefix)
 {
 	const char * const migrate_usage[] = {
@@ -58,15 +63,46 @@ static int cmd_refs_migrate(int argc, const char **argv, const char *prefix)
 	return err;
 }
 
+static int cmd_refs_verify(int argc, const char **argv, const char *prefix)
+{
+	struct fsck_refs_options fsck_refs_options = FSCK_REFS_OPTIONS_DEFAULT;
+	const char * const verify_usage[] = {
+		REFS_VERIFY_USAGE,
+		NULL,
+	};
+	unsigned int verbose = 0, strict = 0;
+	struct option options[] = {
+		OPT__VERBOSE(&verbose, N_("be verbose")),
+		OPT_BOOL(0, "strict", &strict, N_("enable strict checking")),
+		OPT_END(),
+	};
+
+	argc = parse_options(argc, argv, prefix, options, verify_usage, 0);
+	if (argc)
+		usage(_("too many arguments"));
+
+	if (verbose)
+		fsck_refs_options.fsck_options.verbose = 1;
+	if (strict)
+		fsck_refs_options.fsck_options.strict = 1;
+
+	git_config(git_fsck_config, &fsck_refs_options.fsck_options);
+	prepare_repo_settings(the_repository);
+
+	return refs_fsck(get_main_ref_store(the_repository), &fsck_refs_options);
+}
+
 int cmd_refs(int argc, const char **argv, const char *prefix)
 {
 	const char * const refs_usage[] = {
 		REFS_MIGRATE_USAGE,
+		REFS_VERIFY_USAGE,
 		NULL,
 	};
 	parse_opt_subcommand_fn *fn = NULL;
 	struct option opts[] = {
 		OPT_SUBCOMMAND("migrate", &fn, cmd_refs_migrate),
+		OPT_SUBCOMMAND("verify", &fn, cmd_refs_verify),
 		OPT_END(),
 	};
 
Signature · 1 line
2.45.2
author
#71 lore raw

PATCH v5 9/12 builtin/fsck: add `git-refs verify` child process

In reply to shejialuo

Introduce a new function "fsck_refs" that initializes and runs a child process to execute the "git-refs verify" command.

---
 builtin/fsck.c | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
builtin/fsck.c +17 -0
index 2a3b536c1b..a5654da62b 100644
@@ -899,6 +899,21 @@ static int check_pack_rev_indexes(struct repository *r, int show_progress)
 	return res;
 }
 
+static void fsck_refs(void)
+{
+	struct child_process refs_verify = CHILD_PROCESS_INIT;
+	child_process_init(&refs_verify);
+	refs_verify.git_cmd = 1;
+	strvec_pushl(&refs_verify.args, "refs", "verify", NULL);
+	if (verbose)
+		strvec_push(&refs_verify.args, "--verbose");
+	if (check_strict)
+		strvec_push(&refs_verify.args, "--strict");
+
+	if (run_command(&refs_verify))
+		errors_found |= ERROR_REFS;
+}
+
 static char const * const fsck_usage[] = {
 	N_("git fsck [--tags] [--root] [--unreachable] [--cache] [--no-reflogs]\n"
 	   "         [--[no-]full] [--strict] [--verbose] [--lost-found]\n"
@@ -1068,6 +1083,8 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
 
 	check_connectivity();
 
+	fsck_refs();
+
 	if (the_repository->settings.core_commit_graph) {
 		struct child_process commit_graph_verify = CHILD_PROCESS_INIT;
 
Signature · 1 line
2.45.2
author
#72 lore raw

PATCH v5 10/12 files-backend: add unified interface for refs scanning

In reply to shejialuo

For refs and reflogs, we need to scan its corresponding directories to check every regular file or symbolic link which shares the same pattern. Introduce a unified interface for scanning directories for files-backend.

---
 refs/files-backend.c | 77 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 76 insertions(+), 1 deletion(-)
refs/files-backend.c +76 -1
index 7257fffb77..aefc947fc3 100644
@@ -4,6 +4,7 @@
 #include "../gettext.h"
 #include "../hash.h"
 #include "../hex.h"
+#include "../fsck.h"
 #include "../refs.h"
 #include "refs-internal.h"
 #include "ref-cache.h"
@@ -3406,6 +3407,78 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,
 	return ret;
 }
 
+/*
+ * For refs and reflogs, they share a unified interface when scanning
+ * the whole directory. This function is used as the callback for each
+ * regular file or symlink in the directory.
+ */
+typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o,
+				  const char *gitdir,
+				  const char *refs_check_dir,
+				  struct dir_iterator *iter);
+
+static int files_fsck_refs_dir(struct ref_store *ref_store,
+			       struct fsck_refs_options *o,
+			       const char *refs_check_dir,
+			       files_fsck_refs_fn *fsck_refs_fns)
+{
+	const char *gitdir = ref_store->gitdir;
+	struct strbuf sb = STRBUF_INIT;
+	struct dir_iterator *iter;
+	int iter_status;
+	int ret = 0;
+
+	strbuf_addf(&sb, "%s/%s", gitdir, refs_check_dir);
+
+	iter = dir_iterator_begin(sb.buf, 0);
+
+	if (!iter) {
+		ret = error_errno("cannot open directory %s", sb.buf);
+		goto out;
+	}
+
+	while ((iter_status = dir_iterator_advance(iter)) == ITER_OK) {
+		if (S_ISDIR(iter->st.st_mode)) {
+			continue;
+		} else if (S_ISREG(iter->st.st_mode) ||
+			   S_ISLNK(iter->st.st_mode)) {
+			if (o->fsck_options.verbose)
+				fprintf_ln(stderr, "Checking %s/%s",
+					   refs_check_dir, iter->relative_path);
+			for (size_t i = 0; fsck_refs_fns[i]; i++) {
+				if (fsck_refs_fns[i](o, gitdir, refs_check_dir, iter))
+					ret = -1;
+			}
+		} else {
+			ret = error(_("unexpected file type for '%s'"),
+				    iter->basename);
+		}
+	}
+
+	if (iter_status != ITER_DONE)
+		ret = error(_("failed to iterate over '%s'"), sb.buf);
+
+out:
+	strbuf_release(&sb);
+	return ret;
+}
+
+static int files_fsck_refs(struct ref_store *ref_store,
+			   struct fsck_refs_options *o)
+{
+	int ret;
+	files_fsck_refs_fn fsck_refs_fns[]= {
+		NULL
+	};
+
+	if (o->fsck_options.verbose)
+		fprintf_ln(stderr, "Checking references consistency");
+
+	ret = files_fsck_refs_dir(ref_store, o, "refs", fsck_refs_fns);
+
+	return ret;
+}
+
 static int files_fsck(struct ref_store *ref_store,
 		      struct fsck_refs_options *o)
 {
@@ -3413,7 +3486,9 @@ static int files_fsck(struct ref_store *ref_store,
 	struct files_ref_store *refs =
 		files_downcast(ref_store, REF_STORE_READ, "fsck");
 
-	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o);
+	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o)
+	    | files_fsck_refs(ref_store, o);
+
 	return ret;
 }
 
Signature · 1 line
2.45.2
author
#73 lore raw

PATCH v5 11/12 fsck: add ref name check for files backend

In reply to shejialuo

The git-fsck(1) only implicitly checks the reference, it does not fully check refs with bad format name such as standalone "@" and name ending with ".lock".

In order to provide such checks, add a new fsck message id "badRefName" with default ERROR type. Use existing "check_refname_format" to explicit check the ref name. And add a new unit test to verify the functionality.

---
 Documentation/fsck-msgids.txt |   3 +
 fsck.h                        |   1 +
 refs/files-backend.c          |  20 +++++++
 t/t0602-reffiles-fsck.sh      | 101 ++++++++++++++++++++++++++++++++++
 4 files changed, 125 insertions(+)
 create mode 100755 t/t0602-reffiles-fsck.sh
Documentation/fsck-msgids.txt +3 -0
index f643585a34..dab4012246 100644
@@ -19,6 +19,9 @@
 `badParentSha1`::
 	(ERROR) A commit object has a bad parent sha1.
 
+`badRefName`::
+	(ERROR) A ref has a bad name.
+
 `badTagName`::
 	(INFO) A tag has an invalid format.
 
fsck.h +1 -0
index e903845690..aad2211bff 100644
@@ -31,6 +31,7 @@ enum fsck_msg_type {
 	FUNC(BAD_NAME, ERROR) \
 	FUNC(BAD_OBJECT_SHA1, ERROR) \
 	FUNC(BAD_PARENT_SHA1, ERROR) \
+	FUNC(BAD_REF_NAME, ERROR) \
 	FUNC(BAD_TIMEZONE, ERROR) \
 	FUNC(BAD_TREE, ERROR) \
 	FUNC(BAD_TREE_SHA1, ERROR) \
refs/files-backend.c +20 -0
index aefc947fc3..e840a768b4 100644
@@ -3417,6 +3417,25 @@ typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o,
 				  const char *refs_check_dir,
 				  struct dir_iterator *iter);
 
+static int files_fsck_refs_name(struct fsck_refs_options *o,
+				const char *gitdir UNUSED,
+				const char *refs_check_dir,
+				struct dir_iterator *iter)
+{
+	struct strbuf sb = STRBUF_INIT;
+	int ret = 0;
+
+	if (check_refname_format(iter->basename, REFNAME_ALLOW_ONELEVEL)) {
+		strbuf_addf(&sb, "%s/%s", refs_check_dir, iter->relative_path);
+		ret = fsck_refs_report(o, NULL, sb.buf,
+				       FSCK_MSG_BAD_REF_NAME,
+				       "invalid refname format");
+	}
+
+	strbuf_release(&sb);
+	return ret;
+}
+
 static int files_fsck_refs_dir(struct ref_store *ref_store,
 			       struct fsck_refs_options *o,
 			       const char *refs_check_dir,
@@ -3468,6 +3487,7 @@ static int files_fsck_refs(struct ref_store *ref_store,
 {
 	int ret;
 	files_fsck_refs_fn fsck_refs_fns[]= {
+		files_fsck_refs_name,
 		NULL
 	};
 
t/t0602-reffiles-fsck.sh new file +101 -0

101 lines not on this page · raw message

Signature · 1 line
2.45.2
author
#74 lore raw

PATCH v5 12/12 fsck: add ref content check for files backend

In reply to shejialuo

Enhance the git-fsck(1) command by adding a check for reference content in the files backend. The new functionality ensures that symrefs, real symbolic link and regular refs are validated correctly.

In order to check the trailing content of the regular refs, add a new parameter trailing to parse_loose_ref_contents.

For symrefs, parse_loose_ref_contents will set the "referent". However, symbolic link could be either absolute or relative. Use "strbuf_add_real_path" to read the symbolic link and convert the relative path to absolute path. Then use "skip_prefix" to make it align with symref "referent".

Thus, the symrefs and symbolic links could share the same interface. Add a new function "files_fsck_symref_target" which aims at checking the following things:

1. whether the pointee is under the refs/ directory.
2. whether the pointee name is correct.
3. whether the pointee path is a wrong type in filesystem.

Last, add the following FSCK MESSAGEs:

1. "badRefContent(ERROR)": A ref has a bad content
2. "badSymrefPointee(ERROR)": The pointee of a symref is bad.
3. "trailingRefContent(WARN)": A ref content has trailing contents.

---
 Documentation/fsck-msgids.txt |   9 +++
 fsck.h                        |   3 +
 refs.c                        |   2 +-
 refs/files-backend.c          | 145 +++++++++++++++++++++++++++++++++-
 refs/refs-internal.h          |   5 +-
 t/t0602-reffiles-fsck.sh      | 110 ++++++++++++++++++++++++++
 6 files changed, 269 insertions(+), 5 deletions(-)
Documentation/fsck-msgids.txt +9 -0
index dab4012246..b1630a478b 100644
@@ -19,9 +19,15 @@
 `badParentSha1`::
 	(ERROR) A commit object has a bad parent sha1.
 
+`badRefContent`::
+	(ERROR) A ref has a bad content.
+
 `badRefName`::
 	(ERROR) A ref has a bad name.
 
+`badSymrefPointee`::
+	(ERROR) The pointee of a symref is bad.
+
 `badTagName`::
 	(INFO) A tag has an invalid format.
 
@@ -167,6 +173,9 @@
 `nullSha1`::
 	(WARN) Tree contains entries pointing to a null sha1.
 
+`trailingRefContent`::
+	(WARN) A ref content has trailing contents.
+
 `treeNotSorted`::
 	(ERROR) A tree is not properly sorted.
 
fsck.h +3 -0
index aad2211bff..5c1b113674 100644
@@ -32,6 +32,8 @@ enum fsck_msg_type {
 	FUNC(BAD_OBJECT_SHA1, ERROR) \
 	FUNC(BAD_PARENT_SHA1, ERROR) \
 	FUNC(BAD_REF_NAME, ERROR) \
+	FUNC(BAD_REF_CONTENT, ERROR) \
+	FUNC(BAD_SYMREF_POINTEE, ERROR) \
 	FUNC(BAD_TIMEZONE, ERROR) \
 	FUNC(BAD_TREE, ERROR) \
 	FUNC(BAD_TREE_SHA1, ERROR) \
@@ -72,6 +74,7 @@ enum fsck_msg_type {
 	FUNC(HAS_DOTDOT, WARN) \
 	FUNC(HAS_DOTGIT, WARN) \
 	FUNC(NULL_SHA1, WARN) \
+	FUNC(TRAILING_REF_CONTENT, WARN) \
 	FUNC(ZERO_PADDED_FILEMODE, WARN) \
 	FUNC(NUL_IN_COMMIT, WARN) \
 	FUNC(LARGE_PATHNAME, WARN) \
refs.c +1 -1
index 646fdfaf22..55d581ab12 100644
@@ -1758,7 +1758,7 @@ static int refs_read_special_head(struct ref_store *ref_store,
 	}
 
 	result = parse_loose_ref_contents(content.buf, oid, referent, type,
-					  failure_errno);
+					  failure_errno, NULL);
 
 done:
 	strbuf_release(&full_path);
refs/files-backend.c +143 -2

179 lines not on this page · raw message

refs/refs-internal.h +3 -2
index ecc082baf8..915d0d1f20 100644
@@ -709,11 +709,12 @@ struct ref_store {
 
 /*
  * Parse contents of a loose ref file. *failure_errno maybe be set to EINVAL for
- * invalid contents.
+ * invalid contents. Also *trailing is set to the first character after the
+ * refname or NULL if the referent is not empty.
  */
 int parse_loose_ref_contents(const char *buf, struct object_id *oid,
 			     struct strbuf *referent, unsigned int *type,
-			     int *failure_errno);
+			     int *failure_errno, const char **trailing);
 
 /*
  * Fill in the generic part of refs and add it to our collection of
t/t0602-reffiles-fsck.sh +110 -0

114 lines not on this page · raw message

Signature · 1 line
2.45.2
author
#75 lore raw

v6 cover ref consistency check infra setup

In reply to shejialuo

Hi All:

This version follows the Junio's advice. Instead of creating the following data structure:

	struct fsck_options {
		enum fsck_type {
			FSCK_OBJECTS,
			FSCK_REFS,
			...
		} t;
		union {
			struct fsck_objects_options objects;
			struct fsck_refs_options refs;
		} u;
	};

I simply use the combination idea where "fsck_options" will incorporate "fsck_objects_options" and "fsck_refs_options". Karthik has told me that I should balance the job I should does and the extensibility for future.

Quoted 3 lines · So I use the most clear way to do this. Also Junio has

I would have expected that adding ref-related new members that object consistency checkers has no interest in to the fsck_options structure would be sufficient for the purpose of this series.

So this patch makes the design simple here.

Thanks for every reviewer.

Jialuo.

shejialuo (11):
fsck: add "fsck_objects_options" to hold objects-related options
fsck: rename "skiplist" to "oid_skiplist"
fsck: add "fsck_refs_options" into "fsck_options"
fsck: add a unified interface for reporting fsck messages
fsck: add "fsck_refs_options" initialization macros
refs: set up ref consistency check infrastructure builtin/refs: add verify subcommand
builtin/fsck: add git-refs verify child process
files-backend: add unified interface for refs scanning
fsck: add ref name check for files backend
fsck: add ref content check for files backend

 Documentation/fsck-msgids.txt |  12 ++
 Documentation/git-refs.txt    |  13 ++
 builtin/fsck.c                |  44 ++++--
 builtin/index-pack.c          |   4 +-
 builtin/mktag.c               |   3 +-
 builtin/refs.c                |  44 ++++++
 builtin/unpack-objects.c      |   4 +-
 fetch-pack.c                  |  10 +-
 fsck.c                        | 158 +++++++++++++++------
 fsck.h                        | 115 ++++++++++-----
 object-file.c                 |  13 +-
 refs.c                        |   7 +-
 refs.h                        |   8 ++
 refs/debug.c                  |  11 ++
 refs/files-backend.c          | 255 +++++++++++++++++++++++++++++++++-
 refs/packed-backend.c         |   8 ++
 refs/refs-internal.h          |  11 +-
 refs/reftable-backend.c       |   8 ++
 t/t0602-reffiles-fsck.sh      | 211 ++++++++++++++++++++++++++++
 19 files changed, 824 insertions(+), 115 deletions(-)
 create mode 100755 t/t0602-reffiles-fsck.sh

Range-diff against v5:

 1:  d83b5797cb <  -:  ---------- fsck: rename "fsck_options" to "fsck_objects_options"
 2:  88000b092a <  -:  ---------- fsck: use "fsck_configs" to set up configs
 3:  65093bae64 <  -:  ---------- fsck: abstract common options for reusing
 4:  403750da30 <  -:  ---------- fsck: add "fsck_refs_options" struct
 5:  8dea5654d5 <  -:  ---------- fsck: add a unified interface for reporting fsck messages
 -:  ---------- >  1:  d32ae41a4d fsck: add "fsck_objects_options" to hold objects-related options
 -:  ---------- >  2:  589a7a6fac fsck: rename "skiplist" to "oid_skiplist"
 -:  ---------- >  3:  bab97d7c82 fsck: add "fsck_refs_options" into "fsck_options"
 -:  ---------- >  4:  276da52a6b fsck: add a unified interface for reporting fsck messages
 6:  8fb62b54c6 !  5:  e93940c50c fsck: add "fsck_refs_options" initialization macros

@@ Commit message
fsck: add "fsck_refs_options" initialization macros

Add "FSCK_REFS_OPTIONS_DEFAULT" and "FSCK_REFS_OPTIONS_STRICT" macros to
- create the "fsck_refs_options" easily. Add refs-specific "error_func"
- callback "fsck_refs_error_function".
+ create the refs options easily. Add refs-specific "error_func" callback
+ "fsck_refs_error_function".

"fsck_refs_error_function" will use the "oid" parameter. When the caller passed the oid, it will use "oid_to_hex" to get the corresponding hex
@@ Commit message

## fsck.c ##
-@@ fsck.c: int fsck_error_function(struct fsck_objects_options *objects_options, +@@ fsck.c: int fsck_objects_error_function(struct fsck_options *o,
return 1;
}

-+int fsck_refs_error_function(struct fsck_objects_options *objects_options UNUSED,
-+ struct fsck_refs_options *refs_options,
++int fsck_refs_error_function(struct fsck_options *options UNUSED,
+ const struct object_id *oid,
+ enum object_type object_type UNUSED,
+ const char *checked_ref_name,
@@ fsck.c: int fsck_error_function(struct fsck_objects_options *objects_options,
+
static int fsck_blobs(struct oidset *blobs_found, struct oidset *blobs_done,
enum fsck_msg_id msg_missing, enum fsck_msg_id msg_type,
- struct fsck_objects_options *options,
+ struct fsck_options *options, const char *blob_type)

## fsck.h ##
-@@ fsck.h: int fsck_error_cb_print_missing_gitmodules(struct fsck_objects_options *objects_
- enum fsck_msg_type msg_type,
- enum fsck_msg_id msg_id,
- const char *message);
-+int fsck_refs_error_function(struct fsck_objects_options *objects_options,
-+ struct fsck_refs_options *refs_options,
+@@ fsck.h: int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o,
+ enum fsck_msg_type msg_type,
+ enum fsck_msg_id msg_id,
+ const char *message);
++int fsck_refs_error_function(struct fsck_options *options,
+ const struct object_id *oid,
+ enum object_type object_type,
+ const char *checked_ref_name,
@@ fsck.h: int fsck_error_cb_print_missing_gitmodules(struct fsck_objects_options *
+ enum fsck_msg_id msg_id,
+ const char *message);

- struct fsck_options {
- fsck_error error_func;
-@@ fsck.h: struct fsck_options {
struct fsck_refs_options {
- struct fsck_options fsck_options;
- };
+ unsigned verbose:1;
+@@ fsck.h: struct fsck_options {
+ .gitattributes_done = OIDSET_INIT, \
+ } \
+ }
+#define FSCK_REFS_OPTIONS_DEFAULT { \
-+ .fsck_options = { \
-+ .error_func = fsck_refs_error_function, \
-+ }, \
++ .error_func = fsck_refs_error_function, \
+}
+#define FSCK_REFS_OPTIONS_STRICT { \
-+ .fsck_options = { \
-+ .error_func = fsck_refs_error_function, \
-+ .strict = 1, \
-+ }, \
++ .strict = 1, \
++ .error_func = fsck_refs_error_function, \
+}

- struct fsck_objects_options {
- struct fsck_options fsck_options;
+ /* descend in all linked child objects
+ * the return value is:
7: 2ecd1f5407 ! 6: 85aa953f6d refs: set up ref consistency check infrastructure

    @@ refs.c: int check_refname_format(const char *refname, int flags)
      	return check_or_sanitize_refname(refname, flags, NULL);
      }
      
    -+int refs_fsck(struct ref_store *refs, struct fsck_refs_options *o)
    ++int refs_fsck(struct ref_store *refs, struct fsck_options *o)
     +{
     +	return refs->be->fsck(refs, o);
     +}
    @@ refs.h
      
      #include "commit.h"
      
    -+struct fsck_refs_options;
    ++struct fsck_options;
      struct object_id;
      struct ref_store;
      struct repository;
    @@ refs.h: int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void
     + * reflogs are consistent, and non-zero otherwise. The errors will be
     + * written to stderr.
     + */
    -+int refs_fsck(struct ref_store *refs, struct fsck_refs_options *o);
    ++int refs_fsck(struct ref_store *refs, struct fsck_options *o);
     +
      /*
       * Apply the rules from check_refname_format, but mutate the result until it
    @@ refs/debug.c: static int debug_reflog_expire(struct ref_store *ref_store, const
      }
      
     +static int debug_fsck(struct ref_store *ref_store,
    -+		      struct fsck_refs_options *o)
    ++		      struct fsck_options *o)
     +{
     +	struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store;
     +	int res = drefs->refs->be->fsck(drefs->refs, o);
    @@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
      }
      
     +static int files_fsck(struct ref_store *ref_store,
    -+		      struct fsck_refs_options *o)
    ++		      struct fsck_options *o)
     +{
     +	int ret;
     +	struct files_ref_store *refs =
    @@ refs/packed-backend.c: static struct ref_iterator *packed_reflog_iterator_begin(
      }
      
     +static int packed_fsck(struct ref_store *ref_store,
    -+		       struct fsck_refs_options *o)
    ++		       struct fsck_options *o)
     +{
     +	return 0;
     +}
    @@ refs/refs-internal.h: typedef int read_raw_ref_fn(struct ref_store *ref_store, c
      				 struct strbuf *referent);
      
     +typedef int fsck_fn(struct ref_store *ref_store,
    -+		    struct fsck_refs_options *o);
    ++		    struct fsck_options *o);
     +
      struct ref_storage_be {
      	const char *name;
    @@ refs/reftable-backend.c: static int reftable_be_reflog_expire(struct ref_store *
      }
      
     +static int reftable_be_fsck(struct ref_store *ref_store,
    -+			    struct fsck_refs_options *o)
    ++			    struct fsck_options *o)
     +{
     +	return 0;
     +}
 8:  534a3d00af !  7:  ce7adc7372 builtin/refs: add verify subcommand
    @@ builtin/refs.c: static int cmd_refs_migrate(int argc, const char **argv, const c
      
     +static int cmd_refs_verify(int argc, const char **argv, const char *prefix)
     +{
    -+	struct fsck_refs_options fsck_refs_options = FSCK_REFS_OPTIONS_DEFAULT;
    ++	struct fsck_options fsck_refs_options = FSCK_REFS_OPTIONS_DEFAULT;
     +	const char * const verify_usage[] = {
     +		REFS_VERIFY_USAGE,
     +		NULL,
    @@ builtin/refs.c: static int cmd_refs_migrate(int argc, const char **argv, const c
     +		OPT_BOOL(0, "strict", &strict, N_("enable strict checking")),
     +		OPT_END(),
     +	};
    ++	int ret = 0;
     +
     +	argc = parse_options(argc, argv, prefix, options, verify_usage, 0);
     +	if (argc)
     +		usage(_("too many arguments"));
     +
     +	if (verbose)
    -+		fsck_refs_options.fsck_options.verbose = 1;
    ++		fsck_refs_options.refs_options.verbose = 1;
     +	if (strict)
    -+		fsck_refs_options.fsck_options.strict = 1;
    ++		fsck_refs_options.strict = 1;
     +
    -+	git_config(git_fsck_config, &fsck_refs_options.fsck_options);
    ++	git_config(git_fsck_config, &fsck_refs_options);
     +	prepare_repo_settings(the_repository);
     +
    -+	return refs_fsck(get_main_ref_store(the_repository), &fsck_refs_options);
    ++	ret = refs_fsck(get_main_ref_store(the_repository), &fsck_refs_options);
    ++
    ++	/*
    ++	 * Explicitly free the allocated array and "oid_skiplist"
    ++	 */
    ++	free(fsck_refs_options.msg_type);
    ++	oidset_clear(&fsck_refs_options.oid_skiplist);
    ++	return ret;
     +}
     +
      int cmd_refs(int argc, const char **argv, const char *prefix)
 9:  054c8cea89 =  8:  035eafe10b builtin/fsck: add `git-refs verify` child process
10:  7f87c6a26e !  9:  9398bf3f0d files-backend: add unified interface for refs scanning
    @@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
     + * the whole directory. This function is used as the callback for each
     + * regular file or symlink in the directory.
     + */
    -+typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o,
    ++typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
     +				  const char *gitdir,
     +				  const char *refs_check_dir,
     +				  struct dir_iterator *iter);
     +
     +static int files_fsck_refs_dir(struct ref_store *ref_store,
    -+			       struct fsck_refs_options *o,
    ++			       struct fsck_options *o,
     +			       const char *refs_check_dir,
     +			       files_fsck_refs_fn *fsck_refs_fns)
     +{
    @@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
     +			continue;
     +		} else if (S_ISREG(iter->st.st_mode) ||
     +			   S_ISLNK(iter->st.st_mode)) {
    -+			if (o->fsck_options.verbose)
    ++			if (o->refs_options.verbose)
     +				fprintf_ln(stderr, "Checking %s/%s",
     +					   refs_check_dir, iter->relative_path);
     +			for (size_t i = 0; fsck_refs_fns[i]; i++) {
    @@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
     +}
     +
     +static int files_fsck_refs(struct ref_store *ref_store,
    -+			   struct fsck_refs_options *o)
    ++			   struct fsck_options *o)
     +{
     +	int ret;
     +	files_fsck_refs_fn fsck_refs_fns[]= {
     +		NULL
     +	};
     +
    -+	if (o->fsck_options.verbose)
    ++	if (o->refs_options.verbose)
     +		fprintf_ln(stderr, "Checking references consistency");
     +
     +	ret = files_fsck_refs_dir(ref_store, o, "refs", fsck_refs_fns);
    @@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
     +}
     +
      static int files_fsck(struct ref_store *ref_store,
    - 		      struct fsck_refs_options *o)
    + 		      struct fsck_options *o)
      {
     @@ refs/files-backend.c: static int files_fsck(struct ref_store *ref_store,
      	struct files_ref_store *refs =

11: 7d78014e5f ! 10: 046773e35c fsck: add ref name check for files backend
@@ fsck.h: enum fsck_msg_type {
FUNC(BAD_TREE_SHA1, ERROR) \

## refs/files-backend.c ##
-@@ refs/files-backend.c: typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o, +@@ refs/files-backend.c: typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
const char *refs_check_dir,
struct dir_iterator *iter);

-+static int files_fsck_refs_name(struct fsck_refs_options *o, ++static int files_fsck_refs_name(struct fsck_options *o,
+ const char *gitdir UNUSED,
+ const char *refs_check_dir,
+ struct dir_iterator *iter)
@@ refs/files-backend.c: typedef int (*files_fsck_refs_fn)(struct fsck_refs_options
+}
+
static int files_fsck_refs_dir(struct ref_store *ref_store,
- struct fsck_refs_options *o,
+ struct fsck_options *o,
const char *refs_check_dir,
@@ refs/files-backend.c: static int files_fsck_refs(struct ref_store *ref_store,
{
12: 16209a73ed ! 11: c9b9599589 fsck: add ref content check for files backend
@@ refs/files-backend.c: int parse_loose_ref_contents(const char *buf, struct objec
return 0;
}

-@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *o, +@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
return ret;
}

@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
+ * would be the content after "refs:". For symblic link, "pointee_name" would
+ * be the relative path agaignst "gitdir".
+ */
-+static int files_fsck_symref_target(struct fsck_refs_options *o, ++static int files_fsck_symref_target(struct fsck_options *o,
+ const char *refname,
+ const char *pointee_name,
+ const char *pointee_path)
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
+ return ret;
+}
+
-+static int files_fsck_refs_content(struct fsck_refs_options *o, ++static int files_fsck_refs_content(struct fsck_options *o,
+ const char *gitdir,
+ const char *refs_check_dir,
+ struct dir_iterator *iter)
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
+}
+
static int files_fsck_refs_dir(struct ref_store *ref_store,
- struct fsck_refs_options *o,
+ struct fsck_options *o,
const char *refs_check_dir,
@@ refs/files-backend.c: static int files_fsck_refs(struct ref_store *ref_store,
int ret;

Signature · 1 line
2.45.2
author
#76 lore raw

PATCH v6 1/11 fsck: add "fsck_objects_options" to hold objects-related options

The git-fsck(1) focuses on object database consistency check. It relies on the "fsck_options" to interact with fsck error levels. However, "fsck_options" aims at checking the object database which contains a lot of fields only related to object database.

In order to add ref operations, remove the options related to objects in "fsck_options" to "fsck_objects_options" and let the "fsck_options" incorporate "fsck_objects_options". Change the macros and influenced code to adapt into the above design.

---
 builtin/fsck.c           | 12 +++++-----
 builtin/index-pack.c     |  4 ++--
 builtin/mktag.c          |  2 +-
 builtin/unpack-objects.c |  4 ++--
 fetch-pack.c             | 10 ++++----
 fsck.c                   | 51 +++++++++++++++++++++------------------
 fsck.h                   | 52 ++++++++++++++++++++++++----------------
 object-file.c            |  2 +-
 8 files changed, 77 insertions(+), 60 deletions(-)
builtin/fsck.c +6 -6
index d13a226c2e..13b64f723f 100644
@@ -42,8 +42,8 @@ static int check_full = 1;
 static int connectivity_only;
 static int check_strict;
 static int keep_cache_objects;
-static struct fsck_options fsck_walk_options = FSCK_OPTIONS_DEFAULT;
-static struct fsck_options fsck_obj_options = FSCK_OPTIONS_DEFAULT;
+static struct fsck_options fsck_walk_options = FSCK_OBJECTS_OPTIONS_DEFAULT;
+static struct fsck_options fsck_obj_options = FSCK_OBJECTS_OPTIONS_DEFAULT;
 static int errors_found;
 static int write_lost_and_found;
 static int verbose;
@@ -214,7 +214,7 @@ static int mark_used(struct object *obj, enum object_type type UNUSED,
 
 static void mark_unreachable_referents(const struct object_id *oid)
 {
-	struct fsck_options options = FSCK_OPTIONS_DEFAULT;
+	struct fsck_options options = FSCK_OBJECTS_OPTIONS_DEFAULT;
 	struct object *obj = lookup_object(the_repository, oid);
 
 	if (!obj || !(obj->flags & HAS_OBJ))
@@ -233,7 +233,7 @@ static void mark_unreachable_referents(const struct object_id *oid)
 			object_as_type(obj, type, 0);
 	}
 
-	options.walk = mark_used;
+	options.objects_options.walk = mark_used;
 	fsck_walk(obj, NULL, &options);
 	if (obj->type == OBJ_TREE)
 		free_tree_buffer((struct tree *)obj);
@@ -936,8 +936,8 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
 
 	argc = parse_options(argc, argv, prefix, fsck_opts, fsck_usage, 0);
 
-	fsck_walk_options.walk = mark_object;
-	fsck_obj_options.walk = mark_used;
+	fsck_walk_options.objects_options.walk = mark_object;
+	fsck_obj_options.objects_options.walk = mark_used;
 	fsck_obj_options.error_func = fsck_error_func;
 	if (check_strict)
 		fsck_obj_options.strict = 1;
builtin/index-pack.c +2 -2
index 856428fef9..59eb8c0355 100644
@@ -127,7 +127,7 @@ static int nr_threads;
 static int from_stdin;
 static int strict;
 static int do_fsck_object;
-static struct fsck_options fsck_options = FSCK_OPTIONS_MISSING_GITMODULES;
+static struct fsck_options fsck_options = FSCK_OBJECTS_OPTIONS_MISSING_GITMODULES;
 static int verbose;
 static const char *progress_title;
 static int show_resolving_progress;
@@ -1746,7 +1746,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
 		usage(index_pack_usage);
 
 	disable_replace_refs();
-	fsck_options.walk = mark_link;
+	fsck_options.objects_options.walk = mark_link;
 
 	reset_pack_idx_option(&opts);
 	opts.flags |= WRITE_REV;
builtin/mktag.c +1 -1
index 4767f1a97e..c6fbeb58d4 100644
@@ -15,7 +15,7 @@ static char const * const builtin_mktag_usage[] = {
 };
 static int option_strict = 1;
 
-static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
+static struct fsck_options fsck_options = FSCK_OBJECTS_OPTIONS_STRICT;
 
 static int mktag_fsck_error_func(struct fsck_options *o UNUSED,
 				 const struct object_id *oid UNUSED,
builtin/unpack-objects.c +2 -2
index f1c85a00ae..53cff0a91c 100644
@@ -25,7 +25,7 @@ static unsigned int offset, len;
 static off_t consumed_bytes;
 static off_t max_input_size;
 static git_hash_ctx ctx;
-static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
+static struct fsck_options fsck_options = FSCK_OBJECTS_OPTIONS_STRICT;
 static struct progress *progress;
 
 /*
@@ -239,7 +239,7 @@ static int check_object(struct object *obj, enum object_type type,
 		die("Whoops! Cannot find object '%s'", oid_to_hex(&obj->oid));
 	if (fsck_object(obj, obj_buf->buffer, obj_buf->size, &fsck_options))
 		die("fsck error in packed object");
-	fsck_options.walk = check_object;
+	fsck_options.objects_options.walk = check_object;
 	if (fsck_walk(obj, NULL, &fsck_options))
 		die("Error on reachable objects of %s", oid_to_hex(&obj->oid));
 	write_cached_object(obj, obj_buf);
fetch-pack.c +6 -4
index 42f48fbc31..adce10fb4c 100644
@@ -46,7 +46,7 @@ static int server_supports_filtering;
 static int advertise_sid;
 static struct shallow_lock shallow_lock;
 static const char *alternate_shallow_file;
-static struct fsck_options fsck_options = FSCK_OPTIONS_MISSING_GITMODULES;
+static struct fsck_options fsck_options = FSCK_OBJECTS_OPTIONS_MISSING_GITMODULES;
 static struct strbuf fsck_msg_types = STRBUF_INIT;
 static struct string_list uri_protocols = STRING_LIST_INIT_DUP;
 
@@ -1222,7 +1222,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 	} else
 		alternate_shallow_file = NULL;
 	if (get_pack(args, fd, pack_lockfiles, NULL, sought, nr_sought,
-		     &fsck_options.gitmodules_found))
+		     &fsck_options.objects_options.gitmodules_found))
 		die(_("git fetch-pack: fetch failed."));
 	if (fsck_finish(&fsck_options))
 		die("fsck failed");
@@ -1782,7 +1782,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
 
 			if (get_pack(args, fd, pack_lockfiles,
 				     packfile_uris.nr ? &index_pack_args : NULL,
-				     sought, nr_sought, &fsck_options.gitmodules_found))
+				     sought, nr_sought,
+				     &fsck_options.objects_options.gitmodules_found))
 				die(_("git fetch-pack: fetch failed."));
 			do_check_stateless_delimiter(args->stateless_rpc, &reader);
 
@@ -1825,7 +1826,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
 
 		packname[the_hash_algo->hexsz] = '\0';
 
-		parse_gitmodules_oids(cmd.out, &fsck_options.gitmodules_found);
+		parse_gitmodules_oids(cmd.out,
+				      &fsck_options.objects_options.gitmodules_found);
 
 		close(cmd.out);
 
fsck.c +28 -23

158 lines not on this page · raw message

fsck.h +31 -21

72 lines not on this page · raw message

object-file.c +1 -1
index d3cf4b8b2e..4e10dd9804 100644
@@ -2507,7 +2507,7 @@ static int index_mem(struct index_state *istate,
 		}
 	}
 	if (flags & HASH_FORMAT_CHECK) {
-		struct fsck_options opts = FSCK_OPTIONS_DEFAULT;
+		struct fsck_options opts = FSCK_OBJECTS_OPTIONS_DEFAULT;
 
 		opts.strict = 1;
 		opts.error_func = hash_format_check_report;
Signature · 1 line
2.45.2
author
#77 lore raw

PATCH v6 2/11 fsck: rename "skiplist" to "oid_skiplist"

In reply to shejialuo

Because we introduce ref consistency check. The original "skiplist" is a common option which is set up during handling user configs. To avoid causing ambiguity, rename "skiplist" to "oid_skiplist".

---
 fsck.c | 4 ++--
 fsck.h | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)
fsck.c +2 -2
index e9848f2678..04d0aa1766 100644
@@ -203,7 +203,7 @@ void fsck_set_msg_types(struct fsck_options *options, const char *values)
 		if (!strcmp(buf, "skiplist")) {
 			if (equal == len)
 				die("skiplist requires a path");
-			oidset_parse_file(&options->skiplist, buf + equal + 1);
+			oidset_parse_file(&options->oid_skiplist, buf + equal + 1);
 			buf += len + 1;
 			continue;
 		}
@@ -220,7 +220,7 @@ void fsck_set_msg_types(struct fsck_options *options, const char *values)
 static int object_on_skiplist(struct fsck_options *opts,
 			      const struct object_id *oid)
 {
-	return opts && oid && oidset_contains(&opts->skiplist, oid);
+	return opts && oid && oidset_contains(&opts->oid_skiplist, oid);
 }
 
 __attribute__((format (printf, 5, 6)))
fsck.h +2 -2
index 37b6f6676f..0c581ef082 100644
@@ -143,13 +143,13 @@ struct fsck_objects_options {
 struct fsck_options {
 	unsigned strict:1;
 	enum fsck_msg_type *msg_type;
-	struct oidset skiplist;
+	struct oidset oid_skiplist;
 	fsck_error error_func;
 	struct fsck_objects_options objects_options;
 };
 
 #define FSCK_OBJECTS_OPTIONS_DEFAULT { \
-	.skiplist = OIDSET_INIT, \
+	.oid_skiplist = OIDSET_INIT, \
 	.error_func = fsck_error_function, \
 	.objects_options = { \
 		.gitmodules_found = OIDSET_INIT, \
Signature · 1 line
2.45.2
author
#78 lore raw

PATCH v6 3/11 fsck: add "fsck_refs_options" into "fsck_options"

In reply to shejialuo

Introduce "fsck_refs_options" to represent the refs-related options.

---
 fsck.h | 5 +++++
 1 file changed, 5 insertions(+)
fsck.h +5 -0
index 0c581ef082..ff7281b410 100644
@@ -131,6 +131,10 @@ int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o,
 					   enum fsck_msg_id msg_id,
 					   const char *message);
 
+struct fsck_refs_options {
+	unsigned verbose:1;
+};
+
 struct fsck_objects_options {
 	fsck_walk_func walk;
 	struct oidset gitmodules_found;
@@ -146,6 +150,7 @@ struct fsck_options {
 	struct oidset oid_skiplist;
 	fsck_error error_func;
 	struct fsck_objects_options objects_options;
+	struct fsck_refs_options refs_options;
 };
 
 #define FSCK_OBJECTS_OPTIONS_DEFAULT { \
Signature · 1 line
2.45.2
author
#79 lore raw

PATCH v6 4/11 fsck: add a unified interface for reporting fsck messages

In reply to shejialuo

The static function "report" provided by "fsck.c" aims at reporting the problems related to object database which cannot be reused for refs. In order to provide a unified interface which can report either objects or refs, create a new function "vfsck_report" by adding "checked_ref_name" parameter following the "report" prototype. However, instead of using "...", provide "va_list" to allow more flexibility.

The "vfsck_report" function will use "error_func" registered in "fsck_options" function to report customized messages. Change "error_func" prototype to align with the "vfsck_report".

Change "report" function to make it use "vfsck_report" to report objects-related messages. Add a new function called "fsck_refs_report" to use "vfsck_report" to report refs-related messages.

---
 builtin/fsck.c  | 15 ++++-----
 builtin/mktag.c |  1 +
 fsck.c          | 81 ++++++++++++++++++++++++++++++++++++-------------
 fsck.h          | 40 +++++++++++++++---------
 object-file.c   | 11 ++++---
 5 files changed, 101 insertions(+), 47 deletions(-)
builtin/fsck.c +8 -7
index 13b64f723f..cdf2218dfd 100644
@@ -89,12 +89,13 @@ static int objerror(struct object *obj, const char *err)
 	return -1;
 }
 
-static int fsck_error_func(struct fsck_options *o UNUSED,
-			   const struct object_id *oid,
-			   enum object_type object_type,
-			   enum fsck_msg_type msg_type,
-			   enum fsck_msg_id msg_id UNUSED,
-			   const char *message)
+static int fsck_objects_error_func(struct fsck_options *o UNUSED,
+				   const struct object_id *oid,
+				   enum object_type object_type,
+				   const char *checked_ref_name UNUSED,
+				   enum fsck_msg_type msg_type,
+				   enum fsck_msg_id msg_id UNUSED,
+				   const char *message)
 {
 	switch (msg_type) {
 	case FSCK_WARN:
@@ -938,7 +939,7 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
 
 	fsck_walk_options.objects_options.walk = mark_object;
 	fsck_obj_options.objects_options.walk = mark_used;
-	fsck_obj_options.error_func = fsck_error_func;
+	fsck_obj_options.error_func = fsck_objects_error_func;
 	if (check_strict)
 		fsck_obj_options.strict = 1;
 
builtin/mktag.c +1 -0
index c6fbeb58d4..d8e4b91134 100644
@@ -20,6 +20,7 @@ static struct fsck_options fsck_options = FSCK_OBJECTS_OPTIONS_STRICT;
 static int mktag_fsck_error_func(struct fsck_options *o UNUSED,
 				 const struct object_id *oid UNUSED,
 				 enum object_type object_type UNUSED,
+				 const char *checked_ref_name UNUSED,
 				 enum fsck_msg_type msg_type,
 				 enum fsck_msg_id msg_id UNUSED,
 				 const char *message)
fsck.c +60 -21

115 lines not on this page · raw message

fsck.h +26 -14
index ff7281b410..e611b3b65b 100644
@@ -114,22 +114,27 @@ int is_valid_msg_type(const char *msg_id, const char *msg_type);
 typedef int (*fsck_walk_func)(struct object *obj, enum object_type object_type,
 			      void *data, struct fsck_options *options);
 
-/* callback for fsck_object, type is FSCK_ERROR or FSCK_WARN */
+/*
+ * callback function for reporting errors when checking either objects or refs
+ */
 typedef int (*fsck_error)(struct fsck_options *o,
 			  const struct object_id *oid, enum object_type object_type,
+			  const char *checked_ref_name,
 			  enum fsck_msg_type msg_type, enum fsck_msg_id msg_id,
 			  const char *message);
 
-int fsck_error_function(struct fsck_options *o,
-			const struct object_id *oid, enum object_type object_type,
-			enum fsck_msg_type msg_type, enum fsck_msg_id msg_id,
-			const char *message);
-int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o,
-					   const struct object_id *oid,
-					   enum object_type object_type,
-					   enum fsck_msg_type msg_type,
-					   enum fsck_msg_id msg_id,
-					   const char *message);
+int fsck_objects_error_function(struct fsck_options *o,
+				const struct object_id *oid, enum object_type object_type,
+				const char *checked_ref_name,
+				enum fsck_msg_type msg_type, enum fsck_msg_id msg_id,
+				const char *message);
+int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o,
+						   const struct object_id *oid,
+						   enum object_type object_type,
+						   const char *checked_ref_name,
+						   enum fsck_msg_type msg_type,
+						   enum fsck_msg_id msg_id,
+						   const char *message);
 
 struct fsck_refs_options {
 	unsigned verbose:1;
@@ -155,7 +160,7 @@ struct fsck_options {
 
 #define FSCK_OBJECTS_OPTIONS_DEFAULT { \
 	.oid_skiplist = OIDSET_INIT, \
-	.error_func = fsck_error_function, \
+	.error_func = fsck_objects_error_function, \
 	.objects_options = { \
 		.gitmodules_found = OIDSET_INIT, \
 		.gitmodules_done = OIDSET_INIT, \
@@ -165,7 +170,7 @@ struct fsck_options {
 }
 #define FSCK_OBJECTS_OPTIONS_STRICT { \
 	.strict = 1, \
-	.error_func = fsck_error_function, \
+	.error_func = fsck_objects_error_function, \
 	.objects_options = { \
 		.gitmodules_found = OIDSET_INIT, \
 		.gitmodules_done = OIDSET_INIT, \
@@ -175,7 +180,7 @@ struct fsck_options {
 }
 #define FSCK_OBJECTS_OPTIONS_MISSING_GITMODULES { \
 	.strict = 1, \
-	.error_func = fsck_error_cb_print_missing_gitmodules, \
+	.error_func = fsck_objects_error_cb_print_missing_gitmodules, \
 	.objects_options = { \
 		.gitmodules_found = OIDSET_INIT, \
 		.gitmodules_done = OIDSET_INIT, \
@@ -224,6 +229,13 @@ int fsck_tag_standalone(const struct object_id *oid, const char *buffer,
  */
 int fsck_finish(struct fsck_options *options);
 
+__attribute__((format (printf, 5, 6)))
+int fsck_refs_report(struct fsck_options *options,
+		     const struct object_id *oid,
+		     const char *checked_ref_name,
+		     enum fsck_msg_id msg_id,
+		     const char *fmt, ...);
+
 /*
  * Subsystem for storing human-readable names for each object.
  *
object-file.c +6 -5
index 4e10dd9804..a681087625 100644
@@ -2473,11 +2473,12 @@ int repo_has_object_file(struct repository *r,
  * give more context.
  */
 static int hash_format_check_report(struct fsck_options *opts UNUSED,
-				     const struct object_id *oid UNUSED,
-				     enum object_type object_type UNUSED,
-				     enum fsck_msg_type msg_type UNUSED,
-				     enum fsck_msg_id msg_id UNUSED,
-				     const char *message)
+				    const struct object_id *oid UNUSED,
+				    enum object_type object_type UNUSED,
+				    const char *ref_checked_name UNUSED,
+				    enum fsck_msg_type msg_type UNUSED,
+				    enum fsck_msg_id msg_id UNUSED,
+				    const char *message)
 {
 	error(_("object fails fsck: %s"), message);
 	return 1;
Signature · 1 line
2.45.2
author
#80 lore raw

PATCH v6 5/11 fsck: add "fsck_refs_options" initialization macros

In reply to shejialuo

Add "FSCK_REFS_OPTIONS_DEFAULT" and "FSCK_REFS_OPTIONS_STRICT" macros to create the refs options easily. Add refs-specific "error_func" callback "fsck_refs_error_function".

"fsck_refs_error_function" will use the "oid" parameter. When the caller passed the oid, it will use "oid_to_hex" to get the corresponding hex value to report to the user.

---
 fsck.c | 22 ++++++++++++++++++++++
 fsck.h | 14 ++++++++++++++
 2 files changed, 36 insertions(+)
fsck.c +22 -0
index 31ae8d57ce..ea00fb4790 100644
@@ -1251,6 +1251,28 @@ int fsck_objects_error_function(struct fsck_options *o,
 	return 1;
 }
 
+int fsck_refs_error_function(struct fsck_options *options UNUSED,
+			     const struct object_id *oid,
+			     enum object_type object_type UNUSED,
+			     const char *checked_ref_name,
+			     enum fsck_msg_type msg_type,
+			     enum fsck_msg_id msg_id UNUSED,
+			     const char *message)
+{
+	static struct strbuf sb = STRBUF_INIT;
+
+	strbuf_addstr(&sb, checked_ref_name);
+	if (oid)
+		strbuf_addf(&sb, " -> (%s)", oid_to_hex(oid));
+
+	if (msg_type == FSCK_WARN) {
+		warning("%s: %s", sb.buf, message);
+		return 0;
+	}
+	error("%s: %s", sb.buf, message);
+	return 1;
+}
+
 static int fsck_blobs(struct oidset *blobs_found, struct oidset *blobs_done,
 		      enum fsck_msg_id msg_missing, enum fsck_msg_id msg_type,
 		      struct fsck_options *options, const char *blob_type)
fsck.h +14 -0
index e611b3b65b..887fe2969f 100644
@@ -135,6 +135,13 @@ int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o,
 						   enum fsck_msg_type msg_type,
 						   enum fsck_msg_id msg_id,
 						   const char *message);
+int fsck_refs_error_function(struct fsck_options *options,
+			     const struct object_id *oid,
+			     enum object_type object_type,
+			     const char *checked_ref_name,
+			     enum fsck_msg_type msg_type,
+			     enum fsck_msg_id msg_id,
+			     const char *message);
 
 struct fsck_refs_options {
 	unsigned verbose:1;
@@ -188,6 +195,13 @@ struct fsck_options {
 		.gitattributes_done = OIDSET_INIT, \
 	} \
 }
+#define FSCK_REFS_OPTIONS_DEFAULT { \
+	.error_func = fsck_refs_error_function, \
+}
+#define FSCK_REFS_OPTIONS_STRICT { \
+	.strict = 1, \
+	.error_func = fsck_refs_error_function, \
+}
 
 /* descend in all linked child objects
  * the return value is:
Signature · 1 line
2.45.2
author
#81 lore raw

PATCH v6 6/11 refs: set up ref consistency check infrastructure

In reply to shejialuo

The interfaces defined in the ref_storage_be are carefully structured in semantic. It's organized as the five parts:

1. The name and the initialization interfaces.
2. The ref transaction interfaces.
3. The ref internal interfaces (pack, rename and copy).
4. The ref filesystem interfaces.
5. The reflog related interfaces.

To keep consistent with the git-fsck(1), add a new interface named "fsck_refs_fn" to the end of "ref_storage_be". This semantic cannot be grouped into any above five categories. Explicitly add blank line to make it different from others.

---
 refs.c                  |  5 +++++
 refs.h                  |  8 ++++++++
 refs/debug.c            | 11 +++++++++++
 refs/files-backend.c    | 15 ++++++++++++++-
 refs/packed-backend.c   |  8 ++++++++
 refs/refs-internal.h    |  6 ++++++
 refs/reftable-backend.c |  8 ++++++++
 7 files changed, 60 insertions(+), 1 deletion(-)
refs.c +5 -0
index dd0d9c360f..5f17de38e2 100644
@@ -316,6 +316,11 @@ int check_refname_format(const char *refname, int flags)
 	return check_or_sanitize_refname(refname, flags, NULL);
 }
 
+int refs_fsck(struct ref_store *refs, struct fsck_options *o)
+{
+	return refs->be->fsck(refs, o);
+}
+
 void sanitize_refname_component(const char *refname, struct strbuf *out)
 {
 	if (check_or_sanitize_refname(refname, REFNAME_ALLOW_ONELEVEL, out))
refs.h +8 -0
index df8cbf6124..946e929af9 100644
@@ -3,6 +3,7 @@
 
 #include "commit.h"
 
+struct fsck_options;
 struct object_id;
 struct ref_store;
 struct repository;
@@ -547,6 +548,13 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat
  */
 int check_refname_format(const char *refname, int flags);
 
+/*
+ * Check the reference database for consistency. Return 0 if refs and
+ * reflogs are consistent, and non-zero otherwise. The errors will be
+ * written to stderr.
+ */
+int refs_fsck(struct ref_store *refs, struct fsck_options *o);
+
 /*
  * Apply the rules from check_refname_format, but mutate the result until it
  * is acceptable, and place the result in "out".
refs/debug.c +11 -0
index 547d9245b9..45e2e784a0 100644
@@ -419,6 +419,15 @@ static int debug_reflog_expire(struct ref_store *ref_store, const char *refname,
 	return res;
 }
 
+static int debug_fsck(struct ref_store *ref_store,
+		      struct fsck_options *o)
+{
+	struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store;
+	int res = drefs->refs->be->fsck(drefs->refs, o);
+	trace_printf_key(&trace_refs, "fsck: %d\n", res);
+	return res;
+}
+
 struct ref_storage_be refs_be_debug = {
 	.name = "debug",
 	.init = NULL,
@@ -451,4 +460,6 @@ struct ref_storage_be refs_be_debug = {
 	.create_reflog = debug_create_reflog,
 	.delete_reflog = debug_delete_reflog,
 	.reflog_expire = debug_reflog_expire,
+
+	.fsck = debug_fsck,
 };
refs/files-backend.c +14 -1
index 44c5c3b201..817813e723 100644
@@ -3406,6 +3406,17 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,
 	return ret;
 }
 
+static int files_fsck(struct ref_store *ref_store,
+		      struct fsck_options *o)
+{
+	int ret;
+	struct files_ref_store *refs =
+		files_downcast(ref_store, REF_STORE_READ, "fsck");
+
+	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o);
+	return ret;
+}
+
 struct ref_storage_be refs_be_files = {
 	.name = "files",
 	.init = files_ref_store_init,
@@ -3432,5 +3443,7 @@ struct ref_storage_be refs_be_files = {
 	.reflog_exists = files_reflog_exists,
 	.create_reflog = files_create_reflog,
 	.delete_reflog = files_delete_reflog,
-	.reflog_expire = files_reflog_expire
+	.reflog_expire = files_reflog_expire,
+
+	.fsck = files_fsck,
 };
refs/packed-backend.c +8 -0
index c4c1e36aa2..ad3e8fb1d1 100644
@@ -1733,6 +1733,12 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_s
 	return empty_ref_iterator_begin();
 }
 
+static int packed_fsck(struct ref_store *ref_store,
+		       struct fsck_options *o)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_packed = {
 	.name = "packed",
 	.init = packed_ref_store_init,
@@ -1760,4 +1766,6 @@ struct ref_storage_be refs_be_packed = {
 	.create_reflog = NULL,
 	.delete_reflog = NULL,
 	.reflog_expire = NULL,
+
+	.fsck = packed_fsck,
 };
refs/refs-internal.h +6 -0
index fa975d69aa..a905e187cd 100644
@@ -4,6 +4,7 @@
 #include "refs.h"
 #include "iterator.h"
 
+struct fsck_options;
 struct ref_transaction;
 
 /*
@@ -650,6 +651,9 @@ typedef int read_raw_ref_fn(struct ref_store *ref_store, const char *refname,
 typedef int read_symbolic_ref_fn(struct ref_store *ref_store, const char *refname,
 				 struct strbuf *referent);
 
+typedef int fsck_fn(struct ref_store *ref_store,
+		    struct fsck_options *o);
+
 struct ref_storage_be {
 	const char *name;
 	ref_store_init_fn *init;
@@ -677,6 +681,8 @@ struct ref_storage_be {
 	create_reflog_fn *create_reflog;
 	delete_reflog_fn *delete_reflog;
 	reflog_expire_fn *reflog_expire;
+
+	fsck_fn *fsck;
 };
 
 extern struct ref_storage_be refs_be_files;
refs/reftable-backend.c +8 -0
index 7d872a32ac..7f07de8d45 100644
@@ -2292,6 +2292,12 @@ static int reftable_be_reflog_expire(struct ref_store *ref_store,
 	return ret;
 }
 
+static int reftable_be_fsck(struct ref_store *ref_store,
+			    struct fsck_options *o)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_reftable = {
 	.name = "reftable",
 	.init = reftable_be_init,
@@ -2319,4 +2325,6 @@ struct ref_storage_be refs_be_reftable = {
 	.create_reflog = reftable_be_create_reflog,
 	.delete_reflog = reftable_be_delete_reflog,
 	.reflog_expire = reftable_be_reflog_expire,
+
+	.fsck = reftable_be_fsck,
 };
Signature · 1 line
2.45.2
author
#82 lore raw

PATCH v6 7/11 builtin/refs: add verify subcommand

In reply to shejialuo

Introduce a new subcommand "verify" in git-refs(1) to allow the user to check the reference database consistency.

---
 Documentation/git-refs.txt | 13 +++++++++++
 builtin/refs.c             | 44 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
Documentation/git-refs.txt +13 -0
index 5b99e04385..1244a85b64 100644
@@ -10,6 +10,7 @@ SYNOPSIS
 --------
 [verse]
 'git refs migrate' --ref-format=<format> [--dry-run]
+'git refs verify' [--strict] [--verbose]
 
 DESCRIPTION
 -----------
@@ -22,6 +23,9 @@ COMMANDS
 migrate::
 	Migrate ref store between different formats.
 
+verify::
+	Verify reference database consistency.
+
 OPTIONS
 -------
 
@@ -39,6 +43,15 @@ include::ref-storage-format.txt[]
 	can be used to double check that the migration works as expected before
 	performing the actual migration.
 
+The following options are specific to 'git refs verify':
+
+--strict::
+	Enable more strict checking, every WARN severity for the `Fsck Messages`
+	be seen as ERROR. See linkgit:git-fsck[1].
+
+--verbose::
+	When verifying the reference database consistency, be chatty.
+
 KNOWN LIMITATIONS
 -----------------
 
builtin/refs.c +44 -0

69 lines not on this page · raw message

Signature · 1 line
2.45.2
author
#83 lore raw

PATCH v6 8/11 builtin/fsck: add `git-refs verify` child process

In reply to shejialuo

Introduce a new function "fsck_refs" that initializes and runs a child process to execute the "git-refs verify" command.

---
 builtin/fsck.c | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
builtin/fsck.c +17 -0
index cdf2218dfd..aea635ea00 100644
@@ -897,6 +897,21 @@ static int check_pack_rev_indexes(struct repository *r, int show_progress)
 	return res;
 }
 
+static void fsck_refs(void)
+{
+	struct child_process refs_verify = CHILD_PROCESS_INIT;
+	child_process_init(&refs_verify);
+	refs_verify.git_cmd = 1;
+	strvec_pushl(&refs_verify.args, "refs", "verify", NULL);
+	if (verbose)
+		strvec_push(&refs_verify.args, "--verbose");
+	if (check_strict)
+		strvec_push(&refs_verify.args, "--strict");
+
+	if (run_command(&refs_verify))
+		errors_found |= ERROR_REFS;
+}
+
 static char const * const fsck_usage[] = {
 	N_("git fsck [--tags] [--root] [--unreachable] [--cache] [--no-reflogs]\n"
 	   "         [--[no-]full] [--strict] [--verbose] [--lost-found]\n"
@@ -1066,6 +1081,8 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
 
 	check_connectivity();
 
+	fsck_refs();
+
 	if (the_repository->settings.core_commit_graph) {
 		struct child_process commit_graph_verify = CHILD_PROCESS_INIT;
 
Signature · 1 line
2.45.2
author
#84 lore raw

PATCH v6 9/11 files-backend: add unified interface for refs scanning

In reply to shejialuo

For refs and reflogs, we need to scan its corresponding directories to check every regular file or symbolic link which shares the same pattern. Introduce a unified interface for scanning directories for files-backend.

---
 refs/files-backend.c | 77 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 76 insertions(+), 1 deletion(-)
refs/files-backend.c +76 -1
index 817813e723..744ee5c5db 100644
@@ -4,6 +4,7 @@
 #include "../gettext.h"
 #include "../hash.h"
 #include "../hex.h"
+#include "../fsck.h"
 #include "../refs.h"
 #include "refs-internal.h"
 #include "ref-cache.h"
@@ -3406,6 +3407,78 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,
 	return ret;
 }
 
+/*
+ * For refs and reflogs, they share a unified interface when scanning
+ * the whole directory. This function is used as the callback for each
+ * regular file or symlink in the directory.
+ */
+typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
+				  const char *gitdir,
+				  const char *refs_check_dir,
+				  struct dir_iterator *iter);
+
+static int files_fsck_refs_dir(struct ref_store *ref_store,
+			       struct fsck_options *o,
+			       const char *refs_check_dir,
+			       files_fsck_refs_fn *fsck_refs_fns)
+{
+	const char *gitdir = ref_store->gitdir;
+	struct strbuf sb = STRBUF_INIT;
+	struct dir_iterator *iter;
+	int iter_status;
+	int ret = 0;
+
+	strbuf_addf(&sb, "%s/%s", gitdir, refs_check_dir);
+
+	iter = dir_iterator_begin(sb.buf, 0);
+
+	if (!iter) {
+		ret = error_errno("cannot open directory %s", sb.buf);
+		goto out;
+	}
+
+	while ((iter_status = dir_iterator_advance(iter)) == ITER_OK) {
+		if (S_ISDIR(iter->st.st_mode)) {
+			continue;
+		} else if (S_ISREG(iter->st.st_mode) ||
+			   S_ISLNK(iter->st.st_mode)) {
+			if (o->refs_options.verbose)
+				fprintf_ln(stderr, "Checking %s/%s",
+					   refs_check_dir, iter->relative_path);
+			for (size_t i = 0; fsck_refs_fns[i]; i++) {
+				if (fsck_refs_fns[i](o, gitdir, refs_check_dir, iter))
+					ret = -1;
+			}
+		} else {
+			ret = error(_("unexpected file type for '%s'"),
+				    iter->basename);
+		}
+	}
+
+	if (iter_status != ITER_DONE)
+		ret = error(_("failed to iterate over '%s'"), sb.buf);
+
+out:
+	strbuf_release(&sb);
+	return ret;
+}
+
+static int files_fsck_refs(struct ref_store *ref_store,
+			   struct fsck_options *o)
+{
+	int ret;
+	files_fsck_refs_fn fsck_refs_fns[]= {
+		NULL
+	};
+
+	if (o->refs_options.verbose)
+		fprintf_ln(stderr, "Checking references consistency");
+
+	ret = files_fsck_refs_dir(ref_store, o, "refs", fsck_refs_fns);
+
+	return ret;
+}
+
 static int files_fsck(struct ref_store *ref_store,
 		      struct fsck_options *o)
 {
@@ -3413,7 +3486,9 @@ static int files_fsck(struct ref_store *ref_store,
 	struct files_ref_store *refs =
 		files_downcast(ref_store, REF_STORE_READ, "fsck");
 
-	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o);
+	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o)
+	    | files_fsck_refs(ref_store, o);
+
 	return ret;
 }
 
Signature · 1 line
2.45.2
author
#85 lore raw

PATCH v6 10/11 fsck: add ref name check for files backend

In reply to shejialuo

The git-fsck(1) only implicitly checks the reference, it does not fully check refs with bad format name such as standalone "@" and name ending with ".lock".

In order to provide such checks, add a new fsck message id "badRefName" with default ERROR type. Use existing "check_refname_format" to explicit check the ref name. And add a new unit test to verify the functionality.

---
 Documentation/fsck-msgids.txt |   3 +
 fsck.h                        |   1 +
 refs/files-backend.c          |  20 +++++++
 t/t0602-reffiles-fsck.sh      | 101 ++++++++++++++++++++++++++++++++++
 4 files changed, 125 insertions(+)
 create mode 100755 t/t0602-reffiles-fsck.sh
Documentation/fsck-msgids.txt +3 -0
index f643585a34..dab4012246 100644
@@ -19,6 +19,9 @@
 `badParentSha1`::
 	(ERROR) A commit object has a bad parent sha1.
 
+`badRefName`::
+	(ERROR) A ref has a bad name.
+
 `badTagName`::
 	(INFO) A tag has an invalid format.
 
fsck.h +1 -0
index 887fe2969f..fb035d5457 100644
@@ -31,6 +31,7 @@ enum fsck_msg_type {
 	FUNC(BAD_NAME, ERROR) \
 	FUNC(BAD_OBJECT_SHA1, ERROR) \
 	FUNC(BAD_PARENT_SHA1, ERROR) \
+	FUNC(BAD_REF_NAME, ERROR) \
 	FUNC(BAD_TIMEZONE, ERROR) \
 	FUNC(BAD_TREE, ERROR) \
 	FUNC(BAD_TREE_SHA1, ERROR) \
refs/files-backend.c +20 -0
index 744ee5c5db..3badec62ce 100644
@@ -3417,6 +3417,25 @@ typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
 				  const char *refs_check_dir,
 				  struct dir_iterator *iter);
 
+static int files_fsck_refs_name(struct fsck_options *o,
+				const char *gitdir UNUSED,
+				const char *refs_check_dir,
+				struct dir_iterator *iter)
+{
+	struct strbuf sb = STRBUF_INIT;
+	int ret = 0;
+
+	if (check_refname_format(iter->basename, REFNAME_ALLOW_ONELEVEL)) {
+		strbuf_addf(&sb, "%s/%s", refs_check_dir, iter->relative_path);
+		ret = fsck_refs_report(o, NULL, sb.buf,
+				       FSCK_MSG_BAD_REF_NAME,
+				       "invalid refname format");
+	}
+
+	strbuf_release(&sb);
+	return ret;
+}
+
 static int files_fsck_refs_dir(struct ref_store *ref_store,
 			       struct fsck_options *o,
 			       const char *refs_check_dir,
@@ -3468,6 +3487,7 @@ static int files_fsck_refs(struct ref_store *ref_store,
 {
 	int ret;
 	files_fsck_refs_fn fsck_refs_fns[]= {
+		files_fsck_refs_name,
 		NULL
 	};
 
t/t0602-reffiles-fsck.sh new file +101 -0

101 lines not on this page · raw message

Signature · 1 line
2.45.2
author
#86 lore raw

PATCH v6 11/11 fsck: add ref content check for files backend

In reply to shejialuo

Enhance the git-fsck(1) command by adding a check for reference content in the files backend. The new functionality ensures that symrefs, real symbolic link and regular refs are validated correctly.

In order to check the trailing content of the regular refs, add a new parameter trailing to parse_loose_ref_contents.

For symrefs, parse_loose_ref_contents will set the "referent". However, symbolic link could be either absolute or relative. Use "strbuf_add_real_path" to read the symbolic link and convert the relative path to absolute path. Then use "skip_prefix" to make it align with symref "referent".

Thus, the symrefs and symbolic links could share the same interface. Add a new function "files_fsck_symref_target" which aims at checking the following things:

1. whether the pointee is under the refs/ directory.
2. whether the pointee name is correct.
3. whether the pointee path is a wrong type in filesystem.

Last, add the following FSCK MESSAGEs:

1. "badRefContent(ERROR)": A ref has a bad content
2. "badSymrefPointee(ERROR)": The pointee of a symref is bad.
3. "trailingRefContent(WARN)": A ref content has trailing contents.

---
 Documentation/fsck-msgids.txt |   9 +++
 fsck.h                        |   3 +
 refs.c                        |   2 +-
 refs/files-backend.c          | 145 +++++++++++++++++++++++++++++++++-
 refs/refs-internal.h          |   5 +-
 t/t0602-reffiles-fsck.sh      | 110 ++++++++++++++++++++++++++
 6 files changed, 269 insertions(+), 5 deletions(-)
Documentation/fsck-msgids.txt +9 -0
index dab4012246..b1630a478b 100644
@@ -19,9 +19,15 @@
 `badParentSha1`::
 	(ERROR) A commit object has a bad parent sha1.
 
+`badRefContent`::
+	(ERROR) A ref has a bad content.
+
 `badRefName`::
 	(ERROR) A ref has a bad name.
 
+`badSymrefPointee`::
+	(ERROR) The pointee of a symref is bad.
+
 `badTagName`::
 	(INFO) A tag has an invalid format.
 
@@ -167,6 +173,9 @@
 `nullSha1`::
 	(WARN) Tree contains entries pointing to a null sha1.
 
+`trailingRefContent`::
+	(WARN) A ref content has trailing contents.
+
 `treeNotSorted`::
 	(ERROR) A tree is not properly sorted.
 
fsck.h +3 -0
index fb035d5457..e917b221f6 100644
@@ -32,6 +32,8 @@ enum fsck_msg_type {
 	FUNC(BAD_OBJECT_SHA1, ERROR) \
 	FUNC(BAD_PARENT_SHA1, ERROR) \
 	FUNC(BAD_REF_NAME, ERROR) \
+	FUNC(BAD_REF_CONTENT, ERROR) \
+	FUNC(BAD_SYMREF_POINTEE, ERROR) \
 	FUNC(BAD_TIMEZONE, ERROR) \
 	FUNC(BAD_TREE, ERROR) \
 	FUNC(BAD_TREE_SHA1, ERROR) \
@@ -72,6 +74,7 @@ enum fsck_msg_type {
 	FUNC(HAS_DOTDOT, WARN) \
 	FUNC(HAS_DOTGIT, WARN) \
 	FUNC(NULL_SHA1, WARN) \
+	FUNC(TRAILING_REF_CONTENT, WARN) \
 	FUNC(ZERO_PADDED_FILEMODE, WARN) \
 	FUNC(NUL_IN_COMMIT, WARN) \
 	FUNC(LARGE_PATHNAME, WARN) \
refs.c +1 -1
index 5f17de38e2..4e94f92997 100644
@@ -1758,7 +1758,7 @@ static int refs_read_special_head(struct ref_store *ref_store,
 	}
 
 	result = parse_loose_ref_contents(content.buf, oid, referent, type,
-					  failure_errno);
+					  failure_errno, NULL);
 
 done:
 	strbuf_release(&full_path);
refs/files-backend.c +143 -2

179 lines not on this page · raw message

refs/refs-internal.h +3 -2
index a905e187cd..2fabf41d14 100644
@@ -709,11 +709,12 @@ struct ref_store {
 
 /*
  * Parse contents of a loose ref file. *failure_errno maybe be set to EINVAL for
- * invalid contents.
+ * invalid contents. Also *trailing is set to the first character after the
+ * refname or NULL if the referent is not empty.
  */
 int parse_loose_ref_contents(const char *buf, struct object_id *oid,
 			     struct strbuf *referent, unsigned int *type,
-			     int *failure_errno);
+			     int *failure_errno, const char **trailing);
 
 /*
  * Fill in the generic part of refs and add it to our collection of
t/t0602-reffiles-fsck.sh +110 -0

114 lines not on this page · raw message

Signature · 1 line
2.45.2
#87 lore raw
In reply to shejialuo

Hello,

Quoted 17 lines · shejialuo

Hi All:

This version follows the Junio's advice. Instead of creating the following data structure:

	struct fsck_options {
		enum fsck_type {
			FSCK_OBJECTS,
			FSCK_REFS,
			...
		} t;
		union {
			struct fsck_objects_options objects;
			struct fsck_refs_options refs;
		} u;
	};
Quoted 5 lines

I simply use the combination idea where "fsck_options" will incorporate "fsck_objects_options" and "fsck_refs_options". Karthik has told me that I should balance the job I should does and the extensibility for future. So I use the most clear way to do this. Also Junio has said:

If I understood Junio's comments correctly, he was drawing out the point about if we even need the separation of options for refs. Since the only option we're adding is a verbose:

    struct fsck_refs_options {
    	unsigned verbose:1;
    };

wouldn't it be better if we simply amended fsck_options as so:

    diff --git a/fsck.h b/fsck.h
    index 6085a384f6..ea97f48acc 100644
    --- a/fsck.h
    +++ b/fsck.h
    @@ -135,6 +135,7 @@ struct fsck_options {
     	fsck_walk_func walk;
     	fsck_error error_func;
     	unsigned strict:1;
    +	unsigned verbose_refs:1;
     	enum fsck_msg_type *msg_type;
     	struct oidset skiplist;
     	struct oidset gitmodules_found;

Your approach seems to take a different path though, where we create a new route of creating two new structs, one for refs and another for objects and adding both to fsck_objects. If we're doing this, wouldn't it be better to use the enum+union idea, like Junio mentioned? That way we would have clarity around which type it represents.

[snip]

author
#88 lore raw
Quoted 50 lines · Karthik Nayak

Hello,

Quoted 17 lines · shejialuo

Hi All:

This version follows the Junio's advice. Instead of creating the following data structure:

	struct fsck_options {
		enum fsck_type {
			FSCK_OBJECTS,
			FSCK_REFS,
			...
		} t;
		union {
			struct fsck_objects_options objects;
			struct fsck_refs_options refs;
		} u;
	};
Quoted 5 lines

I simply use the combination idea where "fsck_options" will incorporate "fsck_objects_options" and "fsck_refs_options". Karthik has told me that I should balance the job I should does and the extensibility for future. So I use the most clear way to do this. Also Junio has said:

If I understood Junio's comments correctly, he was drawing out the point about if we even need the separation of options for refs. Since the only option we're adding is a verbose:

    struct fsck_refs_options {
    	unsigned verbose:1;
    };

wouldn't it be better if we simply amended fsck_options as so:

    diff --git a/fsck.h b/fsck.h
    index 6085a384f6..ea97f48acc 100644
    --- a/fsck.h
    +++ b/fsck.h
    @@ -135,6 +135,7 @@ struct fsck_options {
     	fsck_walk_func walk;
     	fsck_error error_func;
     	unsigned strict:1;
    +	unsigned verbose_refs:1;
     	enum fsck_msg_type *msg_type;
     	struct oidset skiplist;
     	struct oidset gitmodules_found;
Quoted 6 lines

Your approach seems to take a different path though, where we create a new route of creating two new structs, one for refs and another for objects and adding both to fsck_objects. If we're doing this, wouldn't it be better to use the enum+union idea, like Junio mentioned? That way we would have clarity around which type it represents.

I agree. Let's give up breaking the structs. I will send a new version immediately.

Thanks.

Quoted 1 line

[snip]

#89 lore raw
In reply to Karthik Nayak
Quoted 9 lines · Karthik Nayak

If I understood Junio's comments correctly, he was drawing out the point about if we even need the separation of options for refs. Since the only option we're adding is a verbose:
...
Your approach seems to take a different path though, where we create a new route of creating two new structs, one for refs and another for objects and adding both to fsck_objects. If we're doing this, wouldn't it be better to use the enum+union idea, like Junio mentioned? That way we would have clarity around which type it represents.

Yup. If we are going to over-engineer this, enum+union would be a reasonable way to do so, but we should ask if we need to split (and more importantly, if we know the problem space well enough to make the right split) in the first place. Just like premature optimization is bad, premature factoring and over-modularization is bad.

Thanks.

author
#90 lore raw

v7 cover ref consistency check infra setup

In reply to shejialuo

Hi All:

We have come to an agreement that there is no need to add complexity in "fsck_options". So this version just adds a new field into "fsck_option" for refs. The other parts are just the same as the previous version.

CI: https://github.com/shejialuo/git/pull/7

This patch is made upon the latest commit:

06e570c0df (Sync with 'maint', 2024-07-02)

Quoted 2 lines

Just like premature optimization is bad, premature factoring and over-modularization is bad.

I have learned a lot from yesterday's email. I always think we should modularize the code which made a lot of trouble in this series. However, the range-diff below may contain some commits from others. (Because this patch is built upon the latest, sorry for the inconvenience)

Thanks for every reviewer.

shejialuo (9):
fsck: rename "skiplist" to "oid_skiplist"
fsck: add a unified interface for reporting fsck messages
fsck: add refs-related options and error report function
refs: set up ref consistency check infrastructure builtin/refs: add verify subcommand builtin/fsck: add git-refs verify child process
files-backend: add unified interface for refs scanning
fsck: add ref name check for files backend
fsck: add ref content check for files backend

 Documentation/fsck-msgids.txt |  12 ++
 Documentation/git-refs.txt    |  13 ++
 builtin/fsck.c                |  32 ++++-
 builtin/mktag.c               |   1 +
 builtin/refs.c                |  44 ++++++
 fsck.c                        | 107 +++++++++++---
 fsck.h                        |  63 ++++++---
 object-file.c                 |  11 +-
 refs.c                        |   7 +-
 refs.h                        |   8 ++
 refs/debug.c                  |  11 ++
 refs/files-backend.c          | 255 +++++++++++++++++++++++++++++++++-
 refs/packed-backend.c         |   8 ++
 refs/refs-internal.h          |  11 +-
 refs/reftable-backend.c       |   8 ++
 t/t0602-reffiles-fsck.sh      | 211 ++++++++++++++++++++++++++++
 16 files changed, 745 insertions(+), 57 deletions(-)
 create mode 100755 t/t0602-reffiles-fsck.sh

Range-diff against v6:

 1:  d32ae41a4d <  -:  ---------- fsck: add "fsck_objects_options" to hold objects-related options
 2:  589a7a6fac <  -:  ---------- fsck: rename "skiplist" to "oid_skiplist"
 3:  bab97d7c82 <  -:  ---------- fsck: add "fsck_refs_options" into "fsck_options"
 -:  ---------- >  1:  f5fcf36132 fsck: rename "skiplist" to "oid_skiplist"
 4:  276da52a6b !  2:  c81b5b6fba fsck: add a unified interface for reporting fsck messages

@@ builtin/fsck.c: static int objerror(struct object *obj, const char *err)
case FSCK_WARN:
@@ builtin/fsck.c: int cmd_fsck(int argc, const char **argv, const char *prefix)

- fsck_walk_options.objects_options.walk = mark_object;
- fsck_obj_options.objects_options.walk = mark_used;
+ fsck_walk_options.walk = mark_object;
+ fsck_obj_options.walk = mark_used;
- fsck_obj_options.error_func = fsck_error_func;
+ fsck_obj_options.error_func = fsck_objects_error_func;
if (check_strict)
@@ builtin/fsck.c: int cmd_fsck(int argc, const char **argv, const char *prefix)

## builtin/mktag.c ##
-@@ builtin/mktag.c: static struct fsck_options fsck_options = FSCK_OBJECTS_OPTIONS_STRICT; +@@ builtin/mktag.c: static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
static int mktag_fsck_error_func(struct fsck_options *o UNUSED,
const struct object_id *oid UNUSED,
enum object_type object_type UNUSED,
@@ fsck.c: static int report(struct fsck_options *options,
+
void fsck_enable_object_names(struct fsck_options *options)
{
- if (!options->objects_options.object_names)
+ if (!options->object_names)
@@ fsck.c: int fsck_buffer(const struct object_id *oid, enum object_type type,
type);
}
@@ fsck.h: int is_valid_msg_type(const char *msg_id, const char *msg_type);
+ enum fsck_msg_id msg_id,
+ const char *message);

- struct fsck_refs_options {
- unsigned verbose:1;
+ struct fsck_options {
+ fsck_walk_func walk;
@@ fsck.h: struct fsck_options {
+ };

- #define FSCK_OBJECTS_OPTIONS_DEFAULT { \
- .oid_skiplist = OIDSET_INIT, \
-- .error_func = fsck_error_function, \
-+ .error_func = fsck_objects_error_function, \
- .objects_options = { \
- .gitmodules_found = OIDSET_INIT, \
- .gitmodules_done = OIDSET_INIT, \
-@@ fsck.h: struct fsck_options {
+ #define FSCK_OPTIONS_DEFAULT { \
+- .skiplist = OIDSET_INIT, \
++ .oid_skiplist = OIDSET_INIT, \
+ .gitmodules_found = OIDSET_INIT, \
+ .gitmodules_done = OIDSET_INIT, \
+ .gitattributes_found = OIDSET_INIT, \
+ .gitattributes_done = OIDSET_INIT, \
+- .error_func = fsck_error_function \
++ .error_func = fsck_objects_error_function \
}
- #define FSCK_OBJECTS_OPTIONS_STRICT { \
+ #define FSCK_OPTIONS_STRICT { \
.strict = 1, \
+@@ fsck.h: struct fsck_options {
+ .gitmodules_done = OIDSET_INIT, \
+ .gitattributes_found = OIDSET_INIT, \
+ .gitattributes_done = OIDSET_INIT, \
- .error_func = fsck_error_function, \
+ .error_func = fsck_objects_error_function, \
- .objects_options = { \
- .gitmodules_found = OIDSET_INIT, \
- .gitmodules_done = OIDSET_INIT, \
-@@ fsck.h: struct fsck_options {
}
- #define FSCK_OBJECTS_OPTIONS_MISSING_GITMODULES { \
+ #define FSCK_OPTIONS_MISSING_GITMODULES { \
.strict = 1, \
+@@ fsck.h: struct fsck_options {
+ .gitmodules_done = OIDSET_INIT, \
+ .gitattributes_found = OIDSET_INIT, \
+ .gitattributes_done = OIDSET_INIT, \
- .error_func = fsck_error_cb_print_missing_gitmodules, \
+ .error_func = fsck_objects_error_cb_print_missing_gitmodules, \
- .objects_options = { \
- .gitmodules_found = OIDSET_INIT, \
- .gitmodules_done = OIDSET_INIT, \
+ }
+
+ /* descend in all linked child objects
@@ fsck.h: int fsck_tag_standalone(const struct object_id *oid, const char *buffer,
*/
int fsck_finish(struct fsck_options *options);
5: e93940c50c ! 3: 53156dc847 fsck: add "fsck_refs_options" initialization macros
@@ Metadata
Author: shejialuo <[email protected]>

## Commit message ##
- fsck: add "fsck_refs_options" initialization macros
+ fsck: add refs-related options and error report function

- Add "FSCK_REFS_OPTIONS_DEFAULT" and "FSCK_REFS_OPTIONS_STRICT" macros to
- create the refs options easily. Add refs-specific "error_func" callback
- "fsck_refs_error_function".
+ Add refs-related options to the "fsck_options", create refs-specific
+ "error_func" callback "fsck_refs_error_function".

"fsck_refs_error_function" will use the "oid" parameter. When the caller
- passed the oid, it will use "oid_to_hex" to get the corresponding hex
- value to report to the user.
+ passes the oid, it will use "oid_to_hex" to get the corresponding hex
+ value to report to the caller.
+
+ Last, add "FSCK_REFS_OPTIONS_DEFAULT" and "FSCK_REFS_OPTIONS_STRICT"
+ macros to create refs options easily.

@@ fsck.h: int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *
+ enum fsck_msg_id msg_id,
+ const char *message);

- struct fsck_refs_options {
- unsigned verbose:1;
+ struct fsck_options {
+ fsck_walk_func walk;
+ fsck_error error_func;
+ unsigned strict:1;
++ unsigned verbose_refs:1;
+ enum fsck_msg_type *msg_type;
+ struct oidset oid_skiplist;
+ struct oidset gitmodules_found;
@@ fsck.h: struct fsck_options {
- .gitattributes_done = OIDSET_INIT, \
- } \
+ .gitattributes_done = OIDSET_INIT, \
+ .error_func = fsck_objects_error_cb_print_missing_gitmodules, \
}
+#define FSCK_REFS_OPTIONS_DEFAULT { \
+ .error_func = fsck_refs_error_function, \
6: 85aa953f6d ! 4: 358f4a1be9 refs: set up ref consistency check infrastructure

    @@ refs.c: int check_refname_format(const char *refname, int flags)
     
      ## refs.h ##
     @@
    - 
      #include "commit.h"
    + #include "repository.h"
      
     +struct fsck_options;
      struct object_id;
      struct ref_store;
    - struct repository;
    + struct strbuf;
     @@ refs.h: int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat
       */
      int check_refname_format(const char *refname, int flags);
 7:  ce7adc7372 !  5:  e632859df6 builtin/refs: add verify subcommand
    @@ builtin/refs.c: static int cmd_refs_migrate(int argc, const char **argv, const c
     +		usage(_("too many arguments"));
     +
     +	if (verbose)
    -+		fsck_refs_options.refs_options.verbose = 1;
    ++		fsck_refs_options.verbose_refs = 1;
     +	if (strict)
     +		fsck_refs_options.strict = 1;
     +
 8:  035eafe10b =  6:  5d7a6261ae builtin/fsck: add `git-refs verify` child process
 9:  9398bf3f0d !  7:  33311af1b1 files-backend: add unified interface for refs scanning
    @@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
     +			continue;
     +		} else if (S_ISREG(iter->st.st_mode) ||
     +			   S_ISLNK(iter->st.st_mode)) {
    -+			if (o->refs_options.verbose)
    ++			if (o->verbose_refs)
     +				fprintf_ln(stderr, "Checking %s/%s",
     +					   refs_check_dir, iter->relative_path);
     +			for (size_t i = 0; fsck_refs_fns[i]; i++) {
    @@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
     +		NULL
     +	};
     +
    -+	if (o->refs_options.verbose)
    ++	if (o->verbose_refs)
     +		fprintf_ln(stderr, "Checking references consistency");
     +
     +	ret = files_fsck_refs_dir(ref_store, o, "refs", fsck_refs_fns);
10:  046773e35c =  8:  02bd3ac952 fsck: add ref name check for files backend
11:  c9b9599589 !  9:  94d123471d fsck: add ref content check for files backend
    @@ refs.c: static int refs_read_special_head(struct ref_store *ref_store,
     
      ## refs/files-backend.c ##
     @@
    + #define USE_THE_REPOSITORY_VARIABLE
    + 
      #include "../git-compat-util.h"
     +#include "../abspath.h"
      #include "../copy.h"
Signature · 1 line
2.45.2
author
#91 lore raw

PATCH v7 1/9 fsck: rename "skiplist" to "oid_skiplist"

Because we introduce ref consistency check. The original "skiplist" is a common option which is set up during handling user configs. To avoid causing ambiguity, rename "skiplist" to "oid_skiplist".

---
 fsck.c | 4 ++--
 fsck.h | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)
fsck.c +2 -2
index eea7145470..1960bfeba9 100644
@@ -205,7 +205,7 @@ void fsck_set_msg_types(struct fsck_options *options, const char *values)
 		if (!strcmp(buf, "skiplist")) {
 			if (equal == len)
 				die("skiplist requires a path");
-			oidset_parse_file(&options->skiplist, buf + equal + 1,
+			oidset_parse_file(&options->oid_skiplist, buf + equal + 1,
 					  the_repository->hash_algo);
 			buf += len + 1;
 			continue;
@@ -223,7 +223,7 @@ void fsck_set_msg_types(struct fsck_options *options, const char *values)
 static int object_on_skiplist(struct fsck_options *opts,
 			      const struct object_id *oid)
 {
-	return opts && oid && oidset_contains(&opts->skiplist, oid);
+	return opts && oid && oidset_contains(&opts->oid_skiplist, oid);
 }
 
 __attribute__((format (printf, 5, 6)))
fsck.h +1 -1
index 6085a384f6..1ee3dd85ba 100644
@@ -136,7 +136,7 @@ struct fsck_options {
 	fsck_error error_func;
 	unsigned strict:1;
 	enum fsck_msg_type *msg_type;
-	struct oidset skiplist;
+	struct oidset oid_skiplist;
 	struct oidset gitmodules_found;
 	struct oidset gitmodules_done;
 	struct oidset gitattributes_found;
Signature · 1 line
2.45.2
#92 lore raw

Re: [GSoC][PATCH v7 1/9] fsck: rename "skiplist" to "oid_skiplist"

Quoted 3 lines · shejialuo

Because we introduce ref consistency check. The original "skiplist" is a common option which is set up during handling user configs. To avoid causing ambiguity, rename "skiplist" to "oid_skiplist".

I think the commit message could be expanded on to provide additional context and reasoning for the change. From reading this alone, it sounds like we have already introduced the ref consistency check and are now looking to rename a variable. When really this is a preparatory change. Without reading ahead, I'm also left wondering why this name change reduces ambiguity.

-Justin

author
#93 lore raw

Re: [GSoC][PATCH v7 1/9] fsck: rename "skiplist" to "oid_skiplist"

Quoted 5 lines · Justin Tobler
Quoted 3 lines · shejialuo

Because we introduce ref consistency check. The original "skiplist" is a common option which is set up during handling user configs. To avoid causing ambiguity, rename "skiplist" to "oid_skiplist".

Quoted 7 lines

I think the commit message could be expanded on to provide additional context and reasoning for the change. From reading this alone, it sounds like we have already introduced the ref consistency check and are now looking to rename a variable. When really this is a preparatory change. Without reading ahead, I'm also left wondering why this name change reduces ambiguity.

Yes, I will add more information to show the context. "skiplist" is initialized using "git_fsck_config" to parse the user-specific config. In this series, we introduce ref-specific check. "skiplist" is a general name which may make the caller think "skiplist" is related to both the refs and objects.

For later implementation, we may also introduce "skiplist" for refs, but for refs, we concern about the name not the "oidset". I will add more information to explain how this change reduces ambiguity.

Thanks,
Jialuo

author
#94 lore raw

PATCH v7 2/9 fsck: add a unified interface for reporting fsck messages

In reply to shejialuo

The static function "report" provided by "fsck.c" aims at reporting the problems related to object database which cannot be reused for refs. In order to provide a unified interface which can report either objects or refs, create a new function "vfsck_report" by adding "checked_ref_name" parameter following the "report" prototype. However, instead of using "...", provide "va_list" to allow more flexibility.

The "vfsck_report" function will use "error_func" registered in "fsck_options" function to report customized messages. Change "error_func" prototype to align with the "vfsck_report".

Change "report" function to make it use "vfsck_report" to report objects-related messages. Add a new function called "fsck_refs_report" to use "vfsck_report" to report refs-related messages.

---
 builtin/fsck.c  | 15 ++++-----
 builtin/mktag.c |  1 +
 fsck.c          | 81 ++++++++++++++++++++++++++++++++++++-------------
 fsck.h          | 42 ++++++++++++++++---------
 object-file.c   | 11 ++++---
 5 files changed, 102 insertions(+), 48 deletions(-)
builtin/fsck.c +8 -7
index d13a226c2e..de34538c4f 100644
@@ -89,12 +89,13 @@ static int objerror(struct object *obj, const char *err)
 	return -1;
 }
 
-static int fsck_error_func(struct fsck_options *o UNUSED,
-			   const struct object_id *oid,
-			   enum object_type object_type,
-			   enum fsck_msg_type msg_type,
-			   enum fsck_msg_id msg_id UNUSED,
-			   const char *message)
+static int fsck_objects_error_func(struct fsck_options *o UNUSED,
+				   const struct object_id *oid,
+				   enum object_type object_type,
+				   const char *checked_ref_name UNUSED,
+				   enum fsck_msg_type msg_type,
+				   enum fsck_msg_id msg_id UNUSED,
+				   const char *message)
 {
 	switch (msg_type) {
 	case FSCK_WARN:
@@ -938,7 +939,7 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
 
 	fsck_walk_options.walk = mark_object;
 	fsck_obj_options.walk = mark_used;
-	fsck_obj_options.error_func = fsck_error_func;
+	fsck_obj_options.error_func = fsck_objects_error_func;
 	if (check_strict)
 		fsck_obj_options.strict = 1;
 
builtin/mktag.c +1 -0
index 4767f1a97e..42f945c584 100644
@@ -20,6 +20,7 @@ static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
 static int mktag_fsck_error_func(struct fsck_options *o UNUSED,
 				 const struct object_id *oid UNUSED,
 				 enum object_type object_type UNUSED,
+				 const char *checked_ref_name UNUSED,
 				 enum fsck_msg_type msg_type,
 				 enum fsck_msg_id msg_id UNUSED,
 				 const char *message)
fsck.c +60 -21

115 lines not on this page · raw message

fsck.h +27 -15

81 lines not on this page · raw message

object-file.c +6 -5
index 065103be3e..d2c6427935 100644
@@ -2470,11 +2470,12 @@ int repo_has_object_file(struct repository *r,
  * give more context.
  */
 static int hash_format_check_report(struct fsck_options *opts UNUSED,
-				     const struct object_id *oid UNUSED,
-				     enum object_type object_type UNUSED,
-				     enum fsck_msg_type msg_type UNUSED,
-				     enum fsck_msg_id msg_id UNUSED,
-				     const char *message)
+				    const struct object_id *oid UNUSED,
+				    enum object_type object_type UNUSED,
+				    const char *ref_checked_name UNUSED,
+				    enum fsck_msg_type msg_type UNUSED,
+				    enum fsck_msg_id msg_id UNUSED,
+				    const char *message)
 {
 	error(_("object fails fsck: %s"), message);
 	return 1;
Signature · 1 line
2.45.2
#95 lore raw

Re: [GSoC][PATCH v7 2/9] fsck: add a unified interface for reporting fsck messages

Quoted 7 lines · shejialuo

The static function "report" provided by "fsck.c" aims at reporting the problems related to object database which cannot be reused for refs. In order to provide a unified interface which can report either objects or refs, create a new function "vfsck_report" by adding "checked_ref_name" parameter following the "report" prototype. However, instead of using "...", provide "va_list" to allow more flexibility.

Quoted 7 lines

The "vfsck_report" function will use "error_func" registered in "fsck_options" function to report customized messages. Change "error_func" prototype to align with the "vfsck_report".

Change "report" function to make it use "vfsck_report" to report objects-related messages. Add a new function called "fsck_refs_report" to use "vfsck_report" to report refs-related messages.

To restate in my own words, the existing "report" function is checked if an fsck message should be returned and also formats the message. It is currently setup to only handle object database related fsck problems. The fsck_error function type is generalized to also accept a reference name enabling fsck problems to be reported for references.

Quoted 107 lines
---
 builtin/fsck.c  | 15 ++++-----
 builtin/mktag.c |  1 +
 fsck.c          | 81 ++++++++++++++++++++++++++++++++++++-------------
 fsck.h          | 42 ++++++++++++++++---------
 object-file.c   | 11 ++++---
 5 files changed, 102 insertions(+), 48 deletions(-)
builtin/fsck.c +8 -7
index d13a226c2e..de34538c4f 100644
@@ -89,12 +89,13 @@ static int objerror(struct object *obj, const char *err)
 	return -1;
 }
 
-static int fsck_error_func(struct fsck_options *o UNUSED,
-			   const struct object_id *oid,
-			   enum object_type object_type,
-			   enum fsck_msg_type msg_type,
-			   enum fsck_msg_id msg_id UNUSED,
-			   const char *message)
+static int fsck_objects_error_func(struct fsck_options *o UNUSED,
+				   const struct object_id *oid,
+				   enum object_type object_type,
+				   const char *checked_ref_name UNUSED,
+				   enum fsck_msg_type msg_type,
+				   enum fsck_msg_id msg_id UNUSED,
+				   const char *message)
 {
 	switch (msg_type) {
 	case FSCK_WARN:
@@ -938,7 +939,7 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
 
 	fsck_walk_options.walk = mark_object;
 	fsck_obj_options.walk = mark_used;
-	fsck_obj_options.error_func = fsck_error_func;
+	fsck_obj_options.error_func = fsck_objects_error_func;
 	if (check_strict)
 		fsck_obj_options.strict = 1;
 
builtin/mktag.c +1 -0
index 4767f1a97e..42f945c584 100644
@@ -20,6 +20,7 @@ static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
 static int mktag_fsck_error_func(struct fsck_options *o UNUSED,
 				 const struct object_id *oid UNUSED,
 				 enum object_type object_type UNUSED,
+				 const char *checked_ref_name UNUSED,
 				 enum fsck_msg_type msg_type,
 				 enum fsck_msg_id msg_id UNUSED,
 				 const char *message)
fsck.c +19 -8
index 1960bfeba9..7182ce8e80 100644
@@ -226,12 +226,18 @@ static int object_on_skiplist(struct fsck_options *opts,
 	return opts && oid && oidset_contains(&opts->oid_skiplist, oid);
 }
 
-__attribute__((format (printf, 5, 6)))
-static int report(struct fsck_options *options,
-		  const struct object_id *oid, enum object_type object_type,
-		  enum fsck_msg_id msg_id, const char *fmt, ...)
+/*
+ * Provide a unified interface for either fscking refs or objects.
+ * It will get the current msg error type and call the error_func callback
+ * which is registered in the "fsck_options" struct.
+ */
+static int vfsck_report(struct fsck_options *options,
+			const struct object_id *oid,
+			enum object_type object_type,
+			const char *checked_ref_name,
+			enum fsck_msg_id msg_id, const char *fmt, va_list ap)
 {
-	va_list ap;
+	va_list ap_copy;
 	struct strbuf sb = STRBUF_INIT;
 	enum fsck_msg_type msg_type = fsck_msg_type(msg_id, options);
 	int result;
@@ -250,9 +256,9 @@ static int report(struct fsck_options *options,
 	prepare_msg_ids();
 	strbuf_addf(&sb, "%s: ", msg_id_info[msg_id].camelcased);
 
-	va_start(ap, fmt);
-	strbuf_vaddf(&sb, fmt, ap);
-	result = options->error_func(options, oid, object_type,
+	va_copy(ap_copy, ap);
+	strbuf_vaddf(&sb, fmt, ap_copy);
+	result = options->error_func(options, oid, object_type, checked_ref_name,
 				     msg_type, msg_id, sb.buf);
 	strbuf_release(&sb);
 	va_end(ap);
@@ -260,6 +266,36 @@ static int report(struct fsck_options *options,
 	return result;
 }
 
+__attribute__((format (printf, 5, 6)))
+static int report(struct fsck_options *options,
+		  const struct object_id *oid, enum object_type object_type,
+		  enum fsck_msg_id msg_id, const char *fmt, ...)
+{

fsck.c static int report(struct fsck_options *options, · line 283

Quoted 10 lines
fsck.c +10 -0
+	va_list ap;
+	int result;
+	va_start(ap, fmt);
+	result = vfsck_report(options, oid, object_type, NULL,
+			      msg_id, fmt, ap);
+	va_end(ap);
+	return result;
+}
+
+

Looks like there are some extra new lines here.

fsck.c static int report(struct fsck_options *options, · line 294

Quoted 11 lines
fsck.c +11 -0
+
+int fsck_refs_report(struct fsck_options *options,
+		     const struct object_id *oid,
+		     const char *checked_ref_name,
+		     enum fsck_msg_id msg_id, const char *fmt, ...)
+{
+	va_list ap;
+	int result;
+	va_start(ap, fmt);
+	result = vfsck_report(options, oid, OBJ_NONE,
+			      checked_ref_name, msg_id, fmt, ap);

Do we expect that reference related fsck problems may also be skipped via the oid_skiplist?

Quoted 97 lines
fsck.c +20 -13
+	va_end(ap);
+	return result;
+}
+
 void fsck_enable_object_names(struct fsck_options *options)
 {
 	if (!options->object_names)
@@ -1200,12 +1236,13 @@ int fsck_buffer(const struct object_id *oid, enum object_type type,
 		      type);
 }
 
-int fsck_error_function(struct fsck_options *o,
-			const struct object_id *oid,
-			enum object_type object_type UNUSED,
-			enum fsck_msg_type msg_type,
-			enum fsck_msg_id msg_id UNUSED,
-			const char *message)
+int fsck_objects_error_function(struct fsck_options *o,
+				const struct object_id *oid,
+				enum object_type object_type UNUSED,
+				const char *checked_ref_name UNUSED,
+				enum fsck_msg_type msg_type,
+				enum fsck_msg_id msg_id UNUSED,
+				const char *message)
 {
 	if (msg_type == FSCK_WARN) {
 		warning("object %s: %s", fsck_describe_object(o, oid), message);
@@ -1303,16 +1340,18 @@ int git_fsck_config(const char *var, const char *value,
  * Custom error callbacks that are used in more than one place.
  */
 
-int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o,
-					   const struct object_id *oid,
-					   enum object_type object_type,
-					   enum fsck_msg_type msg_type,
-					   enum fsck_msg_id msg_id,
-					   const char *message)
+int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o,
+						   const struct object_id *oid,
+						   enum object_type object_type,
+						   const char *checked_ref_name,
+						   enum fsck_msg_type msg_type,
+						   enum fsck_msg_id msg_id,
+						   const char *message)
 {
 	if (msg_id == FSCK_MSG_GITMODULES_MISSING) {
 		puts(oid_to_hex(oid));
 		return 0;
 	}
-	return fsck_error_function(o, oid, object_type, msg_type, msg_id, message);
+	return fsck_objects_error_function(o, oid, object_type, checked_ref_name,
+					   msg_type, msg_id, message);
 }
fsck.h +16 -11
index 1ee3dd85ba..f703dfb5e8 100644
@@ -114,22 +114,27 @@ int is_valid_msg_type(const char *msg_id, const char *msg_type);
 typedef int (*fsck_walk_func)(struct object *obj, enum object_type object_type,
 			      void *data, struct fsck_options *options);
 
-/* callback for fsck_object, type is FSCK_ERROR or FSCK_WARN */
+/*
+ * callback function for reporting errors when checking either objects or refs
+ */
 typedef int (*fsck_error)(struct fsck_options *o,
 			  const struct object_id *oid, enum object_type object_type,
+			  const char *checked_ref_name,
 			  enum fsck_msg_type msg_type, enum fsck_msg_id msg_id,
 			  const char *message);
 
-int fsck_error_function(struct fsck_options *o,
-			const struct object_id *oid, enum object_type object_type,
-			enum fsck_msg_type msg_type, enum fsck_msg_id msg_id,
-			const char *message);
-int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o,
-					   const struct object_id *oid,
-					   enum object_type object_type,
-					   enum fsck_msg_type msg_type,
-					   enum fsck_msg_id msg_id,
-					   const char *message);
+int fsck_objects_error_function(struct fsck_options *o,
+				const struct object_id *oid, enum object_type object_type,
+				const char *checked_ref_name,
+				enum fsck_msg_type msg_type, enum fsck_msg_id msg_id,
+				const char *message);
+int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o,
+						   const struct object_id *oid,
+						   enum object_type object_type,
+						   const char *checked_ref_name,
+						   enum fsck_msg_type msg_type,
+						   enum fsck_msg_id msg_id,
+						   const char *message);
 
 struct fsck_options {
 	fsck_walk_func walk;

fsck.h struct fsck_options { · line 153

Quoted 5 lines
fsck.h +1 -1
@@ -145,12 +150,12 @@ struct fsck_options {
 };
 
 #define FSCK_OPTIONS_DEFAULT { \
-	.skiplist = OIDSET_INIT, \
+	.oid_skiplist = OIDSET_INIT, \

Since we renamed skiplist -> oid_skiplist in the previous patch, shouldn't we update this there?

Quoted 28 lines
fsck.h +3 -3
 	.gitmodules_found = OIDSET_INIT, \
 	.gitmodules_done = OIDSET_INIT, \
 	.gitattributes_found = OIDSET_INIT, \
 	.gitattributes_done = OIDSET_INIT, \
-	.error_func = fsck_error_function \
+	.error_func = fsck_objects_error_function \
 }
 #define FSCK_OPTIONS_STRICT { \
 	.strict = 1, \
@@ -158,7 +163,7 @@ struct fsck_options {
 	.gitmodules_done = OIDSET_INIT, \
 	.gitattributes_found = OIDSET_INIT, \
 	.gitattributes_done = OIDSET_INIT, \
-	.error_func = fsck_error_function, \
+	.error_func = fsck_objects_error_function, \
 }
 #define FSCK_OPTIONS_MISSING_GITMODULES { \
 	.strict = 1, \
@@ -166,7 +171,7 @@ struct fsck_options {
 	.gitmodules_done = OIDSET_INIT, \
 	.gitattributes_found = OIDSET_INIT, \
 	.gitattributes_done = OIDSET_INIT, \
-	.error_func = fsck_error_cb_print_missing_gitmodules, \
+	.error_func = fsck_objects_error_cb_print_missing_gitmodules, \
 }
 
 /* descend in all linked child objects

fsck.h int fsck_tag_standalone(const struct object_id *oid, const char *buffer, · line 216

Quoted 3 lines
fsck.h +0 -0
@@ -209,6 +214,13 @@ int fsck_tag_standalone(const struct object_id *oid, const char *buffer,
  */
 int fsck_finish(struct fsck_options *options);
 

In my opinion it would be nice to add a comment to document fsck_refs_report() here.

Quoted 36 lines
fsck.h +7 -0
+__attribute__((format (printf, 5, 6)))
+int fsck_refs_report(struct fsck_options *options,
+		     const struct object_id *oid,
+		     const char *checked_ref_name,
+		     enum fsck_msg_id msg_id,
+		     const char *fmt, ...);
+
 /*
  * Subsystem for storing human-readable names for each object.
  *
object-file.c +6 -5
index 065103be3e..d2c6427935 100644
@@ -2470,11 +2470,12 @@ int repo_has_object_file(struct repository *r,
  * give more context.
  */
 static int hash_format_check_report(struct fsck_options *opts UNUSED,
-				     const struct object_id *oid UNUSED,
-				     enum object_type object_type UNUSED,
-				     enum fsck_msg_type msg_type UNUSED,
-				     enum fsck_msg_id msg_id UNUSED,
-				     const char *message)
+				    const struct object_id *oid UNUSED,
+				    enum object_type object_type UNUSED,
+				    const char *ref_checked_name UNUSED,
+				    enum fsck_msg_type msg_type UNUSED,
+				    enum fsck_msg_id msg_id UNUSED,
+				    const char *message)
 {
 	error(_("object fails fsck: %s"), message);
 	return 1;
Signature · 1 line
2.45.2
author
#96 lore raw

Re: [GSoC][PATCH v7 2/9] fsck: add a unified interface for reporting fsck messages

Quoted 16 lines · Justin Tobler
Quoted 7 lines · shejialuo

The static function "report" provided by "fsck.c" aims at reporting the problems related to object database which cannot be reused for refs. In order to provide a unified interface which can report either objects or refs, create a new function "vfsck_report" by adding "checked_ref_name" parameter following the "report" prototype. However, instead of using "...", provide "va_list" to allow more flexibility.

Quoted 7 lines

The "vfsck_report" function will use "error_func" registered in "fsck_options" function to report customized messages. Change "error_func" prototype to align with the "vfsck_report".

Change "report" function to make it use "vfsck_report" to report objects-related messages. Add a new function called "fsck_refs_report" to use "vfsck_report" to report refs-related messages.

Quoted 6 lines

To restate in my own words, the existing "report" function is checked if an fsck message should be returned and also formats the message. It is currently setup to only handle object database related fsck problems. The fsck_error function type is generalized to also accept a reference name enabling fsck problems to be reported for references.

Exactly, I will add more information in the commit message.

Thanks,
Jialuo

author
#97 lore raw

PATCH v7 3/9 fsck: add refs-related options and error report function

In reply to shejialuo

Add refs-related options to the "fsck_options", create refs-specific "error_func" callback "fsck_refs_error_function".

"fsck_refs_error_function" will use the "oid" parameter. When the caller passes the oid, it will use "oid_to_hex" to get the corresponding hex value to report to the caller.

Last, add "FSCK_REFS_OPTIONS_DEFAULT" and "FSCK_REFS_OPTIONS_STRICT" macros to create refs options easily.

---
 fsck.c | 22 ++++++++++++++++++++++
 fsck.h | 15 +++++++++++++++
 2 files changed, 37 insertions(+)
fsck.c +22 -0
index 7182ce8e80..d1dcbdcac2 100644
@@ -1252,6 +1252,28 @@ int fsck_objects_error_function(struct fsck_options *o,
 	return 1;
 }
 
+int fsck_refs_error_function(struct fsck_options *options UNUSED,
+			     const struct object_id *oid,
+			     enum object_type object_type UNUSED,
+			     const char *checked_ref_name,
+			     enum fsck_msg_type msg_type,
+			     enum fsck_msg_id msg_id UNUSED,
+			     const char *message)
+{
+	static struct strbuf sb = STRBUF_INIT;
+
+	strbuf_addstr(&sb, checked_ref_name);
+	if (oid)
+		strbuf_addf(&sb, " -> (%s)", oid_to_hex(oid));
+
+	if (msg_type == FSCK_WARN) {
+		warning("%s: %s", sb.buf, message);
+		return 0;
+	}
+	error("%s: %s", sb.buf, message);
+	return 1;
+}
+
 static int fsck_blobs(struct oidset *blobs_found, struct oidset *blobs_done,
 		      enum fsck_msg_id msg_missing, enum fsck_msg_id msg_type,
 		      struct fsck_options *options, const char *blob_type)
fsck.h +15 -0
index f703dfb5e8..246055c0f9 100644
@@ -135,11 +135,19 @@ int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o,
 						   enum fsck_msg_type msg_type,
 						   enum fsck_msg_id msg_id,
 						   const char *message);
+int fsck_refs_error_function(struct fsck_options *options,
+			     const struct object_id *oid,
+			     enum object_type object_type,
+			     const char *checked_ref_name,
+			     enum fsck_msg_type msg_type,
+			     enum fsck_msg_id msg_id,
+			     const char *message);
 
 struct fsck_options {
 	fsck_walk_func walk;
 	fsck_error error_func;
 	unsigned strict:1;
+	unsigned verbose_refs:1;
 	enum fsck_msg_type *msg_type;
 	struct oidset oid_skiplist;
 	struct oidset gitmodules_found;
@@ -173,6 +181,13 @@ struct fsck_options {
 	.gitattributes_done = OIDSET_INIT, \
 	.error_func = fsck_objects_error_cb_print_missing_gitmodules, \
 }
+#define FSCK_REFS_OPTIONS_DEFAULT { \
+	.error_func = fsck_refs_error_function, \
+}
+#define FSCK_REFS_OPTIONS_STRICT { \
+	.strict = 1, \
+	.error_func = fsck_refs_error_function, \
+}
 
 /* descend in all linked child objects
  * the return value is:
Signature · 1 line
2.45.2
author
#98 lore raw

PATCH v7 4/9 refs: set up ref consistency check infrastructure

In reply to shejialuo

The interfaces defined in the ref_storage_be are carefully structured in semantic. It's organized as the five parts:

1. The name and the initialization interfaces.
2. The ref transaction interfaces.
3. The ref internal interfaces (pack, rename and copy).
4. The ref filesystem interfaces.
5. The reflog related interfaces.

To keep consistent with the git-fsck(1), add a new interface named "fsck_refs_fn" to the end of "ref_storage_be". This semantic cannot be grouped into any above five categories. Explicitly add blank line to make it different from others.

---
 refs.c                  |  5 +++++
 refs.h                  |  8 ++++++++
 refs/debug.c            | 11 +++++++++++
 refs/files-backend.c    | 15 ++++++++++++++-
 refs/packed-backend.c   |  8 ++++++++
 refs/refs-internal.h    |  6 ++++++
 refs/reftable-backend.c |  8 ++++++++
 7 files changed, 60 insertions(+), 1 deletion(-)
refs.c +5 -0
index bb90a18875..410919246b 100644
@@ -318,6 +318,11 @@ int check_refname_format(const char *refname, int flags)
 	return check_or_sanitize_refname(refname, flags, NULL);
 }
 
+int refs_fsck(struct ref_store *refs, struct fsck_options *o)
+{
+	return refs->be->fsck(refs, o);
+}
+
 void sanitize_refname_component(const char *refname, struct strbuf *out)
 {
 	if (check_or_sanitize_refname(refname, REFNAME_ALLOW_ONELEVEL, out))
refs.h +8 -0
index 0ecba21b4a..804d6a7fce 100644
@@ -4,6 +4,7 @@
 #include "commit.h"
 #include "repository.h"
 
+struct fsck_options;
 struct object_id;
 struct ref_store;
 struct strbuf;
@@ -541,6 +542,13 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat
  */
 int check_refname_format(const char *refname, int flags);
 
+/*
+ * Check the reference database for consistency. Return 0 if refs and
+ * reflogs are consistent, and non-zero otherwise. The errors will be
+ * written to stderr.
+ */
+int refs_fsck(struct ref_store *refs, struct fsck_options *o);
+
 /*
  * Apply the rules from check_refname_format, but mutate the result until it
  * is acceptable, and place the result in "out".
refs/debug.c +11 -0
index 547d9245b9..45e2e784a0 100644
@@ -419,6 +419,15 @@ static int debug_reflog_expire(struct ref_store *ref_store, const char *refname,
 	return res;
 }
 
+static int debug_fsck(struct ref_store *ref_store,
+		      struct fsck_options *o)
+{
+	struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store;
+	int res = drefs->refs->be->fsck(drefs->refs, o);
+	trace_printf_key(&trace_refs, "fsck: %d\n", res);
+	return res;
+}
+
 struct ref_storage_be refs_be_debug = {
 	.name = "debug",
 	.init = NULL,
@@ -451,4 +460,6 @@ struct ref_storage_be refs_be_debug = {
 	.create_reflog = debug_create_reflog,
 	.delete_reflog = debug_delete_reflog,
 	.reflog_expire = debug_reflog_expire,
+
+	.fsck = debug_fsck,
 };
refs/files-backend.c +14 -1
index aa52d9be7c..d89eeda8ef 100644
@@ -3408,6 +3408,17 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,
 	return ret;
 }
 
+static int files_fsck(struct ref_store *ref_store,
+		      struct fsck_options *o)
+{
+	int ret;
+	struct files_ref_store *refs =
+		files_downcast(ref_store, REF_STORE_READ, "fsck");
+
+	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o);
+	return ret;
+}
+
 struct ref_storage_be refs_be_files = {
 	.name = "files",
 	.init = files_ref_store_init,
@@ -3434,5 +3445,7 @@ struct ref_storage_be refs_be_files = {
 	.reflog_exists = files_reflog_exists,
 	.create_reflog = files_create_reflog,
 	.delete_reflog = files_delete_reflog,
-	.reflog_expire = files_reflog_expire
+	.reflog_expire = files_reflog_expire,
+
+	.fsck = files_fsck,
 };
refs/packed-backend.c +8 -0
index a0666407cd..5209b0b212 100644
@@ -1735,6 +1735,12 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_s
 	return empty_ref_iterator_begin();
 }
 
+static int packed_fsck(struct ref_store *ref_store,
+		       struct fsck_options *o)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_packed = {
 	.name = "packed",
 	.init = packed_ref_store_init,
@@ -1762,4 +1768,6 @@ struct ref_storage_be refs_be_packed = {
 	.create_reflog = NULL,
 	.delete_reflog = NULL,
 	.reflog_expire = NULL,
+
+	.fsck = packed_fsck,
 };
refs/refs-internal.h +6 -0
index fa975d69aa..a905e187cd 100644
@@ -4,6 +4,7 @@
 #include "refs.h"
 #include "iterator.h"
 
+struct fsck_options;
 struct ref_transaction;
 
 /*
@@ -650,6 +651,9 @@ typedef int read_raw_ref_fn(struct ref_store *ref_store, const char *refname,
 typedef int read_symbolic_ref_fn(struct ref_store *ref_store, const char *refname,
 				 struct strbuf *referent);
 
+typedef int fsck_fn(struct ref_store *ref_store,
+		    struct fsck_options *o);
+
 struct ref_storage_be {
 	const char *name;
 	ref_store_init_fn *init;
@@ -677,6 +681,8 @@ struct ref_storage_be {
 	create_reflog_fn *create_reflog;
 	delete_reflog_fn *delete_reflog;
 	reflog_expire_fn *reflog_expire;
+
+	fsck_fn *fsck;
 };
 
 extern struct ref_storage_be refs_be_files;
refs/reftable-backend.c +8 -0
index fbe74c239d..b5a1a526df 100644
@@ -2303,6 +2303,12 @@ static int reftable_be_reflog_expire(struct ref_store *ref_store,
 	return ret;
 }
 
+static int reftable_be_fsck(struct ref_store *ref_store,
+			    struct fsck_options *o)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_reftable = {
 	.name = "reftable",
 	.init = reftable_be_init,
@@ -2330,4 +2336,6 @@ struct ref_storage_be refs_be_reftable = {
 	.create_reflog = reftable_be_create_reflog,
 	.delete_reflog = reftable_be_delete_reflog,
 	.reflog_expire = reftable_be_reflog_expire,
+
+	.fsck = reftable_be_fsck,
 };
Signature · 1 line
2.45.2
author
#99 lore raw

PATCH v7 5/9 builtin/refs: add verify subcommand

In reply to shejialuo

Introduce a new subcommand "verify" in git-refs(1) to allow the user to check the reference database consistency.

---
 Documentation/git-refs.txt | 13 +++++++++++
 builtin/refs.c             | 44 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
Documentation/git-refs.txt +13 -0
index 5b99e04385..1244a85b64 100644
@@ -10,6 +10,7 @@ SYNOPSIS
 --------
 [verse]
 'git refs migrate' --ref-format=<format> [--dry-run]
+'git refs verify' [--strict] [--verbose]
 
 DESCRIPTION
 -----------
@@ -22,6 +23,9 @@ COMMANDS
 migrate::
 	Migrate ref store between different formats.
 
+verify::
+	Verify reference database consistency.
+
 OPTIONS
 -------
 
@@ -39,6 +43,15 @@ include::ref-storage-format.txt[]
 	can be used to double check that the migration works as expected before
 	performing the actual migration.
 
+The following options are specific to 'git refs verify':
+
+--strict::
+	Enable more strict checking, every WARN severity for the `Fsck Messages`
+	be seen as ERROR. See linkgit:git-fsck[1].
+
+--verbose::
+	When verifying the reference database consistency, be chatty.
+
 KNOWN LIMITATIONS
 -----------------
 
builtin/refs.c +44 -0

69 lines not on this page · raw message

Signature · 1 line
2.45.2
author
#100 lore raw

PATCH v7 6/9 builtin/fsck: add `git-refs verify` child process

In reply to shejialuo

Introduce a new function "fsck_refs" that initializes and runs a child process to execute the "git-refs verify" command.

---
 builtin/fsck.c | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
builtin/fsck.c +17 -0
index de34538c4f..ec3357722c 100644
@@ -897,6 +897,21 @@ static int check_pack_rev_indexes(struct repository *r, int show_progress)
 	return res;
 }
 
+static void fsck_refs(void)
+{
+	struct child_process refs_verify = CHILD_PROCESS_INIT;
+	child_process_init(&refs_verify);
+	refs_verify.git_cmd = 1;
+	strvec_pushl(&refs_verify.args, "refs", "verify", NULL);
+	if (verbose)
+		strvec_push(&refs_verify.args, "--verbose");
+	if (check_strict)
+		strvec_push(&refs_verify.args, "--strict");
+
+	if (run_command(&refs_verify))
+		errors_found |= ERROR_REFS;
+}
+
 static char const * const fsck_usage[] = {
 	N_("git fsck [--tags] [--root] [--unreachable] [--cache] [--no-reflogs]\n"
 	   "         [--[no-]full] [--strict] [--verbose] [--lost-found]\n"
@@ -1066,6 +1081,8 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
 
 	check_connectivity();
 
+	fsck_refs();
+
 	if (the_repository->settings.core_commit_graph) {
 		struct child_process commit_graph_verify = CHILD_PROCESS_INIT;
 
Signature · 1 line
2.45.2
author
#101 lore raw

PATCH v7 7/9 files-backend: add unified interface for refs scanning

In reply to shejialuo

For refs and reflogs, we need to scan its corresponding directories to check every regular file or symbolic link which shares the same pattern. Introduce a unified interface for scanning directories for files-backend.

---
 refs/files-backend.c | 77 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 76 insertions(+), 1 deletion(-)
refs/files-backend.c +76 -1
index d89eeda8ef..84acb58782 100644
@@ -6,6 +6,7 @@
 #include "../gettext.h"
 #include "../hash.h"
 #include "../hex.h"
+#include "../fsck.h"
 #include "../refs.h"
 #include "refs-internal.h"
 #include "ref-cache.h"
@@ -3408,6 +3409,78 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,
 	return ret;
 }
 
+/*
+ * For refs and reflogs, they share a unified interface when scanning
+ * the whole directory. This function is used as the callback for each
+ * regular file or symlink in the directory.
+ */
+typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
+				  const char *gitdir,
+				  const char *refs_check_dir,
+				  struct dir_iterator *iter);
+
+static int files_fsck_refs_dir(struct ref_store *ref_store,
+			       struct fsck_options *o,
+			       const char *refs_check_dir,
+			       files_fsck_refs_fn *fsck_refs_fns)
+{
+	const char *gitdir = ref_store->gitdir;
+	struct strbuf sb = STRBUF_INIT;
+	struct dir_iterator *iter;
+	int iter_status;
+	int ret = 0;
+
+	strbuf_addf(&sb, "%s/%s", gitdir, refs_check_dir);
+
+	iter = dir_iterator_begin(sb.buf, 0);
+
+	if (!iter) {
+		ret = error_errno("cannot open directory %s", sb.buf);
+		goto out;
+	}
+
+	while ((iter_status = dir_iterator_advance(iter)) == ITER_OK) {
+		if (S_ISDIR(iter->st.st_mode)) {
+			continue;
+		} else if (S_ISREG(iter->st.st_mode) ||
+			   S_ISLNK(iter->st.st_mode)) {
+			if (o->verbose_refs)
+				fprintf_ln(stderr, "Checking %s/%s",
+					   refs_check_dir, iter->relative_path);
+			for (size_t i = 0; fsck_refs_fns[i]; i++) {
+				if (fsck_refs_fns[i](o, gitdir, refs_check_dir, iter))
+					ret = -1;
+			}
+		} else {
+			ret = error(_("unexpected file type for '%s'"),
+				    iter->basename);
+		}
+	}
+
+	if (iter_status != ITER_DONE)
+		ret = error(_("failed to iterate over '%s'"), sb.buf);
+
+out:
+	strbuf_release(&sb);
+	return ret;
+}
+
+static int files_fsck_refs(struct ref_store *ref_store,
+			   struct fsck_options *o)
+{
+	int ret;
+	files_fsck_refs_fn fsck_refs_fns[]= {
+		NULL
+	};
+
+	if (o->verbose_refs)
+		fprintf_ln(stderr, "Checking references consistency");
+
+	ret = files_fsck_refs_dir(ref_store, o, "refs", fsck_refs_fns);
+
+	return ret;
+}
+
 static int files_fsck(struct ref_store *ref_store,
 		      struct fsck_options *o)
 {
@@ -3415,7 +3488,9 @@ static int files_fsck(struct ref_store *ref_store,
 	struct files_ref_store *refs =
 		files_downcast(ref_store, REF_STORE_READ, "fsck");
 
-	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o);
+	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o)
+	    | files_fsck_refs(ref_store, o);
+
 	return ret;
 }
 
Signature · 1 line
2.45.2
author
#102 lore raw

PATCH v7 8/9 fsck: add ref name check for files backend

In reply to shejialuo

The git-fsck(1) only implicitly checks the reference, it does not fully check refs with bad format name such as standalone "@" and name ending with ".lock".

In order to provide such checks, add a new fsck message id "badRefName" with default ERROR type. Use existing "check_refname_format" to explicit check the ref name. And add a new unit test to verify the functionality.

---
 Documentation/fsck-msgids.txt |   3 +
 fsck.h                        |   1 +
 refs/files-backend.c          |  20 +++++++
 t/t0602-reffiles-fsck.sh      | 101 ++++++++++++++++++++++++++++++++++
 4 files changed, 125 insertions(+)
 create mode 100755 t/t0602-reffiles-fsck.sh
Documentation/fsck-msgids.txt +3 -0
index f643585a34..dab4012246 100644
@@ -19,6 +19,9 @@
 `badParentSha1`::
 	(ERROR) A commit object has a bad parent sha1.
 
+`badRefName`::
+	(ERROR) A ref has a bad name.
+
 `badTagName`::
 	(INFO) A tag has an invalid format.
 
fsck.h +1 -0
index 246055c0f9..90457d1a1f 100644
@@ -31,6 +31,7 @@ enum fsck_msg_type {
 	FUNC(BAD_NAME, ERROR) \
 	FUNC(BAD_OBJECT_SHA1, ERROR) \
 	FUNC(BAD_PARENT_SHA1, ERROR) \
+	FUNC(BAD_REF_NAME, ERROR) \
 	FUNC(BAD_TIMEZONE, ERROR) \
 	FUNC(BAD_TREE, ERROR) \
 	FUNC(BAD_TREE_SHA1, ERROR) \
refs/files-backend.c +20 -0
index 84acb58782..69a76048d3 100644
@@ -3419,6 +3419,25 @@ typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
 				  const char *refs_check_dir,
 				  struct dir_iterator *iter);
 
+static int files_fsck_refs_name(struct fsck_options *o,
+				const char *gitdir UNUSED,
+				const char *refs_check_dir,
+				struct dir_iterator *iter)
+{
+	struct strbuf sb = STRBUF_INIT;
+	int ret = 0;
+
+	if (check_refname_format(iter->basename, REFNAME_ALLOW_ONELEVEL)) {
+		strbuf_addf(&sb, "%s/%s", refs_check_dir, iter->relative_path);
+		ret = fsck_refs_report(o, NULL, sb.buf,
+				       FSCK_MSG_BAD_REF_NAME,
+				       "invalid refname format");
+	}
+
+	strbuf_release(&sb);
+	return ret;
+}
+
 static int files_fsck_refs_dir(struct ref_store *ref_store,
 			       struct fsck_options *o,
 			       const char *refs_check_dir,
@@ -3470,6 +3489,7 @@ static int files_fsck_refs(struct ref_store *ref_store,
 {
 	int ret;
 	files_fsck_refs_fn fsck_refs_fns[]= {
+		files_fsck_refs_name,
 		NULL
 	};
 
t/t0602-reffiles-fsck.sh new file +101 -0

101 lines not on this page · raw message

Signature · 1 line
2.45.2
author
#103 lore raw

PATCH v7 9/9 fsck: add ref content check for files backend

In reply to shejialuo

Enhance the git-fsck(1) command by adding a check for reference content in the files backend. The new functionality ensures that symrefs, real symbolic link and regular refs are validated correctly.

In order to check the trailing content of the regular refs, add a new parameter trailing to parse_loose_ref_contents.

For symrefs, parse_loose_ref_contents will set the "referent". However, symbolic link could be either absolute or relative. Use "strbuf_add_real_path" to read the symbolic link and convert the relative path to absolute path. Then use "skip_prefix" to make it align with symref "referent".

Thus, the symrefs and symbolic links could share the same interface. Add a new function "files_fsck_symref_target" which aims at checking the following things:

1. whether the pointee is under the refs/ directory.
2. whether the pointee name is correct.
3. whether the pointee path is a wrong type in filesystem.

Last, add the following FSCK MESSAGEs:

1. "badRefContent(ERROR)": A ref has a bad content
2. "badSymrefPointee(ERROR)": The pointee of a symref is bad.
3. "trailingRefContent(WARN)": A ref content has trailing contents.

---
 Documentation/fsck-msgids.txt |   9 +++
 fsck.h                        |   3 +
 refs.c                        |   2 +-
 refs/files-backend.c          | 145 +++++++++++++++++++++++++++++++++-
 refs/refs-internal.h          |   5 +-
 t/t0602-reffiles-fsck.sh      | 110 ++++++++++++++++++++++++++
 6 files changed, 269 insertions(+), 5 deletions(-)
Documentation/fsck-msgids.txt +9 -0
index dab4012246..b1630a478b 100644
@@ -19,9 +19,15 @@
 `badParentSha1`::
 	(ERROR) A commit object has a bad parent sha1.
 
+`badRefContent`::
+	(ERROR) A ref has a bad content.
+
 `badRefName`::
 	(ERROR) A ref has a bad name.
 
+`badSymrefPointee`::
+	(ERROR) The pointee of a symref is bad.
+
 `badTagName`::
 	(INFO) A tag has an invalid format.
 
@@ -167,6 +173,9 @@
 `nullSha1`::
 	(WARN) Tree contains entries pointing to a null sha1.
 
+`trailingRefContent`::
+	(WARN) A ref content has trailing contents.
+
 `treeNotSorted`::
 	(ERROR) A tree is not properly sorted.
 
fsck.h +3 -0
index 90457d1a1f..637f596930 100644
@@ -32,6 +32,8 @@ enum fsck_msg_type {
 	FUNC(BAD_OBJECT_SHA1, ERROR) \
 	FUNC(BAD_PARENT_SHA1, ERROR) \
 	FUNC(BAD_REF_NAME, ERROR) \
+	FUNC(BAD_REF_CONTENT, ERROR) \
+	FUNC(BAD_SYMREF_POINTEE, ERROR) \
 	FUNC(BAD_TIMEZONE, ERROR) \
 	FUNC(BAD_TREE, ERROR) \
 	FUNC(BAD_TREE_SHA1, ERROR) \
@@ -72,6 +74,7 @@ enum fsck_msg_type {
 	FUNC(HAS_DOTDOT, WARN) \
 	FUNC(HAS_DOTGIT, WARN) \
 	FUNC(NULL_SHA1, WARN) \
+	FUNC(TRAILING_REF_CONTENT, WARN) \
 	FUNC(ZERO_PADDED_FILEMODE, WARN) \
 	FUNC(NUL_IN_COMMIT, WARN) \
 	FUNC(LARGE_PATHNAME, WARN) \
refs.c +1 -1
index 410919246b..eb82fb7d4e 100644
@@ -1760,7 +1760,7 @@ static int refs_read_special_head(struct ref_store *ref_store,
 	}
 
 	result = parse_loose_ref_contents(content.buf, oid, referent, type,
-					  failure_errno);
+					  failure_errno, NULL);
 
 done:
 	strbuf_release(&full_path);
refs/files-backend.c +143 -2

181 lines not on this page · raw message

refs/refs-internal.h +3 -2
index a905e187cd..2fabf41d14 100644
@@ -709,11 +709,12 @@ struct ref_store {
 
 /*
  * Parse contents of a loose ref file. *failure_errno maybe be set to EINVAL for
- * invalid contents.
+ * invalid contents. Also *trailing is set to the first character after the
+ * refname or NULL if the referent is not empty.
  */
 int parse_loose_ref_contents(const char *buf, struct object_id *oid,
 			     struct strbuf *referent, unsigned int *type,
-			     int *failure_errno);
+			     int *failure_errno, const char **trailing);
 
 /*
  * Fill in the generic part of refs and add it to our collection of
t/t0602-reffiles-fsck.sh +110 -0

114 lines not on this page · raw message

Signature · 1 line
2.45.2
author
#104 lore raw

v8 cover ref consistency check infra setup

In reply to shejialuo

Hi All:

This version enhances the patch 1 and patch 2 commit messages according to the review of Justin.

Thanks,
Jialuo

shejialuo (9):
fsck: rename "skiplist" to "oid_skiplist"
fsck: add a unified interface for reporting fsck messages
fsck: add refs-related options and error report function
refs: set up ref consistency check infrastructure builtin/refs: add verify subcommand builtin/fsck: add git-refs verify child process
files-backend: add unified interface for refs scanning
fsck: add ref name check for files backend
fsck: add ref content check for files backend

 Documentation/fsck-msgids.txt |  12 ++
 Documentation/git-refs.txt    |  13 ++
 builtin/fsck.c                |  32 ++++-
 builtin/mktag.c               |   1 +
 builtin/refs.c                |  44 ++++++
 fsck.c                        | 107 +++++++++++---
 fsck.h                        |  63 ++++++---
 object-file.c                 |  11 +-
 refs.c                        |   7 +-
 refs.h                        |   8 ++
 refs/debug.c                  |  11 ++
 refs/files-backend.c          | 255 +++++++++++++++++++++++++++++++++-
 refs/packed-backend.c         |   8 ++
 refs/refs-internal.h          |  11 +-
 refs/reftable-backend.c       |   8 ++
 t/t0602-reffiles-fsck.sh      | 211 ++++++++++++++++++++++++++++
 16 files changed, 745 insertions(+), 57 deletions(-)
 create mode 100755 t/t0602-reffiles-fsck.sh

Range-diff against v7:
1: f5fcf36132 ! 1: 61e475840f fsck: rename "skiplist" to "oid_skiplist"
@@ Metadata
## Commit message ##
fsck: rename "skiplist" to "oid_skiplist"

- Because we introduce ref consistency check. The original "skiplist" is a
- common option which is set up during handling user configs. To avoid
- causing ambiguity, rename "skiplist" to "oid_skiplist".
+ The "skiplist" field in "fsck_options" is related to objects. Because we
+ are going to introduce ref consistency check, the "skiplist" name is too
+ general which will make the caller think "skiplist" is related to both
+ the refs and objects.
+
+ It may seem that for both refs and objects, we should provide a general
+ "skiplist" here. However, the type for "skiplist" is struct oidset
+ which is totally unsuitable for refs.
+
+ To avoid above ambiguity, rename "skiplist" to "oid_skiplist".

2: c81b5b6fba ! 2: f2576d88a9 fsck: add a unified interface for reporting fsck messages
@@ Metadata
## Commit message ##
fsck: add a unified interface for reporting fsck messages

- The static function "report" provided by "fsck.c" aims at reporting the
- problems related to object database which cannot be reused for refs.
- In order to provide a unified interface which can report either objects
- or refs, create a new function "vfsck_report" by adding
- "checked_ref_name" parameter following the "report" prototype. However,
- instead of using "...", provide "va_list" to allow more flexibility.
+ The static function "report" provided by "fsck.c" aims at checking fsck
+ error type and calling the callback "error_func" to report the message.
+ However, "report" function is only related to object database which
+ cannot be reused for refs. In order to provide a unified interface which
+ can report either objects or refs, create a new function "vfsck_report"
+ by adding "checked_ref_name" parameter following the "report" prototype.
+ Instead of using "...", provide "va_list" to allow more flexibility.

- The "vfsck_report" function will use "error_func" registered in
- "fsck_options" function to report customized messages. Change
- "error_func" prototype to align with the "vfsck_report".
+ Like "report", the "vfsck_report" function will use "error_func"
+ registered in "fsck_options" to report customized messages. Change
+ "error_func" prototype to align with the new "vfsck_report".

- Change "report" function to make it use "vfsck_report" to report
- objects-related messages. Add a new function called "fsck_refs_report"
- to use "vfsck_report" to report refs-related messages.
+ Then, change "report" function to use "vfsck_report" to report objects
+ related messages. Add a new function called "fsck_refs_report" to use
+ "vfsck_report" to report refs related messages.

 3:  53156dc847 =  3:  c3c2dda50c fsck: add refs-related options and error report function
 4:  358f4a1be9 =  4:  e826dc17ec refs: set up ref consistency check infrastructure
 5:  e632859df6 =  5:  33cac4882b builtin/refs: add verify subcommand
 6:  5d7a6261ae =  6:  32668e3543 builtin/fsck: add `git-refs verify` child process
 7:  33311af1b1 =  7:  df83b2a990 files-backend: add unified interface for refs scanning
 8:  02bd3ac952 =  8:  c696c15651 fsck: add ref name check for files backend
 9:  94d123471d =  9:  8b0f3aeb9c fsck: add ref content check for files backend
Signature · 1 line
2.45.2
author
#105 lore raw

PATCH v8 1/9 fsck: rename "skiplist" to "oid_skiplist"

In reply to shejialuo

The "skiplist" field in "fsck_options" is related to objects. Because we are going to introduce ref consistency check, the "skiplist" name is too general which will make the caller think "skiplist" is related to both the refs and objects.

It may seem that for both refs and objects, we should provide a general "skiplist" here. However, the type for "skiplist" is struct oidset which is totally unsuitable for refs.

To avoid above ambiguity, rename "skiplist" to "oid_skiplist".

---
 fsck.c | 4 ++--
 fsck.h | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)
fsck.c +2 -2
index eea7145470..1960bfeba9 100644
@@ -205,7 +205,7 @@ void fsck_set_msg_types(struct fsck_options *options, const char *values)
 		if (!strcmp(buf, "skiplist")) {
 			if (equal == len)
 				die("skiplist requires a path");
-			oidset_parse_file(&options->skiplist, buf + equal + 1,
+			oidset_parse_file(&options->oid_skiplist, buf + equal + 1,
 					  the_repository->hash_algo);
 			buf += len + 1;
 			continue;
@@ -223,7 +223,7 @@ void fsck_set_msg_types(struct fsck_options *options, const char *values)
 static int object_on_skiplist(struct fsck_options *opts,
 			      const struct object_id *oid)
 {
-	return opts && oid && oidset_contains(&opts->skiplist, oid);
+	return opts && oid && oidset_contains(&opts->oid_skiplist, oid);
 }
 
 __attribute__((format (printf, 5, 6)))
fsck.h +1 -1
index 6085a384f6..1ee3dd85ba 100644
@@ -136,7 +136,7 @@ struct fsck_options {
 	fsck_error error_func;
 	unsigned strict:1;
 	enum fsck_msg_type *msg_type;
-	struct oidset skiplist;
+	struct oidset oid_skiplist;
 	struct oidset gitmodules_found;
 	struct oidset gitmodules_done;
 	struct oidset gitattributes_found;
Signature · 1 line
2.45.2
author
#106 lore raw

PATCH v8 2/9 fsck: add a unified interface for reporting fsck messages

In reply to shejialuo

The static function "report" provided by "fsck.c" aims at checking fsck error type and calling the callback "error_func" to report the message. However, "report" function is only related to object database which cannot be reused for refs. In order to provide a unified interface which can report either objects or refs, create a new function "vfsck_report" by adding "checked_ref_name" parameter following the "report" prototype. Instead of using "...", provide "va_list" to allow more flexibility.

Like "report", the "vfsck_report" function will use "error_func" registered in "fsck_options" to report customized messages. Change "error_func" prototype to align with the new "vfsck_report".

Then, change "report" function to use "vfsck_report" to report objects related messages. Add a new function called "fsck_refs_report" to use "vfsck_report" to report refs related messages.

---
 builtin/fsck.c  | 15 ++++-----
 builtin/mktag.c |  1 +
 fsck.c          | 81 ++++++++++++++++++++++++++++++++++++-------------
 fsck.h          | 42 ++++++++++++++++---------
 object-file.c   | 11 ++++---
 5 files changed, 102 insertions(+), 48 deletions(-)
builtin/fsck.c +8 -7
index d13a226c2e..de34538c4f 100644
@@ -89,12 +89,13 @@ static int objerror(struct object *obj, const char *err)
 	return -1;
 }
 
-static int fsck_error_func(struct fsck_options *o UNUSED,
-			   const struct object_id *oid,
-			   enum object_type object_type,
-			   enum fsck_msg_type msg_type,
-			   enum fsck_msg_id msg_id UNUSED,
-			   const char *message)
+static int fsck_objects_error_func(struct fsck_options *o UNUSED,
+				   const struct object_id *oid,
+				   enum object_type object_type,
+				   const char *checked_ref_name UNUSED,
+				   enum fsck_msg_type msg_type,
+				   enum fsck_msg_id msg_id UNUSED,
+				   const char *message)
 {
 	switch (msg_type) {
 	case FSCK_WARN:
@@ -938,7 +939,7 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
 
 	fsck_walk_options.walk = mark_object;
 	fsck_obj_options.walk = mark_used;
-	fsck_obj_options.error_func = fsck_error_func;
+	fsck_obj_options.error_func = fsck_objects_error_func;
 	if (check_strict)
 		fsck_obj_options.strict = 1;
 
builtin/mktag.c +1 -0
index 4767f1a97e..42f945c584 100644
@@ -20,6 +20,7 @@ static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
 static int mktag_fsck_error_func(struct fsck_options *o UNUSED,
 				 const struct object_id *oid UNUSED,
 				 enum object_type object_type UNUSED,
+				 const char *checked_ref_name UNUSED,
 				 enum fsck_msg_type msg_type,
 				 enum fsck_msg_id msg_id UNUSED,
 				 const char *message)
fsck.c +60 -21

115 lines not on this page · raw message

fsck.h +27 -15

81 lines not on this page · raw message

object-file.c +6 -5
index 065103be3e..d2c6427935 100644
@@ -2470,11 +2470,12 @@ int repo_has_object_file(struct repository *r,
  * give more context.
  */
 static int hash_format_check_report(struct fsck_options *opts UNUSED,
-				     const struct object_id *oid UNUSED,
-				     enum object_type object_type UNUSED,
-				     enum fsck_msg_type msg_type UNUSED,
-				     enum fsck_msg_id msg_id UNUSED,
-				     const char *message)
+				    const struct object_id *oid UNUSED,
+				    enum object_type object_type UNUSED,
+				    const char *ref_checked_name UNUSED,
+				    enum fsck_msg_type msg_type UNUSED,
+				    enum fsck_msg_id msg_id UNUSED,
+				    const char *message)
 {
 	error(_("object fails fsck: %s"), message);
 	return 1;
Signature · 1 line
2.45.2
#107 lore raw

Re: [GSoC][PATCH v8 2/9] fsck: add a unified interface for reporting fsck messages

Quoted 8 lines · shejialuo

The static function "report" provided by "fsck.c" aims at checking fsck error type and calling the callback "error_func" to report the message. However, "report" function is only related to object database which cannot be reused for refs. In order to provide a unified interface which can report either objects or refs, create a new function "vfsck_report" by adding "checked_ref_name" parameter following the "report" prototype. Instead of using "...", provide "va_list" to allow more flexibility.

Quoted 8 lines

Like "report", the "vfsck_report" function will use "error_func" registered in "fsck_options" to report customized messages. Change "error_func" prototype to align with the new "vfsck_report".

Then, change "report" function to use "vfsck_report" to report objects related messages. Add a new function called "fsck_refs_report" to use "vfsck_report" to report refs related messages.

Not sure I really understand why we need to do this. Why can't we simply add const char *checked_ref_name to the existing 'report' and propagate this also to 'error_func'. Why do we need all this parallel flows?

Apart from that, what does 'v' in 'vfsck_report' signify?

Perhaps it is also because this commit is doing a lot of things and we could have simplified it into smaller commits?

Quoted 98 lines
---
 builtin/fsck.c  | 15 ++++-----
 builtin/mktag.c |  1 +
 fsck.c          | 81 ++++++++++++++++++++++++++++++++++++-------------
 fsck.h          | 42 ++++++++++++++++---------
 object-file.c   | 11 ++++---
 5 files changed, 102 insertions(+), 48 deletions(-)
builtin/fsck.c +8 -7
index d13a226c2e..de34538c4f 100644
@@ -89,12 +89,13 @@ static int objerror(struct object *obj, const char *err)
 	return -1;
 }
 
-static int fsck_error_func(struct fsck_options *o UNUSED,
-			   const struct object_id *oid,
-			   enum object_type object_type,
-			   enum fsck_msg_type msg_type,
-			   enum fsck_msg_id msg_id UNUSED,
-			   const char *message)
+static int fsck_objects_error_func(struct fsck_options *o UNUSED,
+				   const struct object_id *oid,
+				   enum object_type object_type,
+				   const char *checked_ref_name UNUSED,
+				   enum fsck_msg_type msg_type,
+				   enum fsck_msg_id msg_id UNUSED,
+				   const char *message)
 {
 	switch (msg_type) {
 	case FSCK_WARN:
@@ -938,7 +939,7 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
 
 	fsck_walk_options.walk = mark_object;
 	fsck_obj_options.walk = mark_used;
-	fsck_obj_options.error_func = fsck_error_func;
+	fsck_obj_options.error_func = fsck_objects_error_func;
 	if (check_strict)
 		fsck_obj_options.strict = 1;
 
builtin/mktag.c +1 -0
index 4767f1a97e..42f945c584 100644
@@ -20,6 +20,7 @@ static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
 static int mktag_fsck_error_func(struct fsck_options *o UNUSED,
 				 const struct object_id *oid UNUSED,
 				 enum object_type object_type UNUSED,
+				 const char *checked_ref_name UNUSED,
 				 enum fsck_msg_type msg_type,
 				 enum fsck_msg_id msg_id UNUSED,
 				 const char *message)
fsck.c +14 -8
index 1960bfeba9..7182ce8e80 100644
@@ -226,12 +226,18 @@ static int object_on_skiplist(struct fsck_options *opts,
 	return opts && oid && oidset_contains(&opts->oid_skiplist, oid);
 }
 
-__attribute__((format (printf, 5, 6)))
-static int report(struct fsck_options *options,
-		  const struct object_id *oid, enum object_type object_type,
-		  enum fsck_msg_id msg_id, const char *fmt, ...)
+/*
+ * Provide a unified interface for either fscking refs or objects.
+ * It will get the current msg error type and call the error_func callback
+ * which is registered in the "fsck_options" struct.
+ */
+static int vfsck_report(struct fsck_options *options,
+			const struct object_id *oid,
+			enum object_type object_type,
+			const char *checked_ref_name,
+			enum fsck_msg_id msg_id, const char *fmt, va_list ap)
 {
-	va_list ap;
+	va_list ap_copy;
 	struct strbuf sb = STRBUF_INIT;
 	enum fsck_msg_type msg_type = fsck_msg_type(msg_id, options);
 	int result;
@@ -250,9 +256,9 @@ static int report(struct fsck_options *options,
 	prepare_msg_ids();
 	strbuf_addf(&sb, "%s: ", msg_id_info[msg_id].camelcased);
 
-	va_start(ap, fmt);
-	strbuf_vaddf(&sb, fmt, ap);
-	result = options->error_func(options, oid, object_type,
+	va_copy(ap_copy, ap);
+	strbuf_vaddf(&sb, fmt, ap_copy);
+	result = options->error_func(options, oid, object_type, checked_ref_name,
 				     msg_type, msg_id, sb.buf);
 	strbuf_release(&sb);
 	va_end(ap);

fsck.c static int report(struct fsck_options *options, · line 270

Quoted 5 lines
fsck.c +1 -0
@@ -260,6 +266,36 @@ static int report(struct fsck_options *options,
 	return result;
 }
 
+__attribute__((format (printf, 5, 6)))
 

Shouldn't this be moved to the header file too?

Quoted 5 lines
fsck.c +5 -0
+static int report(struct fsck_options *options,
+		  const struct object_id *oid, enum object_type object_type,
+		  enum fsck_msg_id msg_id, const char *fmt, ...)
+{
+	va_list ap;

fsck.c static int report(struct fsck_options *options, · line 284

Quoted 10 lines
fsck.c +10 -0
+	int result;
+	va_start(ap, fmt);
+	result = vfsck_report(options, oid, object_type, NULL,
+			      msg_id, fmt, ap);
+	va_end(ap);
+	return result;
+}
+
+
+

There is an extra newline here.

Quoted 177 lines
fsck.c +14 -0
+int fsck_refs_report(struct fsck_options *options,
+		     const struct object_id *oid,
+		     const char *checked_ref_name,
+		     enum fsck_msg_id msg_id, const char *fmt, ...)
+{
+	va_list ap;
+	int result;
+	va_start(ap, fmt);
+	result = vfsck_report(options, oid, OBJ_NONE,
+			      checked_ref_name, msg_id, fmt, ap);
+	va_end(ap);
+	return result;
+}
+
 void fsck_enable_object_names(struct fsck_options *options)
 {
fsck.c +16 -13
 	if (!options->object_names)
@@ -1200,12 +1236,13 @@ int fsck_buffer(const struct object_id *oid, enum object_type type,
 		      type);
 }
 
-int fsck_error_function(struct fsck_options *o,
-			const struct object_id *oid,
-			enum object_type object_type UNUSED,
-			enum fsck_msg_type msg_type,
-			enum fsck_msg_id msg_id UNUSED,
-			const char *message)
+int fsck_objects_error_function(struct fsck_options *o,
+				const struct object_id *oid,
+				enum object_type object_type UNUSED,
+				const char *checked_ref_name UNUSED,
+				enum fsck_msg_type msg_type,
+				enum fsck_msg_id msg_id UNUSED,
+				const char *message)
 {
 	if (msg_type == FSCK_WARN) {
 		warning("object %s: %s", fsck_describe_object(o, oid), message);
@@ -1303,16 +1340,18 @@ int git_fsck_config(const char *var, const char *value,
  * Custom error callbacks that are used in more than one place.
  */
 
-int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o,
-					   const struct object_id *oid,
-					   enum object_type object_type,
-					   enum fsck_msg_type msg_type,
-					   enum fsck_msg_id msg_id,
-					   const char *message)
+int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o,
+						   const struct object_id *oid,
+						   enum object_type object_type,
+						   const char *checked_ref_name,
+						   enum fsck_msg_type msg_type,
+						   enum fsck_msg_id msg_id,
+						   const char *message)
 {
 	if (msg_id == FSCK_MSG_GITMODULES_MISSING) {
 		puts(oid_to_hex(oid));
 		return 0;
 	}
-	return fsck_error_function(o, oid, object_type, msg_type, msg_id, message);
+	return fsck_objects_error_function(o, oid, object_type, checked_ref_name,
+					   msg_type, msg_id, message);
 }
fsck.h +27 -15
index 1ee3dd85ba..f703dfb5e8 100644
@@ -114,22 +114,27 @@ int is_valid_msg_type(const char *msg_id, const char *msg_type);
 typedef int (*fsck_walk_func)(struct object *obj, enum object_type object_type,
 			      void *data, struct fsck_options *options);
 
-/* callback for fsck_object, type is FSCK_ERROR or FSCK_WARN */
+/*
+ * callback function for reporting errors when checking either objects or refs
+ */
 typedef int (*fsck_error)(struct fsck_options *o,
 			  const struct object_id *oid, enum object_type object_type,
+			  const char *checked_ref_name,
 			  enum fsck_msg_type msg_type, enum fsck_msg_id msg_id,
 			  const char *message);
 
-int fsck_error_function(struct fsck_options *o,
-			const struct object_id *oid, enum object_type object_type,
-			enum fsck_msg_type msg_type, enum fsck_msg_id msg_id,
-			const char *message);
-int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o,
-					   const struct object_id *oid,
-					   enum object_type object_type,
-					   enum fsck_msg_type msg_type,
-					   enum fsck_msg_id msg_id,
-					   const char *message);
+int fsck_objects_error_function(struct fsck_options *o,
+				const struct object_id *oid, enum object_type object_type,
+				const char *checked_ref_name,
+				enum fsck_msg_type msg_type, enum fsck_msg_id msg_id,
+				const char *message);
+int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o,
+						   const struct object_id *oid,
+						   enum object_type object_type,
+						   const char *checked_ref_name,
+						   enum fsck_msg_type msg_type,
+						   enum fsck_msg_id msg_id,
+						   const char *message);
 
 struct fsck_options {
 	fsck_walk_func walk;
@@ -145,12 +150,12 @@ struct fsck_options {
 };
 
 #define FSCK_OPTIONS_DEFAULT { \
-	.skiplist = OIDSET_INIT, \
+	.oid_skiplist = OIDSET_INIT, \
 	.gitmodules_found = OIDSET_INIT, \
 	.gitmodules_done = OIDSET_INIT, \
 	.gitattributes_found = OIDSET_INIT, \
 	.gitattributes_done = OIDSET_INIT, \
-	.error_func = fsck_error_function \
+	.error_func = fsck_objects_error_function \
 }
 #define FSCK_OPTIONS_STRICT { \
 	.strict = 1, \
@@ -158,7 +163,7 @@ struct fsck_options {
 	.gitmodules_done = OIDSET_INIT, \
 	.gitattributes_found = OIDSET_INIT, \
 	.gitattributes_done = OIDSET_INIT, \
-	.error_func = fsck_error_function, \
+	.error_func = fsck_objects_error_function, \
 }
 #define FSCK_OPTIONS_MISSING_GITMODULES { \
 	.strict = 1, \
@@ -166,7 +171,7 @@ struct fsck_options {
 	.gitmodules_done = OIDSET_INIT, \
 	.gitattributes_found = OIDSET_INIT, \
 	.gitattributes_done = OIDSET_INIT, \
-	.error_func = fsck_error_cb_print_missing_gitmodules, \
+	.error_func = fsck_objects_error_cb_print_missing_gitmodules, \
 }
 
 /* descend in all linked child objects
@@ -209,6 +214,13 @@ int fsck_tag_standalone(const struct object_id *oid, const char *buffer,
  */
 int fsck_finish(struct fsck_options *options);
 
+__attribute__((format (printf, 5, 6)))
+int fsck_refs_report(struct fsck_options *options,
+		     const struct object_id *oid,
+		     const char *checked_ref_name,
+		     enum fsck_msg_id msg_id,
+		     const char *fmt, ...);
+
 /*
  * Subsystem for storing human-readable names for each object.
  *
object-file.c +6 -6
index 065103be3e..d2c6427935 100644
@@ -2470,11 +2470,12 @@ int repo_has_object_file(struct repository *r,
  * give more context.
  */
 static int hash_format_check_report(struct fsck_options *opts UNUSED,
-				     const struct object_id *oid UNUSED,
-				     enum object_type object_type UNUSED,
-				     enum fsck_msg_type msg_type UNUSED,
-				     enum fsck_msg_id msg_id UNUSED,
-				     const char *message)
+				    const struct object_id *oid UNUSED,
+				    enum object_type object_type UNUSED,
+				    const char *ref_checked_name UNUSED,
+				    enum fsck_msg_type msg_type UNUSED,
+				    enum fsck_msg_id msg_id UNUSED,
+				    const char *message)
 {
 	error(_("object fails fsck: %s"), message);
 	return 1;
--

2.45.2

author
#108 lore raw

Re: [GSoC][PATCH v8 2/9] fsck: add a unified interface for reporting fsck messages

Quoted 19 lines · Karthik Nayak
Quoted 8 lines · shejialuo

The static function "report" provided by "fsck.c" aims at checking fsck error type and calling the callback "error_func" to report the message. However, "report" function is only related to object database which cannot be reused for refs. In order to provide a unified interface which can report either objects or refs, create a new function "vfsck_report" by adding "checked_ref_name" parameter following the "report" prototype. Instead of using "...", provide "va_list" to allow more flexibility.

Quoted 8 lines

Like "report", the "vfsck_report" function will use "error_func" registered in "fsck_options" to report customized messages. Change "error_func" prototype to align with the new "vfsck_report".

Then, change "report" function to use "vfsck_report" to report objects related messages. Add a new function called "fsck_refs_report" to use "vfsck_report" to report refs related messages.

Quoted 5 lines

Not sure I really understand why we need to do this. Why can't we simply add const char *checked_ref_name to the existing 'report' and propagate this also to 'error_func'. Why do we need all this parallel flows?

Yes, we could just add a parameter "const char *checked_ref_name" to the existing "report". This may seem the simplest way to do. However, it will also introduce some trouble below:

1. "report" function should be exported to the outside, we need to rename it to "fsck_report". Well, we need to change a lot of code here. And we MUST do this, because "report" is a general name. When exporting to the outside, it's not proper.
2. When we add a new parameter in "report", for all the "report" calls, we need to pass this new parameter with NULL.

Use this way, we could do not change "report" function prototype and the corresponding calls. Most importantly, we could let the caller feel transparent. Using "report", caller can just ignore "checked_ref_name". Also for "fsck_refs_report", we could ignore some UNUSED parameters.

So I think this design is more elegant than just adding a new parameter in the existing "report" function.

Quoted 2 lines

Apart from that, what does 'v' in 'vfsck_report' signify?

Because I use "va_list" parameter, I want to emphasis on this. And this provides flexibility that we could add a "fsck_report" function later. There are many codes in git code base using this way. I just followed this.

Quoted 3 lines

Perhaps it is also because this commit is doing a lot of things and we could have simplified it into smaller commits?

Actually, this commit is very clear. I just want to provide a unified function "vfsck_report" here. And let the "report" use this function and "fsck_refs_report" function use this.

So I don't know whether we should split this commit into multiple commits. They are just tied together.

#109 lore raw

Re: [GSoC][PATCH v8 2/9] fsck: add a unified interface for reporting fsck messages

Quoted 26 lines · shejialuo
Quoted 19 lines · Karthik Nayak
Quoted 8 lines · shejialuo

The static function "report" provided by "fsck.c" aims at checking fsck error type and calling the callback "error_func" to report the message. However, "report" function is only related to object database which cannot be reused for refs. In order to provide a unified interface which can report either objects or refs, create a new function "vfsck_report" by adding "checked_ref_name" parameter following the "report" prototype. Instead of using "...", provide "va_list" to allow more flexibility.

Quoted 8 lines

Like "report", the "vfsck_report" function will use "error_func" registered in "fsck_options" to report customized messages. Change "error_func" prototype to align with the new "vfsck_report".

Then, change "report" function to use "vfsck_report" to report objects related messages. Add a new function called "fsck_refs_report" to use "vfsck_report" to report refs related messages.

Quoted 5 lines

Not sure I really understand why we need to do this. Why can't we simply add const char *checked_ref_name to the existing 'report' and propagate this also to 'error_func'. Why do we need all this parallel flows?

Quoted 9 lines

Yes, we could just add a parameter "const char *checked_ref_name" to the existing "report". This may seem the simplest way to do. However, it will also introduce some trouble below:

1. "report" function should be exported to the outside, we need to rename it to "fsck_report". Well, we need to change a lot of code here. And we MUST do this, because "report" is a general name. When exporting to the outside, it's not proper.

agreed.

Quoted 3 lines

2. When we add a new parameter in "report", for all the "report" calls, we need to pass this new parameter with NULL.

agreed too.

Quoted 8 lines

Use this way, we could do not change "report" function prototype and the corresponding calls. Most importantly, we could let the caller feel transparent. Using "report", caller can just ignore "checked_ref_name". Also for "fsck_refs_report", we could ignore some UNUSED parameters.

So I think this design is more elegant than just adding a new parameter in the existing "report" function.

I understand what you're saying, I also checked and can see that there are 60 references to the report() function. So perhaps there is some merit in keeping it as is and adding a new 'report_refs()'.

Quoted 8 lines
Quoted 2 lines

Apart from that, what does 'v' in 'vfsck_report' signify?

Because I use "va_list" parameter, I want to emphasis on this. And this provides flexibility that we could add a "fsck_report" function later. There are many codes in git code base using this way. I just followed this.

I see. Makes sense.

Quoted 10 lines
Quoted 3 lines

Perhaps it is also because this commit is doing a lot of things and we could have simplified it into smaller commits?

Actually, this commit is very clear. I just want to provide a unified function "vfsck_report" here. And let the "report" use this function and "fsck_refs_report" function use this.

So I don't know whether we should split this commit into multiple commits. They are just tied together.

author
#110 lore raw

PATCH v8 3/9 fsck: add refs-related options and error report function

In reply to shejialuo

Add refs-related options to the "fsck_options", create refs-specific "error_func" callback "fsck_refs_error_function".

"fsck_refs_error_function" will use the "oid" parameter. When the caller passes the oid, it will use "oid_to_hex" to get the corresponding hex value to report to the caller.

Last, add "FSCK_REFS_OPTIONS_DEFAULT" and "FSCK_REFS_OPTIONS_STRICT" macros to create refs options easily.

---
 fsck.c | 22 ++++++++++++++++++++++
 fsck.h | 15 +++++++++++++++
 2 files changed, 37 insertions(+)
fsck.c +22 -0
index 7182ce8e80..d1dcbdcac2 100644
@@ -1252,6 +1252,28 @@ int fsck_objects_error_function(struct fsck_options *o,
 	return 1;
 }
 
+int fsck_refs_error_function(struct fsck_options *options UNUSED,
+			     const struct object_id *oid,
+			     enum object_type object_type UNUSED,
+			     const char *checked_ref_name,
+			     enum fsck_msg_type msg_type,
+			     enum fsck_msg_id msg_id UNUSED,
+			     const char *message)
+{
+	static struct strbuf sb = STRBUF_INIT;
+
+	strbuf_addstr(&sb, checked_ref_name);
+	if (oid)
+		strbuf_addf(&sb, " -> (%s)", oid_to_hex(oid));
+
+	if (msg_type == FSCK_WARN) {
+		warning("%s: %s", sb.buf, message);
+		return 0;
+	}
+	error("%s: %s", sb.buf, message);
+	return 1;
+}
+
 static int fsck_blobs(struct oidset *blobs_found, struct oidset *blobs_done,
 		      enum fsck_msg_id msg_missing, enum fsck_msg_id msg_type,
 		      struct fsck_options *options, const char *blob_type)
fsck.h +15 -0
index f703dfb5e8..246055c0f9 100644
@@ -135,11 +135,19 @@ int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o,
 						   enum fsck_msg_type msg_type,
 						   enum fsck_msg_id msg_id,
 						   const char *message);
+int fsck_refs_error_function(struct fsck_options *options,
+			     const struct object_id *oid,
+			     enum object_type object_type,
+			     const char *checked_ref_name,
+			     enum fsck_msg_type msg_type,
+			     enum fsck_msg_id msg_id,
+			     const char *message);
 
 struct fsck_options {
 	fsck_walk_func walk;
 	fsck_error error_func;
 	unsigned strict:1;
+	unsigned verbose_refs:1;
 	enum fsck_msg_type *msg_type;
 	struct oidset oid_skiplist;
 	struct oidset gitmodules_found;
@@ -173,6 +181,13 @@ struct fsck_options {
 	.gitattributes_done = OIDSET_INIT, \
 	.error_func = fsck_objects_error_cb_print_missing_gitmodules, \
 }
+#define FSCK_REFS_OPTIONS_DEFAULT { \
+	.error_func = fsck_refs_error_function, \
+}
+#define FSCK_REFS_OPTIONS_STRICT { \
+	.strict = 1, \
+	.error_func = fsck_refs_error_function, \
+}
 
 /* descend in all linked child objects
  * the return value is:
Signature · 1 line
2.45.2
#111 lore raw

Re: [GSoC][PATCH v8 3/9] fsck: add refs-related options and error report function

Quoted 10 lines · shejialuo

Add refs-related options to the "fsck_options", create refs-specific "error_func" callback "fsck_refs_error_function".

"fsck_refs_error_function" will use the "oid" parameter. When the caller passes the oid, it will use "oid_to_hex" to get the corresponding hex value to report to the caller.

Last, add "FSCK_REFS_OPTIONS_DEFAULT" and "FSCK_REFS_OPTIONS_STRICT" macros to create refs options easily.

Carrying over from the previous commit, couldn't we simply do something like:

    diff --git a/fsck.c b/fsck.c
    index eea7145470..32ae36a4fc 100644
    --- a/fsck.c
    +++ b/fsck.c
    @@ -1202,17 +1203,33 @@ int fsck_buffer(const struct object_id

*oid, enum object_type type,

     int fsck_error_function(struct fsck_options *o,
     			const struct object_id *oid,
    -			enum object_type object_type UNUSED,
    +			enum object_type object_type,
    +			const char *checked_ref_name,
     			enum fsck_msg_type msg_type,
     			enum fsck_msg_id msg_id UNUSED,
     			const char *message)
     {
    +	static struct strbuf sb = STRBUF_INIT;
    +	int ret = 0;
    +
    +	if (checked_ref_name) {
    +		strbuf_addstr("ref %s", checked_ref_name);
    +		if (oid)
    +			strbuf_addf(&sb, " -> (%s)", oid_to_hex(oid));
    +	} else {
    +		strbuf_addstr("object %s", fsck_describe_object(o, oid));
    +	}
    +
     	if (msg_type == FSCK_WARN) {
    -		warning("object %s: %s", fsck_describe_object(o, oid), message);
    -		return 0;
    +		warning("%s: %s", sb.buf, message);
    +		ret = 0;
    +		goto cleanup;
     	}
    -	error("object %s: %s", fsck_describe_object(o, oid), message);
    -	return 1;
    +	error("%s: %s", sb.buf, message);
    +
    +cleanup:
    +	strbuf_release(&sb);
    +	return ret;
     }
Quoted 28 lines
---
 fsck.c | 22 ++++++++++++++++++++++
 fsck.h | 15 +++++++++++++++
 2 files changed, 37 insertions(+)
fsck.c +12 -0
index 7182ce8e80..d1dcbdcac2 100644
@@ -1252,6 +1252,28 @@ int fsck_objects_error_function(struct fsck_options *o,
 	return 1;
 }
 
+int fsck_refs_error_function(struct fsck_options *options UNUSED,
+			     const struct object_id *oid,
+			     enum object_type object_type UNUSED,
+			     const char *checked_ref_name,
+			     enum fsck_msg_type msg_type,
+			     enum fsck_msg_id msg_id UNUSED,
+			     const char *message)
+{
+	static struct strbuf sb = STRBUF_INIT;
+
+	strbuf_addstr(&sb, checked_ref_name);
+	if (oid)

fsck.c int fsck_objects_error_function(struct fsck_options *o, · line 1276

Quoted 10 lines
fsck.c +10 -0
+		strbuf_addf(&sb, " -> (%s)", oid_to_hex(oid));
+
+	if (msg_type == FSCK_WARN) {
+		warning("%s: %s", sb.buf, message);
+		return 0;
+	}
+	error("%s: %s", sb.buf, message);
+	return 1;
+}
+

We don't free strbuf here.

Quoted 14 lines
fsck.c +0 -0
 static int fsck_blobs(struct oidset *blobs_found, struct oidset *blobs_done,
 		      enum fsck_msg_id msg_missing, enum fsck_msg_id msg_type,
 		      struct fsck_options *options, const char *blob_type)
fsck.h +3 -0
index f703dfb5e8..246055c0f9 100644
@@ -135,11 +135,19 @@ int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o,
 						   enum fsck_msg_type msg_type,
 						   enum fsck_msg_id msg_id,
 						   const char *message);
+int fsck_refs_error_function(struct fsck_options *options,
+			     const struct object_id *oid,
+			     enum object_type object_type,

fsck.h int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o, · line 150

Quoted 10 lines
fsck.h +5 -0
+			     const char *checked_ref_name,
+			     enum fsck_msg_type msg_type,
+			     enum fsck_msg_id msg_id,
+			     const char *message);
 
 struct fsck_options {
 	fsck_walk_func walk;
 	fsck_error error_func;
 	unsigned strict:1;
+	unsigned verbose_refs:1;

Nit: Here we have the subject 'refs' towards the end of the name.

fsck.h int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o, · line 152

Quoted 2 lines
fsck.h +0 -0
 	enum fsck_msg_type *msg_type;
 	struct oidset oid_skiplist;

but here we have the subject 'oid' at the start of the name. Perhaps we can rename this to 'skip_oids'?

Quoted 17 lines
fsck.h +7 -1
 	struct oidset gitmodules_found;
@@ -173,6 +181,13 @@ struct fsck_options {
 	.gitattributes_done = OIDSET_INIT, \
 	.error_func = fsck_objects_error_cb_print_missing_gitmodules, \
 }
+#define FSCK_REFS_OPTIONS_DEFAULT { \
+	.error_func = fsck_refs_error_function, \
+}
+#define FSCK_REFS_OPTIONS_STRICT { \
+	.strict = 1, \
+	.error_func = fsck_refs_error_function, \
+}
 
 /* descend in all linked child objects
  * the return value is:
--

2.45.2

author
#112 lore raw

Re: [GSoC][PATCH v8 3/9] fsck: add refs-related options and error report function

Quoted 50 lines · Karthik Nayak
Quoted 10 lines · shejialuo

Add refs-related options to the "fsck_options", create refs-specific "error_func" callback "fsck_refs_error_function".

"fsck_refs_error_function" will use the "oid" parameter. When the caller passes the oid, it will use "oid_to_hex" to get the corresponding hex value to report to the caller.

Last, add "FSCK_REFS_OPTIONS_DEFAULT" and "FSCK_REFS_OPTIONS_STRICT" macros to create refs options easily.

Carrying over from the previous commit, couldn't we simply do something like:

    diff --git a/fsck.c b/fsck.c
    index eea7145470..32ae36a4fc 100644
    --- a/fsck.c
    +++ b/fsck.c
    @@ -1202,17 +1203,33 @@ int fsck_buffer(const struct object_id

*oid, enum object_type type,

     int fsck_error_function(struct fsck_options *o,
     			const struct object_id *oid,
    -			enum object_type object_type UNUSED,
    +			enum object_type object_type,
    +			const char *checked_ref_name,
     			enum fsck_msg_type msg_type,
     			enum fsck_msg_id msg_id UNUSED,
     			const char *message)
     {
    +	static struct strbuf sb = STRBUF_INIT;
    +	int ret = 0;
    +
    +	if (checked_ref_name) {
    +		strbuf_addstr("ref %s", checked_ref_name);
    +		if (oid)
    +			strbuf_addf(&sb, " -> (%s)", oid_to_hex(oid));
    +	} else {
    +		strbuf_addstr("object %s", fsck_describe_object(o, oid));
    +	}
    +
     	if (msg_type == FSCK_WARN) {
    -		warning("object %s: %s", fsck_describe_object(o, oid), message);
    -		return 0;
    +		warning("%s: %s", sb.buf, message);
    +		ret = 0;
    +		goto cleanup;
     	}
Quoted 10 lines
    -	error("object %s: %s", fsck_describe_object(o, oid), message);
    -	return 1;
    +	error("%s: %s", sb.buf, message);
    +
    +cleanup:
    +	strbuf_release(&sb);
    +	return ret;
     }

However, "fsck_error_function" will be used as the callback for object-related checks. I don't think we should use one function to incorporate all the situations.

For example, if we pass both "checked_ref_name" and "oid" here, how does this one function knows whether we handle refs or objects. Although we may use some flags here to choose the different situations, it's bad idea from my perspective. Adding a new callback here will be cleaner.

Quoted 32 lines
Quoted 32 lines
---
 fsck.c | 22 ++++++++++++++++++++++
 fsck.h | 15 +++++++++++++++
 2 files changed, 37 insertions(+)
fsck.c +16 -0
index 7182ce8e80..d1dcbdcac2 100644
@@ -1252,6 +1252,28 @@ int fsck_objects_error_function(struct fsck_options *o,
 	return 1;
 }
 
+int fsck_refs_error_function(struct fsck_options *options UNUSED,
+			     const struct object_id *oid,
+			     enum object_type object_type UNUSED,
+			     const char *checked_ref_name,
+			     enum fsck_msg_type msg_type,
+			     enum fsck_msg_id msg_id UNUSED,
+			     const char *message)
+{
+	static struct strbuf sb = STRBUF_INIT;
+
+	strbuf_addstr(&sb, checked_ref_name);
+	if (oid)
+		strbuf_addf(&sb, " -> (%s)", oid_to_hex(oid));
+
+	if (msg_type == FSCK_WARN) {
+		warning("%s: %s", sb.buf, message);
Quoted 9 lines

fsck.c int fsck_objects_error_function(struct fsck_options *o, · line 1276

Quoted 6 lines
fsck.c +6 -0
+		return 0;
+	}
+	error("%s: %s", sb.buf, message);
+	return 1;
+}
+

We don't free strbuf here.

Yes, I made a mistake here. I should not define the "sb" static, will fix it in the next version.

Quoted 27 lines
Quoted 14 lines
fsck.c +0 -0
 static int fsck_blobs(struct oidset *blobs_found, struct oidset *blobs_done,
 		      enum fsck_msg_id msg_missing, enum fsck_msg_id msg_type,
 		      struct fsck_options *options, const char *blob_type)
fsck.h +3 -0
index f703dfb5e8..246055c0f9 100644
@@ -135,11 +135,19 @@ int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o,
 						   enum fsck_msg_type msg_type,
 						   enum fsck_msg_id msg_id,
 						   const char *message);
+int fsck_refs_error_function(struct fsck_options *options,
+			     const struct object_id *oid,
+			     enum object_type object_type,

fsck.h int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o, · line 150

Quoted 10 lines
fsck.h +5 -0
+			     const char *checked_ref_name,
+			     enum fsck_msg_type msg_type,
+			     enum fsck_msg_id msg_id,
+			     const char *message);
 
 struct fsck_options {
 	fsck_walk_func walk;
 	fsck_error error_func;
 	unsigned strict:1;
+	unsigned verbose_refs:1;

Nit: Here we have the subject 'refs' towards the end of the name.

Quoted 6 lines

fsck.h int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o, · line 152

Quoted 2 lines
fsck.h +0 -0
 	enum fsck_msg_type *msg_type;
 	struct oidset oid_skiplist;

but here we have the subject 'oid' at the start of the name. Perhaps we can rename this to 'skip_oids'?

I agree. "skip_oids" will be more meaningful. I will fix it in the next version.

Quoted 17 lines
Quoted 17 lines
fsck.h +7 -1
 	struct oidset gitmodules_found;
@@ -173,6 +181,13 @@ struct fsck_options {
 	.gitattributes_done = OIDSET_INIT, \
 	.error_func = fsck_objects_error_cb_print_missing_gitmodules, \
 }
+#define FSCK_REFS_OPTIONS_DEFAULT { \
+	.error_func = fsck_refs_error_function, \
+}
+#define FSCK_REFS_OPTIONS_STRICT { \
+	.strict = 1, \
+	.error_func = fsck_refs_error_function, \
+}
 
 /* descend in all linked child objects
  * the return value is:
--

2.45.2

author
#113 lore raw

PATCH v8 4/9 refs: set up ref consistency check infrastructure

In reply to shejialuo

The interfaces defined in the ref_storage_be are carefully structured in semantic. It's organized as the five parts:

1. The name and the initialization interfaces.
2. The ref transaction interfaces.
3. The ref internal interfaces (pack, rename and copy).
4. The ref filesystem interfaces.
5. The reflog related interfaces.

To keep consistent with the git-fsck(1), add a new interface named "fsck_refs_fn" to the end of "ref_storage_be". This semantic cannot be grouped into any above five categories. Explicitly add blank line to make it different from others.

---
 refs.c                  |  5 +++++
 refs.h                  |  8 ++++++++
 refs/debug.c            | 11 +++++++++++
 refs/files-backend.c    | 15 ++++++++++++++-
 refs/packed-backend.c   |  8 ++++++++
 refs/refs-internal.h    |  6 ++++++
 refs/reftable-backend.c |  8 ++++++++
 7 files changed, 60 insertions(+), 1 deletion(-)
refs.c +5 -0
index bb90a18875..410919246b 100644
@@ -318,6 +318,11 @@ int check_refname_format(const char *refname, int flags)
 	return check_or_sanitize_refname(refname, flags, NULL);
 }
 
+int refs_fsck(struct ref_store *refs, struct fsck_options *o)
+{
+	return refs->be->fsck(refs, o);
+}
+
 void sanitize_refname_component(const char *refname, struct strbuf *out)
 {
 	if (check_or_sanitize_refname(refname, REFNAME_ALLOW_ONELEVEL, out))
refs.h +8 -0
index 0ecba21b4a..804d6a7fce 100644
@@ -4,6 +4,7 @@
 #include "commit.h"
 #include "repository.h"
 
+struct fsck_options;
 struct object_id;
 struct ref_store;
 struct strbuf;
@@ -541,6 +542,13 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat
  */
 int check_refname_format(const char *refname, int flags);
 
+/*
+ * Check the reference database for consistency. Return 0 if refs and
+ * reflogs are consistent, and non-zero otherwise. The errors will be
+ * written to stderr.
+ */
+int refs_fsck(struct ref_store *refs, struct fsck_options *o);
+
 /*
  * Apply the rules from check_refname_format, but mutate the result until it
  * is acceptable, and place the result in "out".
refs/debug.c +11 -0
index 547d9245b9..45e2e784a0 100644
@@ -419,6 +419,15 @@ static int debug_reflog_expire(struct ref_store *ref_store, const char *refname,
 	return res;
 }
 
+static int debug_fsck(struct ref_store *ref_store,
+		      struct fsck_options *o)
+{
+	struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store;
+	int res = drefs->refs->be->fsck(drefs->refs, o);
+	trace_printf_key(&trace_refs, "fsck: %d\n", res);
+	return res;
+}
+
 struct ref_storage_be refs_be_debug = {
 	.name = "debug",
 	.init = NULL,
@@ -451,4 +460,6 @@ struct ref_storage_be refs_be_debug = {
 	.create_reflog = debug_create_reflog,
 	.delete_reflog = debug_delete_reflog,
 	.reflog_expire = debug_reflog_expire,
+
+	.fsck = debug_fsck,
 };
refs/files-backend.c +14 -1
index aa52d9be7c..d89eeda8ef 100644
@@ -3408,6 +3408,17 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,
 	return ret;
 }
 
+static int files_fsck(struct ref_store *ref_store,
+		      struct fsck_options *o)
+{
+	int ret;
+	struct files_ref_store *refs =
+		files_downcast(ref_store, REF_STORE_READ, "fsck");
+
+	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o);
+	return ret;
+}
+
 struct ref_storage_be refs_be_files = {
 	.name = "files",
 	.init = files_ref_store_init,
@@ -3434,5 +3445,7 @@ struct ref_storage_be refs_be_files = {
 	.reflog_exists = files_reflog_exists,
 	.create_reflog = files_create_reflog,
 	.delete_reflog = files_delete_reflog,
-	.reflog_expire = files_reflog_expire
+	.reflog_expire = files_reflog_expire,
+
+	.fsck = files_fsck,
 };
refs/packed-backend.c +8 -0
index a0666407cd..5209b0b212 100644
@@ -1735,6 +1735,12 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_s
 	return empty_ref_iterator_begin();
 }
 
+static int packed_fsck(struct ref_store *ref_store,
+		       struct fsck_options *o)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_packed = {
 	.name = "packed",
 	.init = packed_ref_store_init,
@@ -1762,4 +1768,6 @@ struct ref_storage_be refs_be_packed = {
 	.create_reflog = NULL,
 	.delete_reflog = NULL,
 	.reflog_expire = NULL,
+
+	.fsck = packed_fsck,
 };
refs/refs-internal.h +6 -0
index fa975d69aa..a905e187cd 100644
@@ -4,6 +4,7 @@
 #include "refs.h"
 #include "iterator.h"
 
+struct fsck_options;
 struct ref_transaction;
 
 /*
@@ -650,6 +651,9 @@ typedef int read_raw_ref_fn(struct ref_store *ref_store, const char *refname,
 typedef int read_symbolic_ref_fn(struct ref_store *ref_store, const char *refname,
 				 struct strbuf *referent);
 
+typedef int fsck_fn(struct ref_store *ref_store,
+		    struct fsck_options *o);
+
 struct ref_storage_be {
 	const char *name;
 	ref_store_init_fn *init;
@@ -677,6 +681,8 @@ struct ref_storage_be {
 	create_reflog_fn *create_reflog;
 	delete_reflog_fn *delete_reflog;
 	reflog_expire_fn *reflog_expire;
+
+	fsck_fn *fsck;
 };
 
 extern struct ref_storage_be refs_be_files;
refs/reftable-backend.c +8 -0
index fbe74c239d..b5a1a526df 100644
@@ -2303,6 +2303,12 @@ static int reftable_be_reflog_expire(struct ref_store *ref_store,
 	return ret;
 }
 
+static int reftable_be_fsck(struct ref_store *ref_store,
+			    struct fsck_options *o)
+{
+	return 0;
+}
+
 struct ref_storage_be refs_be_reftable = {
 	.name = "reftable",
 	.init = reftable_be_init,
@@ -2330,4 +2336,6 @@ struct ref_storage_be refs_be_reftable = {
 	.create_reflog = reftable_be_create_reflog,
 	.delete_reflog = reftable_be_delete_reflog,
 	.reflog_expire = reftable_be_reflog_expire,
+
+	.fsck = reftable_be_fsck,
 };
Signature · 1 line
2.45.2
author
#114 lore raw

PATCH v8 5/9 builtin/refs: add verify subcommand

In reply to shejialuo

Introduce a new subcommand "verify" in git-refs(1) to allow the user to check the reference database consistency.

---
 Documentation/git-refs.txt | 13 +++++++++++
 builtin/refs.c             | 44 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
Documentation/git-refs.txt +13 -0
index 5b99e04385..1244a85b64 100644
@@ -10,6 +10,7 @@ SYNOPSIS
 --------
 [verse]
 'git refs migrate' --ref-format=<format> [--dry-run]
+'git refs verify' [--strict] [--verbose]
 
 DESCRIPTION
 -----------
@@ -22,6 +23,9 @@ COMMANDS
 migrate::
 	Migrate ref store between different formats.
 
+verify::
+	Verify reference database consistency.
+
 OPTIONS
 -------
 
@@ -39,6 +43,15 @@ include::ref-storage-format.txt[]
 	can be used to double check that the migration works as expected before
 	performing the actual migration.
 
+The following options are specific to 'git refs verify':
+
+--strict::
+	Enable more strict checking, every WARN severity for the `Fsck Messages`
+	be seen as ERROR. See linkgit:git-fsck[1].
+
+--verbose::
+	When verifying the reference database consistency, be chatty.
+
 KNOWN LIMITATIONS
 -----------------
 
builtin/refs.c +44 -0

69 lines not on this page · raw message

Signature · 1 line
2.45.2
author
#115 lore raw

PATCH v8 6/9 builtin/fsck: add `git-refs verify` child process

In reply to shejialuo

Introduce a new function "fsck_refs" that initializes and runs a child process to execute the "git-refs verify" command.

---
 builtin/fsck.c | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
builtin/fsck.c +17 -0
index de34538c4f..ec3357722c 100644
@@ -897,6 +897,21 @@ static int check_pack_rev_indexes(struct repository *r, int show_progress)
 	return res;
 }
 
+static void fsck_refs(void)
+{
+	struct child_process refs_verify = CHILD_PROCESS_INIT;
+	child_process_init(&refs_verify);
+	refs_verify.git_cmd = 1;
+	strvec_pushl(&refs_verify.args, "refs", "verify", NULL);
+	if (verbose)
+		strvec_push(&refs_verify.args, "--verbose");
+	if (check_strict)
+		strvec_push(&refs_verify.args, "--strict");
+
+	if (run_command(&refs_verify))
+		errors_found |= ERROR_REFS;
+}
+
 static char const * const fsck_usage[] = {
 	N_("git fsck [--tags] [--root] [--unreachable] [--cache] [--no-reflogs]\n"
 	   "         [--[no-]full] [--strict] [--verbose] [--lost-found]\n"
@@ -1066,6 +1081,8 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
 
 	check_connectivity();
 
+	fsck_refs();
+
 	if (the_repository->settings.core_commit_graph) {
 		struct child_process commit_graph_verify = CHILD_PROCESS_INIT;
 
Signature · 1 line
2.45.2
author
#116 lore raw

PATCH v8 7/9 files-backend: add unified interface for refs scanning

In reply to shejialuo

For refs and reflogs, we need to scan its corresponding directories to check every regular file or symbolic link which shares the same pattern. Introduce a unified interface for scanning directories for files-backend.

---
 refs/files-backend.c | 77 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 76 insertions(+), 1 deletion(-)
refs/files-backend.c +76 -1
index d89eeda8ef..84acb58782 100644
@@ -6,6 +6,7 @@
 #include "../gettext.h"
 #include "../hash.h"
 #include "../hex.h"
+#include "../fsck.h"
 #include "../refs.h"
 #include "refs-internal.h"
 #include "ref-cache.h"
@@ -3408,6 +3409,78 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,
 	return ret;
 }
 
+/*
+ * For refs and reflogs, they share a unified interface when scanning
+ * the whole directory. This function is used as the callback for each
+ * regular file or symlink in the directory.
+ */
+typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
+				  const char *gitdir,
+				  const char *refs_check_dir,
+				  struct dir_iterator *iter);
+
+static int files_fsck_refs_dir(struct ref_store *ref_store,
+			       struct fsck_options *o,
+			       const char *refs_check_dir,
+			       files_fsck_refs_fn *fsck_refs_fns)
+{
+	const char *gitdir = ref_store->gitdir;
+	struct strbuf sb = STRBUF_INIT;
+	struct dir_iterator *iter;
+	int iter_status;
+	int ret = 0;
+
+	strbuf_addf(&sb, "%s/%s", gitdir, refs_check_dir);
+
+	iter = dir_iterator_begin(sb.buf, 0);
+
+	if (!iter) {
+		ret = error_errno("cannot open directory %s", sb.buf);
+		goto out;
+	}
+
+	while ((iter_status = dir_iterator_advance(iter)) == ITER_OK) {
+		if (S_ISDIR(iter->st.st_mode)) {
+			continue;
+		} else if (S_ISREG(iter->st.st_mode) ||
+			   S_ISLNK(iter->st.st_mode)) {
+			if (o->verbose_refs)
+				fprintf_ln(stderr, "Checking %s/%s",
+					   refs_check_dir, iter->relative_path);
+			for (size_t i = 0; fsck_refs_fns[i]; i++) {
+				if (fsck_refs_fns[i](o, gitdir, refs_check_dir, iter))
+					ret = -1;
+			}
+		} else {
+			ret = error(_("unexpected file type for '%s'"),
+				    iter->basename);
+		}
+	}
+
+	if (iter_status != ITER_DONE)
+		ret = error(_("failed to iterate over '%s'"), sb.buf);
+
+out:
+	strbuf_release(&sb);
+	return ret;
+}
+
+static int files_fsck_refs(struct ref_store *ref_store,
+			   struct fsck_options *o)
+{
+	int ret;
+	files_fsck_refs_fn fsck_refs_fns[]= {
+		NULL
+	};
+
+	if (o->verbose_refs)
+		fprintf_ln(stderr, "Checking references consistency");
+
+	ret = files_fsck_refs_dir(ref_store, o, "refs", fsck_refs_fns);
+
+	return ret;
+}
+
 static int files_fsck(struct ref_store *ref_store,
 		      struct fsck_options *o)
 {
@@ -3415,7 +3488,9 @@ static int files_fsck(struct ref_store *ref_store,
 	struct files_ref_store *refs =
 		files_downcast(ref_store, REF_STORE_READ, "fsck");
 
-	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o);
+	ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o)
+	    | files_fsck_refs(ref_store, o);
+
 	return ret;
 }
 
Signature · 1 line
2.45.2