ref consistency check infra setup
cover ref consistency check infra setup
Hi,
This patch mainly focuses on two things:
1. Setup the infrastructure of the ref consistency check.
2. Implement ref name and content checks for files backend to verify the correctness about the infrastructure with corresponding unit test.
See https://github.com/shejialuo/git/pull/1 for CI.
Thanks,
Jialuo
shejialuo (2):
refs: setup ref consistency check infrastructure
refs: add name and content check for file backend
builtin/fsck.c | 5 ++ refs.c | 5 ++ refs.h | 5 ++ refs/files-backend.c | 119 ++++++++++++++++++++++++++++++++++- refs/packed-backend.c | 7 +++ refs/refs-internal.h | 4 ++ refs/reftable-backend.c | 7 +++ t/t0602-reffiles-fsck.sh | 132 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 283 insertions(+), 1 deletion(-) create mode 100755 t/t0602-reffiles-fsck.sh
Signature · 1 line
2.45.1
PATCH 1/2 refs: setup ref consistency check infrastructure
Add a new interface fsck_refs_fn for the ref_storage_be. Implement dummy method for files and reftable backends.
--- builtin/fsck.c | 5 +++++ refs.c | 5 +++++ refs.h | 5 +++++ refs/files-backend.c | 9 ++++++++- refs/packed-backend.c | 7 +++++++ refs/refs-internal.h | 4 ++++ refs/reftable-backend.c | 7 +++++++ 7 files changed, 41 insertions(+), 1 deletion(-)
builtin/fsck.c +5 -0
@@ -1065,6 +1065,11 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)check_connectivity (); +if (refs_fsck (get_main_ref_store (the_repository))) { +error ("ref database is corrupt" ); + errors_found |=ERROR_REFS ; + } +if (the_repository->settings.core_commit_graph) {struct child_process commit_graph_verify =CHILD_PROCESS_INIT ;
refs.c +5 -0
@@ -316,6 +316,11 @@ int check_refname_format(const char *refname, int flags)return check_or_sanitize_refname (refname, flags,NULL ); } +int refs_fsck (struct ref_store *refs) +{ +return refs->be->fsck (refs); +} +void sanitize_refname_component (const char *refname,struct strbuf *out) {if (check_or_sanitize_refname (refname,REFNAME_ALLOW_ONELEVEL , out))
refs.h +5 -0
@@ -544,6 +544,11 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat*/ int check_refname_format (const char *refname,int flags); +/* +* Return 0 iff all refs in filesystem are consistent. +*/ +int refs_fsck (struct ref_store *refs); +/* * Apply the rules from check_refname_format, but mutate the result until it * is acceptable, and place the result in "out".
refs/files-backend.c +8 -1
@@ -3299,6 +3299,11 @@ static int files_init_db(struct ref_store *ref_store,return 0 ; } +static int files_fsck (struct ref_store *ref_store) +{ +return 0 ; +} +struct ref_storage_be refs_be_files = { .name ="files" , .init = files_ref_store_create,
@@ -3322,5 +3327,7 @@ struct ref_storage_be refs_be_files = { .reflog_exists = files_reflog_exists, .create_reflog = files_create_reflog, .delete_reflog = files_delete_reflog, - .reflog_expire = files_reflog_expire + .reflog_expire = files_reflog_expire, + + .fsck = files_fsck, };
refs/packed-backend.c +7 -0
@@ -1704,6 +1704,11 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_sreturn empty_ref_iterator_begin (); } +static int packed_fsck (struct ref_store *ref_store) +{ +return 0 ; +} +struct ref_storage_be refs_be_packed = { .name ="packed" , .init = packed_ref_store_create,
@@ -1728,4 +1733,6 @@ struct ref_storage_be refs_be_packed = { .create_reflog = NULL, .delete_reflog = NULL, .reflog_expire = NULL, + + .fsck = packed_fsck, };
refs/refs-internal.h +4 -0
@@ -675,6 +675,8 @@ typedef int read_raw_ref_fn(struct ref_store *ref_store, const char *refname,typedef int read_symbolic_ref_fn (struct ref_store *ref_store,const char *refname,struct strbuf *referent); +typedef int fsck_fn (struct ref_store *ref_store); +struct ref_storage_be {const char *name;ref_store_init_fn *init;
@@ -700,6 +702,8 @@ struct ref_storage_be {create_reflog_fn *create_reflog;delete_reflog_fn *delete_reflog;reflog_expire_fn *reflog_expire; + +fsck_fn *fsck; };extern struct ref_storage_be refs_be_files;
refs/reftable-backend.c +7 -0
@@ -2167,6 +2167,11 @@ static int reftable_be_reflog_expire(struct ref_store *ref_store,return ret; } +static int reftable_be_fsck (struct ref_store *ref_store) +{ +return 0 ; +} +struct ref_storage_be refs_be_reftable = { .name ="reftable" , .init = reftable_be_init,
@@ -2191,4 +2196,6 @@ struct ref_storage_be refs_be_reftable = { .create_reflog = reftable_be_create_reflog, .delete_reflog = reftable_be_delete_reflog, .reflog_expire = reftable_be_reflog_expire, + + .fsck = reftable_be_fsck, };
Signature · 1 line
2.45.1
Re: [GSoC][PATCH 1/2] refs: setup ref consistency check infrastructure
Quoted 17 lines · shejialuo
refs/files-backend.c +5
-0@@ -3299,6 +3299,11 @@ static int files_init_db(struct ref_store *ref_store,return 0 ; } +static int files_fsck (struct ref_store *ref_store) +{ +return 0 ; +} +struct ref_storage_be refs_be_files = { .name ="files" , .init = files_ref_store_create,
refs/files-backend.c struct ref_storage_be refs_be_files = { · line 3332
Quoted 7 lines
refs/files-backend.c +3
-1@@ -3322,5 +3327,7 @@ struct ref_storage_be refs_be_files = { .reflog_exists = files_reflog_exists, .create_reflog = files_create_reflog, .delete_reflog = files_delete_reflog, - .reflog_expire = files_reflog_expire + .reflog_expire = files_reflog_expire, + + .fsck = files_fsck,
What is the extra blank line doing there? It makes reader wonder why the .fsck member is somehow very special and different from others. Is there a valid reason to single it out (and no, "yes this is special because I invented it" does not count as a valid reason)?
The same comment applies to a few other places in this patch.
Re: [GSoC][PATCH 1/2] refs: setup ref consistency check infrastructure
Quoted 7 lines · Junio C Hamano
What is the extra blank line doing there? It makes reader wonder why the .fsck member is somehow very special and different from others. Is there a valid reason to single it out (and no, "yes this is special because I invented it" does not count as a valid reason)?
The same comment applies to a few other places in this patch.
The interfaces defined in the ref_storage_be are carefully structured in semantic. It is organized as the five parts at now:
1. The name and the initialization interfaces.
2. The ref transaction interfaces.
3. The ref internal interfaces (pack, rename and copy).
4. The ref filesystem interfaces.
5. The reflog related interfaces.
I firstly thought that we could just add a new function into the ref internal interfaces part with the name check_refs_fn. However, the ultimate goal is not only to achieve consistency check of refs but also consistency check of reflogs. So it's not suitable to name this interface to check_refs_fn.
In order to keep consistent with the git-fsck, we decide to name this interface to fsck_fn. This semantic cannot be grouped into any above five categories in my view. So I deliberately add a blank line to indicate its independence.
Re: [GSoC][PATCH 1/2] refs: setup ref consistency check infrastructure
In reply to shejialuoQuoted 21 lines · shejialuo
Add a new interface
fsck_refs_fnfor theref_storage_be. Implement dummy method for files and reftable backends.Mentored-by Patrick Steinhardt <[email protected]>Mentored-by Karthik Nayak <[email protected]>Signed-off-by shejialuo <[email protected]>--- builtin/fsck.c | 5 +++++ refs.c | 5 +++++ refs.h | 5 +++++ refs/files-backend.c | 9 ++++++++- refs/packed-backend.c | 7 +++++++ refs/refs-internal.h | 4 ++++ refs/reftable-backend.c | 7 +++++++ 7 files changed, 41 insertions(+), 1 deletion(-)builtin/fsck.c +0
-0
builtin/fsck.c int cmd_fsck(int argc, const char **argv, const char *prefix) · line 1071
Quoted 7 lines
builtin/fsck.c +4
-0@@ -1065,6 +1065,11 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)check_connectivity (); +if (refs_fsck (get_main_ref_store (the_repository))) { +error ("ref database is corrupt" ); + errors_found |=ERROR_REFS ; + }
One thing I was wondering is whether we want to make the infrastructure part of the new git-refs(1) command, which is about to land via my series that introduces ref backend migrations. We already make it a habit that we spawn new processes here, as you can see in two lines below for example. It would also make it trivial to execute ref checks standalone, by saying e.g. git refs verify.
Quoted 3 lines
builtin/fsck.c +0
-0if (the_repository->settings.core_commit_graph) {struct child_process commit_graph_verify =CHILD_PROCESS_INIT ;
[snip]
refs.h int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat · line 550
Quoted 12 lines
refs.h +4
-0@@ -544,6 +544,11 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat*/ int check_refname_format (const char *refname,int flags); +/* +* Return 0 iff all refs in filesystem are consistent. +*/ +int refs_fsck (struct ref_store *refs);
We should probably also mention that errors will be written to stderr?
Also, I noticed that you are missing the implementation for the debug backend in "refs/debug.c".
Quoted 8 lines
refs.h +0
-0/* * Apply the rules from check_refname_format, but mutate the result until it * is acceptable, and place the result in "out". refs/files-backend.c +0
-0
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3305
Quoted 7 lines
refs/files-backend.c +4
-0@@ -3299,6 +3299,11 @@ static int files_init_db(struct ref_store *ref_store,return 0 ; } +static int files_fsck (struct ref_store *ref_store) +{ +return 0 ; +}
We can already wire up the call to the embedded packed-refs ref store here. We always want to call that from the files backend.
Patrick
Re: [GSoC][PATCH 1/2] refs: setup ref consistency check infrastructure
In reply to shejialuoHello,
shejialuo <[email protected]> writes:
[snip]
refs.h int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat · line 549
Quoted 11 lines
refs.h +3
-0@@ -544,6 +544,11 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat*/ int check_refname_format (const char *refname,int flags); +/* +* Return 0 iff all refs in filesystem are consistent. +*/
s/iff/if
The indentation seems to be wrong here. Also since we want to check refs and reflogs, it might be better to mention that the function checks the reference database for consistency.
Quoted 100 lines
refs.h +2
-0+int refs_fsck (struct ref_store *refs); +/* * Apply the rules from check_refname_format, but mutate the result until it * is acceptable, and place the result in "out". refs/files-backend.c +8
-1@@ -3299,6 +3299,11 @@ static int files_init_db(struct ref_store *ref_store,return 0 ; } +static int files_fsck (struct ref_store *ref_store) +{ +return 0 ; +} +struct ref_storage_be refs_be_files = { .name ="files" , .init = files_ref_store_create,@@ -3322,5 +3327,7 @@ struct ref_storage_be refs_be_files = { .reflog_exists = files_reflog_exists, .create_reflog = files_create_reflog, .delete_reflog = files_delete_reflog, - .reflog_expire = files_reflog_expire + .reflog_expire = files_reflog_expire, + + .fsck = files_fsck, };refs/packed-backend.c +7
-0@@ -1704,6 +1704,11 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_sreturn empty_ref_iterator_begin (); } +static int packed_fsck (struct ref_store *ref_store) +{ +return 0 ; +} +struct ref_storage_be refs_be_packed = { .name ="packed" , .init = packed_ref_store_create,@@ -1728,4 +1733,6 @@ struct ref_storage_be refs_be_packed = { .create_reflog = NULL, .delete_reflog = NULL, .reflog_expire = NULL, + + .fsck = packed_fsck, };refs/refs-internal.h +4
-0@@ -675,6 +675,8 @@ typedef int read_raw_ref_fn(struct ref_store *ref_store, const char *refname,typedef int read_symbolic_ref_fn (struct ref_store *ref_store,const char *refname,struct strbuf *referent); +typedef int fsck_fn (struct ref_store *ref_store); +struct ref_storage_be {const char *name;ref_store_init_fn *init;@@ -700,6 +702,8 @@ struct ref_storage_be {create_reflog_fn *create_reflog;delete_reflog_fn *delete_reflog;reflog_expire_fn *reflog_expire; + +fsck_fn *fsck; };extern struct ref_storage_be refs_be_files;refs/reftable-backend.c +7
-1@@ -2167,6 +2167,11 @@ static int reftable_be_reflog_expire(struct ref_store *ref_store,return ret; } +static int reftable_be_fsck (struct ref_store *ref_store) +{ +return 0 ; +} +struct ref_storage_be refs_be_reftable = { .name ="reftable" , .init = reftable_be_init,@@ -2191,4 +2196,6 @@ struct ref_storage_be refs_be_reftable = { .create_reflog = reftable_be_create_reflog, .delete_reflog = reftable_be_delete_reflog, .reflog_expire = reftable_be_reflog_expire, + + .fsck = reftable_be_fsck, }; --2.45.1
Re: [GSoC][PATCH 1/2] refs: setup ref consistency check infrastructure
Quoted 6 lines · Karthik Nayak
refs.h int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat · line 549
Quoted 3 lines · shejialuo
refs.h +3
-0+/* +* Return 0 iff all refs in filesystem are consistent. +*/ s/iff/if
I had spotted this, as well, but figured that the use of "iff" was intentional to mean "if and only if".
PATCH 2/2 refs: add name and content check for file backend
In reply to shejialuoThe existing git-fsck does not fully check bad format ref name such as standalone '@' or name ending with ".lock". And also git-fsck does not fully check ref content, the following contents will not be reported by the original git-fsck command.
1. "c71dba5654404b9b0d378a6cbff1b743b9d31a34 garbage": with trailing characters.
2. "c71dba5654404b9b0d378a6cbff1b743b9d31a34 ": with trailing empty
characters.
3. "C71DBA5654404b9b0d378a6cbff1b743b9d31A34": with uppercase hex.
4. "z71dba5654404b9b0d378a6cbff1b743b9d31a34": with not hex character.
Provide functionality to support such consistency checks for branch and tag refs and add a new unit test to verify this functionality.
--- refs/files-backend.c | 112 ++++++++++++++++++++++++++++++++- t/t0602-reffiles-fsck.sh | 132 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 243 insertions(+), 1 deletion(-) create mode 100755 t/t0602-reffiles-fsck.sh
refs/files-backend.c +111 -1
@@ -3299,11 +3299,121 @@ static int files_init_db(struct ref_store *ref_store,return 0 ; } -static int files_fsck (struct ref_store *ref_store) +typedef int (*files_fsck_refs_fn )(const char *refs_check_dir, +struct dir_iterator *iter); + +static int files_fsck_refs_name (const char *refs_check_dir, +struct dir_iterator *iter) { +if (check_refname_format (iter->basename,REFNAME_ALLOW_ONELEVEL )) { +error (_ ("%s/%s: invalid refname format" ), refs_check_dir, iter->basename); +return -1 ; + } +return 0 ; } +static int files_fsck_refs_content (const char *refs_check_dir, +struct dir_iterator *iter) +{ +struct strbuf ref_content =STRBUF_INIT ; + +if (strbuf_read_file (&ref_content, iter->path.buf,0 ) <0 ) { +error (_ ("%s/%s: unable to read the ref" ), refs_check_dir, iter->basename); + goto clean; + } + +/* +* Case 1: check if the ref content length is valid and the last +* character is a newline. +*/ +if (ref_content.len != the_hash_algo->hexsz +1 || + ref_content.buf[ref_content.len -1 ] !='\n' ) { + goto failure; + } + +/* +* Case 2: the content should be range of [0-9a-f]. +*/ +for (size_t i =0 ; i < the_hash_algo->hexsz; i++) { +if (!isdigit (ref_content.buf[i]) && + (ref_content.buf[i] <'a' || ref_content.buf[i] >'f' )) { + goto failure; + } + } + +strbuf_release (&ref_content); +return 0 ; + +failure: +error (_ ("%s/%s: invalid ref content" ), refs_check_dir, iter->basename); + +clean: +strbuf_release (&ref_content); +return -1 ; +} + +static int files_fsck_refs (struct ref_store *ref_store, +const char * refs_check_dir, +files_fsck_refs_fn *fsck_refs_fns) +{ +struct dir_iterator *iter; +struct strbuf sb =STRBUF_INIT ; +int ret =0 ; +int iter_status; + +strbuf_addf (&sb,"%s/%s" , ref_store->gitdir, refs_check_dir); + + iter =dir_iterator_begin (sb.buf,0 ); + +/* +* The current implementation does not care about the worktree, the worktree +* may have no refs/heads or refs/tags directory. Simply return 0 now. +*/ +if (!iter) { +return 0 ; + } + +while ((iter_status =dir_iterator_advance (iter)) ==ITER_OK ) { +if (S_ISDIR (iter->st.st_mode)) { +continue ; + }else if (S_ISREG (iter->st.st_mode)) { +for (files_fsck_refs_fn *fsck_refs_fn = fsck_refs_fns; + *fsck_refs_fn; fsck_refs_fn++) { + ret |= (*fsck_refs_fn)(refs_check_dir, iter); + } + }else { +error (_ ("unexpected file type for '%s'" ), iter->basename); + ret =-1 ; + } + } + +if (iter_status !=ITER_DONE ) { + ret =-1 ; +error (_ ("failed to iterate over '%s'" ), sb.buf); + } + +strbuf_release (&sb); + +return ret; +} + +static int files_fsck (struct ref_store *ref_store) +{ +int ret =0 ; + +files_fsck_refs_fn fsck_refs_fns[] = { + files_fsck_refs_name, + files_fsck_refs_content, +NULL + }; + + ret =files_fsck_refs (ref_store,"refs/heads" ,fsck_refs_fns) + |files_fsck_refs (ref_store,"refs/tags" , fsck_refs_fns); + +return ret; +} +struct ref_storage_be refs_be_files = { .name ="files" , .init = files_ref_store_create,
t/t0602-reffiles-fsck.sh new file +132 -0
132 lines not on this page · raw message
Signature · 1 line
2.45.1
Re: [GSoC][PATCH 2/2] refs: add name and content check for file backend
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3319
Quoted 4 lines · shejialuo
refs/files-backend.c +4
-0+static int files_fsck_refs_content (const char *refs_check_dir, +struct dir_iterator *iter) +{ +struct strbuf ref_content =STRBUF_INIT ;
The caller makes sure that this gets called only on a regular file, so ...
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3321
Quoted 1 line
refs/files-backend.c +1
-0+if (strbuf_read_file (&ref_content, iter->path.buf,0 ) <0 ) {
... the use of strbuf_read_file() is fine (i.e. we do not have to worry about the iter->path to be pointing at a symbolic link) here.
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3331
Quoted 10 lines
refs/files-backend.c +10
-0+error (_ ("%s/%s: unable to read the ref" ), refs_check_dir, iter->basename); + goto clean; + } + +/* +* Case 1: check if the ref content length is valid and the last +* character is a newline. +*/ +if (ref_content.len != the_hash_algo->hexsz +1 || + ref_content.buf[ref_content.len -1 ] !='\n' ) {
Funny indentation.
if (ref_content.len != the_hash_algo->hexsz + 1 ||
ref_content.buf[ref_content.len - 1] != '\n') {Also, we do not want {braces} around a single statement block.
In any case, the two checks are good ONLY for regular refs and not for symbolic refs. The users are free to create symbolic refs next to their branches, e.g. here is a way to say, "among the maintenance tracks maint-2.30, maint-2.31, ... maint-2.44, maint-2.45, what I consider the primary maintenance track is currently maint-2.45":
$ git symbolic-ref refs/heads/maint
refs/heads/maint-2.45and if your directory walk encounters such a symbolic ref, your strbuf_read_file() will read something like:
$ cat .git/refs/heads/maint
ref: refs/heads/maint-2.45Also, the caller also needs to be prepared to find a real symbolic link that is used as a symbolic ref, but for them you'd need to do a readlink() and the expected contents would not have "ref: " prefix, so the code to implement actual check would have to be different. The way how refs/files-backend.c:read_ref_internal() handles S_ISLNK would be illuminating.
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3341
Quoted 9 lines
refs/files-backend.c +9
-0+ goto failure; + } +/* +* Case 2: the content should be range of [0-9a-f]. +*/ +for (size_t i =0 ; i < the_hash_algo->hexsz; i++) { +if (!isdigit (ref_content.buf[i]) && + (ref_content.buf[i] <'a' || ref_content.buf[i] >'f' )) { + goto failure;
I do not think it is a good idea to suddenly redefine what a valid way to write object names in a loose ref file after ~20 years. Given the popularity of Git, I would not be surprised at all if a third-party tool or two have been writing their own refs with uppercase hexadecimal and we have been happily using them as everybody expects. It is a good idea to be strict when you are a producer, and we do write object names always in lowercase hex, but we are lenient when consuming what is possibly written by others. As long as the hexadecimal string names an existing object (otherwise we have a dangling ref which would be another kind of error, I presume), it should be at most FSCK_WARN when it does not look like what _we_ wrote (e.g., if it uses uppercase hex, or if it has extra bytes after the 40-hex (or 64-hex) other than the final LF). If it is shorter, of course that is an outright FSCK_ERROR.
How well does this interact with the fsck error levels (aka fsck_msg_type), by the way? It should be made to work well if the current design does not.
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3349
Quoted 8 lines
refs/files-backend.c +8
-0+ } + } + +strbuf_release (&ref_content); +return 0 ; + +failure: +error (_ ("%s/%s: invalid ref content" ), refs_check_dir, iter->basename);
In addition, when we honor fsck error levels, such an unconditional call to "error" may become an issue. If the user says a certain kind of anomaly is tolerated by setting a specific finding to FSCK_IGNORE, we should be silent about our finding.
Quoted 22 lines
refs/files-backend.c +22
-0+clean: +strbuf_release (&ref_content); +return -1 ; +} + +static int files_fsck_refs (struct ref_store *ref_store, +const char * refs_check_dir, +files_fsck_refs_fn *fsck_refs_fns) +{ +struct dir_iterator *iter; +struct strbuf sb =STRBUF_INIT ; +int ret =0 ; +int iter_status; + +strbuf_addf (&sb,"%s/%s" , ref_store->gitdir, refs_check_dir); + + iter =dir_iterator_begin (sb.buf,0 ); + +/* +* The current implementation does not care about the worktree, the worktree +* may have no refs/heads or refs/tags directory. Simply return 0 now. +*/
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3382
Quoted 10 lines
refs/files-backend.c +10
-0+if (!iter) { +return 0 ; + } + +while ((iter_status =dir_iterator_advance (iter)) ==ITER_OK ) { +if (S_ISDIR (iter->st.st_mode)) { +continue ; + }else if (S_ISREG (iter->st.st_mode)) { +for (files_fsck_refs_fn *fsck_refs_fn = fsck_refs_fns; + *fsck_refs_fn; fsck_refs_fn++) {
Funny indentation (again---the patch 2/2 has funny "two extra HT" indentation style all over the place). Write it like so:
for (files_fsck_refs_fn *fn = fsck_refs_fns;
fn;
fn++)refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3383
Quoted 1 line
refs/files-backend.c +1
-0+ ret |= (*fsck_refs_fn)(refs_check_dir, iter);
A pointer to a function does not need to be dereferenced explicitly like so. Also, writing
if (fn(refs_check_dir, iter)) ret = -1;
would be more consistent with the rest of the code around here, which does not OR int ret but explicitly set it to -1 when any helper function detects an error.
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3387
Quoted 4 lines
refs/files-backend.c +4
-0+ } + }else { +error (_ ("unexpected file type for '%s'" ), iter->basename); + ret =-1 ;
This is wrong. A symbolic link is a valid symbolic ref, even though we no longer create such a symbolic ref by default.
Quoted 14 lines
refs/files-backend.c +14
-0+ } + } + +if (iter_status !=ITER_DONE ) { + ret =-1 ; +error (_ ("failed to iterate over '%s'" ), sb.buf); + } + +strbuf_release (&sb); + +return ret; +} + +static int files_fsck (struct ref_store *ref_store)
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3411
Quoted 10 lines
refs/files-backend.c +10
-0+{ +int ret =0 ; + +files_fsck_refs_fn fsck_refs_fns[] = { + files_fsck_refs_name, + files_fsck_refs_content, +NULL + }; + + ret =files_fsck_refs (ref_store,"refs/heads" ,fsck_refs_fns)
Missing SP after a comma.
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3412
Quoted 1 line
refs/files-backend.c +1
-0+ | files_fsck_refs(ref_store, "refs/tags", fsck_refs_fns);
Why only these two hierarchies? Shouldn't it also be checking the remote tracking branches and notes?
Quoted 6 lines
refs/files-backend.c +3
-0+return ret; +} +struct ref_storage_be refs_be_files = { .name ="files" , .init = files_ref_store_create,
Re: [GSoC][PATCH 2/2] refs: add name and content check for file backend
Quoted 28 lines · Junio C Hamano
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3319
Quoted 4 lines · shejialuo
refs/files-backend.c +4
-0+static int files_fsck_refs_content (const char *refs_check_dir, +struct dir_iterator *iter) +{ +struct strbuf ref_content =STRBUF_INIT ;The caller makes sure that this gets called only on a regular file, so ...
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3321
Quoted 1 line
refs/files-backend.c +1
-0+if (strbuf_read_file (&ref_content, iter->path.buf,0 ) <0 ) {... the use of
strbuf_read_file()is fine (i.e. we do not have to worry about theiter->pathto be pointing at a symbolic link) here.refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3331
Quoted 10 lines
refs/files-backend.c +10
-0+error (_ ("%s/%s: unable to read the ref" ), refs_check_dir, iter->basename); + goto clean; + } + +/* +* Case 1: check if the ref content length is valid and the last +* character is a newline. +*/ +if (ref_content.len != the_hash_algo->hexsz +1 || + ref_content.buf[ref_content.len -1 ] !='\n' ) {Funny indentation.
Quoted 5 lines
if (ref_content.len != the_hash_algo->hexsz + 1 || ref_content.buf[ref_content.len - 1] != '\n') {Also, we do not want {braces} around a single statement block.
I will clean the code. I am used to applying one extra HT.
Quoted 6 lines
In any case, the two checks are good ONLY for regular refs and not for symbolic refs. The users are free to create symbolic refs next to their branches, e.g. here is a way to say, "among the maintenance tracks maint-2.30, maint-2.31, ... maint-2.44, maint-2.45, what I consider the primary maintenance track is currently maint-2.45":
Quoted 9 lines
$ git symbolic-ref refs/heads/maint refs/heads/maint-2.45and if your directory walk encounters such a symbolic ref, your
strbuf_read_file()will read something like:$ cat .git/refs/heads/maint ref: refs/heads/maint-2.45
Yes, currently I intentionally ignore the symbolic ref. My original idea was to make this patch as small as possible to set up the infrastructure of the ref consistence check.
So this is why I only choose "refs/heads" and "refs/tags" at current. The "refs/remote" will contain "HEAD" which is a symbolic ref. So I ignore "refs/remote".
I just thought git would ONLY have one symbolic ref "HEAD". I will add more patches in the next version to support the symbolic ref checks.
Quoted 7 lines
Also, the caller also needs to be prepared to find a real symbolic link that is used as a symbolic ref, but for them you'd need to do a
readlink()and the expected contents would not have "ref: " prefix, so the code to implement actual check would have to be different. The way how refs/files-backend.c:read_ref_internal() handlesS_ISLNKwould be illuminating.
I will handle this in the next version. However, I am a little curious about the reason why people use a real symbolic in git. git has provided the symbolic ref which should be more consistent and generic against the OS.
Quoted 16 lines
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3341
Quoted 9 lines
refs/files-backend.c +9
-0+ goto failure; + } +/* +* Case 2: the content should be range of [0-9a-f]. +*/ +for (size_t i =0 ; i < the_hash_algo->hexsz; i++) { +if (!isdigit (ref_content.buf[i]) && + (ref_content.buf[i] <'a' || ref_content.buf[i] >'f' )) { + goto failure;I do not think it is a good idea to suddenly redefine what a valid way to write object names in a loose ref file after ~20 years. Given the popularity of Git, I would not be surprised at all if a third-party tool or two have been writing their own refs with uppercase hexadecimal and we have been happily using them as everybody expects. It is a good idea to be strict when you are a
Quoted 9 lines
producer, and we do write object names always in lowercase hex, but we are lenient when consuming what is possibly written by others. As long as the hexadecimal string names an existing object (otherwise we have a dangling ref which would be another kind of error, I presume), it should be at most
FSCK_WARNwhen it does not look like what _we_ wrote (e.g., if it uses uppercase hex, or if it has extra bytes after the 40-hex (or 64-hex) other than the final LF). If it is shorter, of course that is an outrightFSCK_ERROR.
Yes, actually I have done some experiments using git-fsck. I am really surprised by the fact that the few bytes will cause FSCK_ERROR. But the situation with extra bytes are OK. I don't know this design is intentional.
Quoted 4 lines
How well does this interact with the fsck error levels (aka
fsck_msg_type), by the way? It should be made to work well if the current design does not.
Unfortunately, I totally ignored this. I will try to implement the fsck error levels in the next version.
Quoted 5 lines
Quoted 5 lines
refs/files-backend.c +5
-0+ } + } + +strbuf_release (&ref_content); +return 0 ;
Quoted 9 lines
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3349
Quoted 3 lines
refs/files-backend.c +3
-0+ +failure: +error (_ ("%s/%s: invalid ref content" ), refs_check_dir, iter->basename);In addition, when we honor fsck error levels, such an unconditional call to "error" may become an issue. If the user says a certain kind of anomaly is tolerated by setting a specific finding to
FSCK_IGNORE, we should be silent about our finding.
Yes, I will add the code to support this.
Quoted 45 lines
Quoted 22 lines
refs/files-backend.c +22
-0+clean: +strbuf_release (&ref_content); +return -1 ; +} + +static int files_fsck_refs (struct ref_store *ref_store, +const char * refs_check_dir, +files_fsck_refs_fn *fsck_refs_fns) +{ +struct dir_iterator *iter; +struct strbuf sb =STRBUF_INIT ; +int ret =0 ; +int iter_status; + +strbuf_addf (&sb,"%s/%s" , ref_store->gitdir, refs_check_dir); + + iter =dir_iterator_begin (sb.buf,0 ); + +/* +* The current implementation does not care about the worktree, the worktree +* may have no refs/heads or refs/tags directory. Simply return 0 now. +*/ refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3382
Quoted 10 lines
refs/files-backend.c +10
-0+if (!iter) { +return 0 ; + } + +while ((iter_status =dir_iterator_advance (iter)) ==ITER_OK ) { +if (S_ISDIR (iter->st.st_mode)) { +continue ; + }else if (S_ISREG (iter->st.st_mode)) { +for (files_fsck_refs_fn *fsck_refs_fn = fsck_refs_fns; + *fsck_refs_fn; fsck_refs_fn++) {Funny indentation (again---the patch 2/2 has funny "two extra HT" indentation style all over the place). Write it like so:
for (files_fsck_refs_fn *fn = fsck_refs_fns; fn; fn++)refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3383
Quoted 1 line
refs/files-backend.c +1
-0+ ret |= (*fsck_refs_fn)(refs_check_dir, iter);A pointer to a function does not need to be dereferenced explicitly like so. Also, writing
Quoted 7 lines
if (fn(refs_check_dir, iter)) ret = -1;would be more consistent with the rest of the code around here, which does not OR int ret but explicitly set it to -1 when any helper function detects an error.
I will handle this in the next version.
Quoted 8 lines
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3387
Quoted 4 lines
refs/files-backend.c +4
-0+ } + }else { +error (_ ("unexpected file type for '%s'" ), iter->basename); + ret =-1 ;This is wrong. A symbolic link is a valid symbolic ref, even though we no longer create such a symbolic ref by default.
As above said, I have thought git should not handle this situation. I will implement support for real symbolic file.
Quoted 18 lines
Quoted 18 lines
refs/files-backend.c +18
-0+ } + } + +if (iter_status !=ITER_DONE ) { + ret =-1 ; +error (_ ("failed to iterate over '%s'" ), sb.buf); + } + +strbuf_release (&sb); + +return ret; +} + +static int files_fsck (struct ref_store *ref_store) +{ +int ret =0 ; + +files_fsck_refs_fn fsck_refs_fns[] = {
Quoted 9 lines
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3411
Quoted 6 lines
refs/files-backend.c +6
-0+ files_fsck_refs_name, + files_fsck_refs_content, + NULL + }; + + ret =files_fsck_refs (ref_store,"refs/heads" ,fsck_refs_fns)Missing SP after a comma.
I will fix it in the next version.
Quoted 5 lines
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3412
Quoted 1 line
refs/files-backend.c +1
-0+ | files_fsck_refs(ref_store, "refs/tags", fsck_refs_fns);Why only these two hierarchies? Shouldn't it also be checking the remote tracking branches and notes?
I have explained above, because I intentionally ignored the symbolic ref in this patch, so I don't handle "refs/remote" and notes. I guess I will handle these situations in the next patch.
Quoted 6 lines
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3419
Quoted 6 lines
refs/files-backend.c +3
-0+return ret; +} +struct ref_storage_be refs_be_files = { .name ="files" , .init = files_ref_store_create,
Thanks for your dedicated reviews. I have a more clean direction to setup the consistence check for file backend.
Thanks,
Jialuo
Re: [GSoC][PATCH 2/2] refs: add name and content check for file backend
In reply to Junio C HamanoQuoted 6 lines · Junio C Hamano
Review
Quoted 2 lines · shejialuo
(quoted diff) +2
-0+ error(_("unexpected file type for '%s'"), iter->basename); + ret = -1;This is wrong. A symbolic link is a valid symbolic ref, even though we no longer create such a symbolic ref by default.
Very minor additional note: error() unconditionally returns -1 in order to support this pattern:
ret = error(_("..."));Re: [GSoC][PATCH 2/2] refs: add name and content check for file backend
In reply to shejialuoQuoted 12 lines · shejialuo
The existing git-fsck does not fully check bad format ref name such as standalone '@' or name ending with ".lock". And also
git-fsckdoes not fully check ref content, the following contents will not be reported by the original git-fsck command.1. "
c71dba5654404b9b0d378a6cbff1b743b9d31a34garbage": with trailing characters.
2. "c71dba5654404b9b0d378a6cbff1b743b9d31a34": with trailing empty
characters.
3. "C71DBA5654404b9b0d378a6cbff1b743b9d31A34": with uppercase hex.
4. "z71dba5654404b9b0d378a6cbff1b743b9d31a34": with not hex character.
Quoted 2 lines
Provide functionality to support such consistency checks for branch and tag refs and add a new unit test to verify this functionality.
I would recommend to only introduce one check per commit. This commit introduces multiple different checks to the files backend, which makes it harder to review.
[snip]
Quoted 28 lines
refs/files-backend.c +28
-0+static int files_fsck_refs_content (const char *refs_check_dir, +struct dir_iterator *iter) +{ +struct strbuf ref_content =STRBUF_INIT ; + +if (strbuf_read_file (&ref_content, iter->path.buf,0 ) <0 ) { +error (_ ("%s/%s: unable to read the ref" ), refs_check_dir, iter->basename); + goto clean; + } + +/* +* Case 1: check if the ref content length is valid and the last +* character is a newline. +*/ +if (ref_content.len != the_hash_algo->hexsz +1 || + ref_content.buf[ref_content.len -1 ] !='\n' ) { + goto failure; + } + +/* +* Case 2: the content should be range of [0-9a-f]. +*/ +for (size_t i =0 ; i < the_hash_algo->hexsz; i++) { +if (!isdigit (ref_content.buf[i]) && + (ref_content.buf[i] <'a' || ref_content.buf[i] >'f' )) { + goto failure; + } + }
refs/files-backend.c static int files_init_db(struct ref_store *ref_store, · line 3353
Quoted 10 lines
refs/files-backend.c +10
-0+ +strbuf_release (&ref_content); +return 0 ; + +failure: +error (_ ("%s/%s: invalid ref content" ), refs_check_dir, iter->basename); + +clean: +strbuf_release (&ref_content); +return -1 ;
Can we reuse parse_loose_ref_contents() to get rid of much of the duplication here? That function would need to learn to indicate to the caller that there is trailing data, but other than that it's already as strict as we want it to be.
Other than that, Junio has already added quite a bunch of comments, so I'll refrain from repeating them here.
Thanks!
Patrick
v2 cover ref consistency check infra setup
In reply to shejialuoThis patch sets up the ref consistency check infrastructure. We need to integrate the fsck error messages. However, the git-fsck(1) is highly coupled with the object database. And it uses git_fsck_config to initialize the fsck_options->fsck_msg_type. For my implementation, I do not add some new files into fsck_options due to the following reasons:
1. The fileds in fsck_options are all related to objects. We only can reuse the fsck_msg_type and strict member, adding some ref-related fileds into fsck_options will break the semantics.
2. We may laterc hange fsck_options and fsck_objs_options.
The code for integrating fsck error messages for refs may be a liitle redudant now, but if we change the fsck_options, it will be way complicated.
After integrating the fsck error messages, setup the consistency check infrastructure like patch v1. And introduce a new subcommand for builtin git-refs and in git-fsck(1), use a child process to call this subcommand.
At last, like patch v1, implement the consistency check for files backend and corresponding unit tests.
Because I have added more commits compare to v1. So I provide the innerdiff between the v1 and v2.
Thanks,
Jialuo
shejialuo (7):
fsck: add refs check interfaces to interface with fsck error levels
refs: set up ref consistency check infrastructure builtin/refs: add verify subcommand
builtin/fsck: add git-refs verify child process
files-backend: add unified interface for refs scanning
fsck: add ref name check for files backend
fsck: add ref content check for files backend
Documentation/fsck-msgids.txt | 12 ++ Documentation/git-refs.txt | 11 ++ builtin/fsck.c | 17 +++ builtin/refs.c | 45 +++++++ fsck.c | 69 ++++++++++ fsck.h | 43 ++++++- refs.c | 7 +- refs.h | 8 ++ refs/debug.c | 9 ++ refs/files-backend.c | 234 +++++++++++++++++++++++++++++++++- refs/packed-backend.c | 8 ++ refs/refs-internal.h | 11 +- refs/reftable-backend.c | 8 ++ t/t0602-reffiles-fsck.sh | 211 ++++++++++++++++++++++++++++++ 14 files changed, 685 insertions(+), 8 deletions(-) create mode 100755 t/t0602-reffiles-fsck.sh
Interdiff between patch v1 and patch v2:
Documentation/fsck-msgids.txt +12 -1
@@ -19,6 +19,12 @@ `badParentSha1`:: (ERROR) A commit object has a bad parent sha1. +`badRefName`:: + (ERROR) A ref has a bad name. + +`badRefSha`:: + (ERROR) A ref has a bad sha. + `badTagName`:: (INFO) A tag has an invalid format.
@@ -34,6 +40,9 @@ `badType`:: (ERROR) Found an invalid object type. +`danglingSymref`:: + (WARN) Found a dangling symref. + `duplicateEntries`:: (ERROR) A tree contains duplicate file entries.
@@ -176,6 +185,9 @@ `symlinkTargetMissing`:: (ERROR) Unable to read symbolic link target's blob. +`trailingRefContent`:: + (WARN) A ref content has trailing contents. + `treeNotSorted`:: (ERROR) A tree is not properly sorted. -
Documentation/git-refs.txt +11 -0
@@ -10,6 +10,7 @@ SYNOPSIS -------- [verse] 'git refs migrate' --ref-format=<format> [--dry-run] +'git refs verify' [--strict] [--verbose] DESCRIPTION -----------
@@ -22,6 +23,9 @@ COMMANDS migrate:: Migrate ref store between different formats. +verify:: + Verify reference database consistency. + OPTIONS -------
@@ -39,6 +43,13 @@ include::ref-storage-format.txt[] can be used to double check that the migration works as expected before performing the actual migration. +--strict:: + Enable more strict checking, every WARN severity for the `Fsck Messages` + be seen as ERROR. + +--verbose:: + When verifying the reference database consistency, be chatty. + KNOWN LIMITATIONS -----------------
builtin/fsck.c +16 -4
@@ -896,6 +896,21 @@ static int check_pack_rev_indexes(struct repository *r, int show_progress)return res; } +static void fsck_refs (void ) +{ +struct child_process refs_verify =CHILD_PROCESS_INIT ; +child_process_init (&refs_verify); + refs_verify.git_cmd =1 ; +strvec_pushl (&refs_verify.args,"refs" ,"verify" ,NULL ); +if (verbose) +strvec_push (&refs_verify.args,"--verbose" ); +if (check_strict) +strvec_push (&refs_verify.args,"--strict" ); + +if (run_command (&refs_verify)) + errors_found |=ERROR_REFS ; +} +static char const *const fsck_usage[] = { N_("git fsck [--tags] [--root] [--unreachable] [--cache] [--no-reflogs]\n" " [--[no-]full] [--strict] [--verbose] [--lost-found]\n"
@@ -1065,10 +1080,7 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)check_connectivity (); -if (refs_fsck (get_main_ref_store (the_repository))) { -error ("ref database is corrupt" ); - errors_found |=ERROR_REFS ; - } +fsck_refs ();if (the_repository->settings.core_commit_graph) {struct child_process commit_graph_verify =CHILD_PROCESS_INIT ;
-
builtin/refs.c +45 -0
70 lines not on this page · raw message
fsck.c +69 -0
81 lines not on this page · raw message
fsck.h +41 -2
93 lines not on this page · raw message
refs.c +3 -3
@@ -316,9 +316,9 @@ int check_refname_format(const char *refname, int flags)return check_or_sanitize_refname (refname, flags,NULL ); } -int refs_fsck (struct ref_store *refs) +int refs_fsck (struct ref_store *refs,struct fsck_refs_options *o) { -return refs->be->fsck (refs); +return refs->be->fsck (refs, o); }void sanitize_refname_component (const char *refname,struct strbuf *out)
@@ -1744,7 +1744,7 @@ static int refs_read_special_head(struct ref_store *ref_store, } result =parse_loose_ref_contents (content.buf, oid, referent, type, - failure_errno); + failure_errno,NULL ); done:strbuf_release (&full_path);
refs.h +6 -3
@@ -3,6 +3,7 @@#include "commit.h" +struct fsck_refs_options ;struct object_id ;struct ref_store ;struct repository ;
@@ -548,9 +549,11 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_datint check_refname_format (const char *refname,int flags);/* -* Return 0 iff all refs in filesystem are consistent. -*/ -int refs_fsck (struct ref_store *refs); +* Check the reference database for consistency. Return 0 if refs and +* reflogs are consistent, and non-zero otherwise. The errors will be +* written to stderr. +*/ +int refs_fsck (struct ref_store *refs,struct fsck_refs_options *o);/* * Apply the rules from check_refname_format, but mutate the result until it
refs/debug.c +9 -0
@@ -419,6 +419,13 @@ static int debug_reflog_expire(struct ref_store *ref_store, const char *refname,return res; } +static int debug_fsck (struct ref_store *ref_store, +struct fsck_refs_options *o) +{ +trace_printf_key (&trace_refs,"fsck\n" ); +return 0 ; +} +struct ref_storage_be refs_be_debug = { .name ="debug" , .init =NULL ,
@@ -451,4 +458,6 @@ struct ref_storage_be refs_be_debug = { .create_reflog = debug_create_reflog, .delete_reflog = debug_delete_reflog, .reflog_expire = debug_reflog_expire, + + .fsck = debug_fsck, };
refs/files-backend.c +172 -63
325 lines not on this page · raw message
refs/packed-backend.c +2 -1
@@ -1733,7 +1733,8 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_sreturn empty_ref_iterator_begin (); } -static int packed_fsck (struct ref_store *ref_store) +static int packed_fsck (struct ref_store *ref_store, +struct fsck_refs_options *o) {return 0 ; }
refs/refs-internal.h +6 -3
@@ -4,6 +4,7 @@#include "refs.h" #include "iterator.h" +struct fsck_refs_options ;struct ref_transaction ;/*
@@ -650,7 +651,8 @@ typedef int read_raw_ref_fn(struct ref_store *ref_store, const char *refname,typedef int read_symbolic_ref_fn (struct ref_store *ref_store,const char *refname,struct strbuf *referent); -typedef int fsck_fn (struct ref_store *ref_store); +typedef int fsck_fn (struct ref_store *ref_store, +struct fsck_refs_options *o);struct ref_storage_be {const char *name;
@@ -707,11 +709,12 @@ struct ref_store {/* * Parse contents of a loose ref file. *failure_errno maybe be set to EINVAL for -* invalid contents. +* invalid contents. Also trailing is set to 1 when there is any bytes after the +* hex. */ int parse_loose_ref_contents (const char *buf,struct object_id *oid,struct strbuf *referent,unsigned int *type, -int *failure_errno); +int *failure_errno,unsigned int *trailing);/* * Fill in the generic part of refs and add it to our collection of
refs/reftable-backend.c +2 -1
@@ -2242,7 +2242,8 @@ static int reftable_be_reflog_expire(struct ref_store *ref_store,return ret; } -static int reftable_be_fsck (struct ref_store *ref_store) +static int reftable_be_fsck (struct ref_store *ref_store, +struct fsck_refs_options *o) {return 0 ; }
t/t0602-reffiles-fsck.sh +107 -28
206 lines not on this page · raw message
t/t0610-reftable-basics.sh +0 -1
@@ -10,7 +10,6 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME GIT_TEST_DEFAULT_REF_FORMAT=reftable export GIT_TEST_DEFAULT_REF_FORMAT -TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh INVALID_OID=$(test_oid 001)
t/t1300-config.sh +1 -8
@@ -596,8 +596,7 @@ test_expect_success 'get bool variable with empty value' ' test_expect_success 'no arguments, but no crash' ' test_must_fail git config >output 2>&1 && - echo "error: no action specified" >expect && - test_cmp expect output + test_grep usage output ' cat > .git/config << EOF
@@ -2835,12 +2834,6 @@ test_expect_success 'specifying multiple modes causes failure' ' test_cmp expect err ' -test_expect_success 'writing to stdin is rejected' ' - echo "fatal: writing to stdin is not supported" >expect && - test_must_fail git config ${mode_set} --file - foo.bar baz 2>err && - test_cmp expect err -' - done test_done
Signature · 1 line
2.45.2
PATCH v2 1/7 fsck: add refs check interfaces to interface with fsck error levels
The git-fsck(1) focuses on object database consistency check. It relies on the "fsck_options" to interact with fsck error levels. However, "fsck_options" aims at checking the object database which makes it unsuitable to change the semantics of it. Instead, create "fsck_refs_options" structure to handle refs consistency check.
The "git_fsck_config" sets up the "msg_type" and "skiplist" member of the "fsck_options". For refs, we just need the "msg_type". In order to allow setting up more refs-specific options easily later, add a separate function "git_fsck_refs_config" to initialize the refs-specific options.
Move the "msg_type" and "strict" member to the top of the "fsck_options" which allows us to convert "fsck_refs_options *" to "fsck_options *" to reuse the interfaces provided by "fsck.h" without changing the original code.
The static function "report" provided by "fsck.c" aims at reporting the problems related to object database which cannot be reused for refs. Provide "fsck_refs_report" function to integrate the fsck error levels into reference consistency check.
--- fsck.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ fsck.h | 39 +++++++++++++++++++++++++++++++-- 2 files changed, 106 insertions(+), 2 deletions(-)
fsck.c +69 -0
@@ -1249,6 +1249,20 @@ int fsck_buffer(const struct object_id *oid, enum object_type type, type); } +int fsck_refs_error_function(struct fsck_refs_options *o UNUSED, +const char *name, +enum fsck_msg_type msg_type, + enum fsck_msg_id msg_id UNUSED, +const char *message) +{ +if (msg_type ==FSCK_WARN ) { +warning ("%s: %s" , name, message); +return 0 ; + } +error ("%s: %s" , name, message); +return 1 ; +} + int fsck_error_function(struct fsck_options *o, const struct object_id *oid, enum object_type object_type UNUSED,
@@ -1323,6 +1337,61 @@ int fsck_finish(struct fsck_options *options)return ret; } +int fsck_refs_report (struct fsck_refs_options *o, +const char *name, +enum fsck_msg_id msg_id, +const char *fmt, ...) +{ +va_list ap; +struct strbuf sb =STRBUF_INIT ; +enum fsck_msg_type msg_type = +fsck_msg_type (msg_id, (struct fsck_options *)o); +int ret =0 ; + +if (msg_type ==FSCK_IGNORE ) +return 0 ; + +if (msg_type ==FSCK_FATAL ) + msg_type =FSCK_ERROR ; +else if (msg_type ==FSCK_INFO ) + msg_type =FSCK_WARN ; + +prepare_msg_ids (); +strbuf_addf (&sb,"%s: " , msg_id_info[msg_id].camelcased); + +va_start (ap, fmt); +strbuf_vaddf (&sb, fmt, ap); + ret = o->error_func (o, name, msg_type, msg_id, sb.buf); +strbuf_release (&sb); +va_end (ap); + +return ret; +} + +int git_fsck_refs_config (const char *var,const char *value, +const struct config_context *ctx,void *cb) +{ +struct fsck_refs_options *options = cb; +const char *msg_id; + +/* +* We don't check the value of fsck.skiplist here, because it +* is specific to object database, not reference database. +*/ +if (strcmp (var,"fsck.skiplist" ) ==0 ) { +return 0 ; + } + +if (skip_prefix (var,"fsck." , &msg_id)) { +if (!value) +return config_error_nonbool (var); +fsck_set_msg_type ((struct fsck_options *)options, msg_id, value); +return 0 ; + } + +return git_default_config (var, value, ctx, cb); +} +int git_fsck_config (const char *var,const char *value,const struct config_context *ctx,void *cb) {
fsck.h +37 -2
@@ -96,6 +96,7 @@ enum fsck_msg_id { };#undef MSG_ID +struct fsck_refs_options ;struct fsck_options ;struct object ;
@@ -107,6 +108,21 @@ void fsck_set_msg_type(struct fsck_options *options,void fsck_set_msg_types (struct fsck_options *options,const char *values);int is_valid_msg_type (const char *msg_id,const char *msg_type); +/* +* callback function for fsck refs and reflogs. +*/ +typedef int (*fsck_refs_error )(struct fsck_refs_options *o, +const char *name, +enum fsck_msg_type msg_type, +enum fsck_msg_id msg_id, +const char *message); + +int fsck_refs_error_function (struct fsck_refs_options *o, +const char *name, +enum fsck_msg_type msg_type, +enum fsck_msg_id msg_id, +const char *message); +/* * callback function for fsck_walk * type is the expected type of the object or OBJ_ANY
@@ -135,11 +151,22 @@ int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o,enum fsck_msg_id msg_id, const char *message); +struct fsck_refs_options { +enum fsck_msg_type *msg_type; +unsigned strict:1 ; +fsck_refs_error error_func; +unsigned verbose:1 ; +}; + +#define FSCK_REFS_OPTIONS_DEFAULT { \ + .error_func = fsck_refs_error_function, \ +} +struct fsck_options { +enum fsck_msg_type *msg_type; +unsigned strict:1 ;fsck_walk_func walk;fsck_error error_func; -unsigned strict:1 ; -enum fsck_msg_type *msg_type;struct oidset skiplist;struct oidset gitmodules_found;struct oidset gitmodules_done;
@@ -221,6 +248,12 @@ int fsck_tag_standalone(const struct object_id *oid, const char *buffer,*/ int fsck_finish (struct fsck_options *options); +__attribute__((format (printf,4 ,5 ))) +int fsck_refs_report (struct fsck_refs_options *o, +const char *name, +enum fsck_msg_id msg_id, +const char *fmt, ...); +/* * Subsystem for storing human-readable names for each object. *
@@ -247,6 +280,8 @@ const char *fsck_describe_object(struct fsck_options *options, const struct object_id *oid);struct key_value_info ; +int git_fsck_refs_config (const char *var,const char *value, +const struct config_context *ctx,void *cb);/* * git_config() callback for use by fsck-y tools that want to support * fsck.<msg> fsck.skipList etc.
Signature · 1 line
2.45.2
PATCH v2 2/7 refs: set up ref consistency check infrastructure
In reply to shejialuoThe interfaces defined in the ref_storage_be are carefully structured in semantic. It's organized as the five parts:
1. The name and the initialization interfaces.
2. The ref transaction interfaces.
3. The ref internal interfaces (pack, rename and copy).
4. The ref filesystem interfaces.
5. The reflog related interfaces.
To keep consistent with the git-fsck(1), add a new interface named "fsck_refs_fn" to the end of "ref_storage_be". This semantic cannot be grouped into any above five categories. Explicitly add blank line to make it different from others.
--- refs.c | 5 +++++ refs.h | 8 ++++++++ refs/debug.c | 9 +++++++++ refs/files-backend.c | 15 ++++++++++++++- refs/packed-backend.c | 8 ++++++++ refs/refs-internal.h | 6 ++++++ refs/reftable-backend.c | 8 ++++++++ 7 files changed, 58 insertions(+), 1 deletion(-)
refs.c +5 -0
@@ -316,6 +316,11 @@ int check_refname_format(const char *refname, int flags)return check_or_sanitize_refname (refname, flags,NULL ); } +int refs_fsck (struct ref_store *refs,struct fsck_refs_options *o) +{ +return refs->be->fsck (refs, o); +} +void sanitize_refname_component (const char *refname,struct strbuf *out) {if (check_or_sanitize_refname (refname,REFNAME_ALLOW_ONELEVEL , out))
refs.h +8 -0
@@ -3,6 +3,7 @@#include "commit.h" +struct fsck_refs_options ;struct object_id ;struct ref_store ;struct repository ;
@@ -547,6 +548,13 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat*/ int check_refname_format (const char *refname,int flags); +/* +* Check the reference database for consistency. Return 0 if refs and +* reflogs are consistent, and non-zero otherwise. The errors will be +* written to stderr. +*/ +int refs_fsck (struct ref_store *refs,struct fsck_refs_options *o); +/* * Apply the rules from check_refname_format, but mutate the result until it * is acceptable, and place the result in "out".
refs/debug.c +9 -0
@@ -419,6 +419,13 @@ static int debug_reflog_expire(struct ref_store *ref_store, const char *refname,return res; } +static int debug_fsck (struct ref_store *ref_store, +struct fsck_refs_options *o) +{ +trace_printf_key (&trace_refs,"fsck\n" ); +return 0 ; +} +struct ref_storage_be refs_be_debug = { .name ="debug" , .init =NULL ,
@@ -451,4 +458,6 @@ struct ref_storage_be refs_be_debug = { .create_reflog = debug_create_reflog, .delete_reflog = debug_delete_reflog, .reflog_expire = debug_reflog_expire, + + .fsck = debug_fsck, };
refs/files-backend.c +14 -1
@@ -3402,6 +3402,17 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,return ret; } +static int files_fsck (struct ref_store *ref_store, +struct fsck_refs_options *o) +{ +int ret; +struct files_ref_store *refs = +files_downcast (ref_store,REF_STORE_READ ,"fsck" ); + + ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o); +return ret; +} +struct ref_storage_be refs_be_files = { .name ="files" , .init = files_ref_store_init,
@@ -3428,5 +3439,7 @@ struct ref_storage_be refs_be_files = { .reflog_exists = files_reflog_exists, .create_reflog = files_create_reflog, .delete_reflog = files_delete_reflog, - .reflog_expire = files_reflog_expire + .reflog_expire = files_reflog_expire, + + .fsck = files_fsck, };
refs/packed-backend.c +8 -0
@@ -1733,6 +1733,12 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_sreturn empty_ref_iterator_begin (); } +static int packed_fsck (struct ref_store *ref_store, +struct fsck_refs_options *o) +{ +return 0 ; +} +struct ref_storage_be refs_be_packed = { .name ="packed" , .init = packed_ref_store_init,
@@ -1760,4 +1766,6 @@ struct ref_storage_be refs_be_packed = { .create_reflog = NULL, .delete_reflog = NULL, .reflog_expire = NULL, + + .fsck = packed_fsck, };
refs/refs-internal.h +6 -0
@@ -4,6 +4,7 @@#include "refs.h" #include "iterator.h" +struct fsck_refs_options ;struct ref_transaction ;/*
@@ -650,6 +651,9 @@ typedef int read_raw_ref_fn(struct ref_store *ref_store, const char *refname,typedef int read_symbolic_ref_fn (struct ref_store *ref_store,const char *refname,struct strbuf *referent); +typedef int fsck_fn (struct ref_store *ref_store, +struct fsck_refs_options *o); +struct ref_storage_be {const char *name;ref_store_init_fn *init;
@@ -677,6 +681,8 @@ struct ref_storage_be {create_reflog_fn *create_reflog;delete_reflog_fn *delete_reflog;reflog_expire_fn *reflog_expire; + +fsck_fn *fsck; };extern struct ref_storage_be refs_be_files;
refs/reftable-backend.c +8 -0
@@ -2242,6 +2242,12 @@ static int reftable_be_reflog_expire(struct ref_store *ref_store,return ret; } +static int reftable_be_fsck (struct ref_store *ref_store, +struct fsck_refs_options *o) +{ +return 0 ; +} +struct ref_storage_be refs_be_reftable = { .name ="reftable" , .init = reftable_be_init,
@@ -2269,4 +2275,6 @@ struct ref_storage_be refs_be_reftable = { .create_reflog = reftable_be_create_reflog, .delete_reflog = reftable_be_delete_reflog, .reflog_expire = reftable_be_reflog_expire, + + .fsck = reftable_be_fsck, };
Signature · 1 line
2.45.2
Re: [GSoC][PATCH v2 2/7] refs: set up ref consistency check infrastructure
shejialuo <[email protected]> writes:
[snip]
Quoted 6 lines
refs/debug.c +0
-0@@ -419,6 +419,13 @@ static int debug_reflog_expire(struct ref_store *ref_store, const char *refname,return res;
refs/debug.c static int debug_reflog_expire(struct ref_store *ref_store, const char *refname, · line 429
Quoted 10 lines
refs/debug.c +7
-0} +static int debug_fsck (struct ref_store *ref_store, +struct fsck_refs_options *o) +{ +trace_printf_key (&trace_refs,"fsck\n" ); +return 0 ; +} +
The debug interface acts as a wrapper around the actual refs backend being used, as such, you need to still call the underlying backend here.
[snip]
PATCH v2 3/7 builtin/refs: add verify subcommand
In reply to shejialuoIntroduce a new subcommand "verify" in git-refs(1) to allow the user to check the reference database consistency.
--- Documentation/git-refs.txt | 11 ++++++++++ builtin/refs.c | 45 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+)
Documentation/git-refs.txt +11 -0
@@ -10,6 +10,7 @@ SYNOPSIS -------- [verse] 'git refs migrate' --ref-format=<format> [--dry-run] +'git refs verify' [--strict] [--verbose] DESCRIPTION -----------
@@ -22,6 +23,9 @@ COMMANDS migrate:: Migrate ref store between different formats. +verify:: + Verify reference database consistency. + OPTIONS -------
@@ -39,6 +43,13 @@ include::ref-storage-format.txt[] can be used to double check that the migration works as expected before performing the actual migration. +--strict:: + Enable more strict checking, every WARN severity for the `Fsck Messages` + be seen as ERROR. + +--verbose:: + When verifying the reference database consistency, be chatty. + KNOWN LIMITATIONS -----------------
builtin/refs.c +45 -0
70 lines not on this page · raw message
Signature · 1 line
2.45.2
Re: [GSoC][PATCH v2 3/7] builtin/refs: add verify subcommand
Quoted 34 lines · shejialuo
Introduce a new subcommand "verify" in git-refs(1) to allow the user to check the reference database consistency.
Mentored-by Patrick Steinhardt <[email protected]>Mentored-by Karthik Nayak <[email protected]>Signed-off-by shejialuo <[email protected]>--- Documentation/git-refs.txt | 11 ++++++++++ builtin/refs.c | 45 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+)Documentation/git-refs.txt +4
-0@@ -10,6 +10,7 @@ SYNOPSIS -------- [verse] 'git refs migrate' --ref-format=<format> [--dry-run] +'git refs verify' [--strict] [--verbose] DESCRIPTION -----------@@ -22,6 +23,9 @@ COMMANDS migrate:: Migrate ref store between different formats. +verify:: + Verify reference database consistency. + OPTIONS -------
Documentation/git-refs.txt include::ref-storage-format.txt[] · line 45
Quoted 3 lines
Documentation/git-refs.txt +0
-0@@ -39,6 +43,13 @@ include::ref-storage-format.txt[] can be used to double check that the migration works as expected before performing the actual migration.
Above it states that the options mentioned are "specific to 'git refs migrate'", shouldn't we do something similar here?
Quoted 60 lines
Documentation/git-refs.txt +7
-0+--strict:: + Enable more strict checking, every WARN severity for the `Fsck Messages` + be seen as ERROR. + +--verbose:: + When verifying the reference database consistency, be chatty. + KNOWN LIMITATIONS -----------------builtin/refs.c +30
-0@@ -1,4 +1,6 @@#include "builtin.h" +#include "config.h" +#include "fsck.h" #include "parse-options.h" #include "refs.h" #include "repository.h" @@ -7,6 +9,9 @@#define REFS_MIGRATE_USAGE \ N_("git refs migrate --ref-format=<format> [--dry-run]") +#define REFS_VERIFY_USAGE \ + N_("git refs verify [--strict] [--verbose]") +static int cmd_refs_migrate (int argc,const char **argv,const char *prefix) {const char *const migrate_usage[] = {@@ -58,15 +63,55 @@ static int cmd_refs_migrate(int argc, const char **argv, const char *prefix)return err; } +static int cmd_refs_verify (int argc,const char **argv,const char *prefix) +{ +const char *const verify_usage[] = { +REFS_VERIFY_USAGE , +NULL , + }; +int ret =0 ; +unsigned int verbose =0 , strict =0 ; +struct fsck_refs_options fsck_refs_options =FSCK_REFS_OPTIONS_DEFAULT ; +struct option options[] = { +OPT__VERBOSE (&verbose,N_ ("be verbose" )), +OPT_BOOL (0 ,"strict" , &strict,N_ ("enable strict checking" )), +OPT_END (), + }; + + argc =parse_options (argc, argv, prefix, options, verify_usage,0 ); +if (argc) +usage (_ ("too many arguments" )); + +if (verbose) + fsck_refs_options.verbose =1 ; +if (strict) + fsck_refs_options.strict =1 ; + +git_config (git_fsck_refs_config, &fsck_refs_options);
builtin/refs.c static int cmd_refs_migrate(int argc, const char **argv, const char *prefix) · line 100
Quoted 10 lines
builtin/refs.c +10
-0+prepare_repo_settings (the_repository); + + ret =refs_fsck (get_main_ref_store (the_repository), &fsck_refs_options); + +/* +* Explicitly free the allocated array. This is necessary because +* this program is executed as child process of git-fsck(1) and the +* allocated array may not freed when git-fsck(1) aborts somewhere. +*/ +free (fsck_refs_options.msg_type);
Nice, this is a good thing to do even if this isn't called as a child process.
Quoted 19 lines
builtin/refs.c +5
-1+return ret; +} +int cmd_refs (int argc,const char **argv,const char *prefix) {const char *const refs_usage[] = {REFS_MIGRATE_USAGE , +REFS_VERIFY_USAGE ,NULL , };parse_opt_subcommand_fn *fn =NULL ;struct option opts[] = {OPT_SUBCOMMAND ("migrate" , &fn, cmd_refs_migrate), +OPT_SUBCOMMAND ("verify" , &fn, cmd_refs_verify),OPT_END (), }; --2.45.2
PATCH v2 4/7 builtin/fsck: add `git-refs verify` child process
In reply to shejialuoIntroduce a new function "fsck_refs" that initializes and runs a child process to execute the "git-refs verify" command.
--- builtin/fsck.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+)
builtin/fsck.c +17 -0
@@ -896,6 +896,21 @@ static int check_pack_rev_indexes(struct repository *r, int show_progress)return res; } +static void fsck_refs (void ) +{ +struct child_process refs_verify =CHILD_PROCESS_INIT ; +child_process_init (&refs_verify); + refs_verify.git_cmd =1 ; +strvec_pushl (&refs_verify.args,"refs" ,"verify" ,NULL ); +if (verbose) +strvec_push (&refs_verify.args,"--verbose" ); +if (check_strict) +strvec_push (&refs_verify.args,"--strict" ); + +if (run_command (&refs_verify)) + errors_found |=ERROR_REFS ; +} +static char const *const fsck_usage[] = { N_("git fsck [--tags] [--root] [--unreachable] [--cache] [--no-reflogs]\n" " [--[no-]full] [--strict] [--verbose] [--lost-found]\n"
@@ -1065,6 +1080,8 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)check_connectivity (); +fsck_refs (); +if (the_repository->settings.core_commit_graph) {struct child_process commit_graph_verify =CHILD_PROCESS_INIT ;
Signature · 1 line
2.45.2
Re: [GSoC][PATCH v2 4/7] builtin/fsck: add `git-refs verify` child process
Quoted 23 lines · shejialuo
Introduce a new function "
fsck_refs" that initializes and runs a child process to execute the "git-refs verify" command.Mentored-by Patrick Steinhardt <[email protected]>Mentored-by Karthik Nayak <[email protected]>Signed-off-by shejialuo <[email protected]>--- builtin/fsck.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+)builtin/fsck.c +5
-0@@ -896,6 +896,21 @@ static int check_pack_rev_indexes(struct repository *r, int show_progress)return res; } +static void fsck_refs (void ) +{ +struct child_process refs_verify =CHILD_PROCESS_INIT ; +child_process_init (&refs_verify); + refs_verify.git_cmd =1 ;
builtin/fsck.c static int check_pack_rev_indexes(struct repository *r, int show_progress) · line 913
Quoted 10 lines
builtin/fsck.c +10
-0+strvec_pushl (&refs_verify.args,"refs" ,"verify" ,NULL ); +if (verbose) +strvec_push (&refs_verify.args,"--verbose" ); +if (check_strict) +strvec_push (&refs_verify.args,"--strict" ); + +if (run_command (&refs_verify)) + errors_found |=ERROR_REFS ; +} +
At first I thought we need to call child_process_clear() here, but seems like run_command does that internally.
Quoted 14 lines
builtin/fsck.c +2
-1static char const *const fsck_usage[] = { N_("git fsck [--tags] [--root] [--unreachable] [--cache] [--no-reflogs]\n" " [--[no-]full] [--strict] [--verbose] [--lost-found]\n"@@ -1065,6 +1080,8 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)check_connectivity (); +fsck_refs (); +if (the_repository->settings.core_commit_graph) {struct child_process commit_graph_verify =CHILD_PROCESS_INIT ; --2.45.2
PATCH v2 5/7 files-backend: add unified interface for refs scanning
In reply to shejialuoFor refs and reflogs, we need to scan its corresponding directories to check every regular file or symbolic link which shares the same pattern. Introduce a unified interface for scanning directories for files-backend.
--- refs/files-backend.c | 75 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+)
refs/files-backend.c +75 -0
@@ -4,6 +4,7 @@#include "../gettext.h" #include "../hash.h" #include "../hex.h" +#include "../fsck.h" #include "../refs.h" #include "refs-internal.h" #include "ref-cache.h"
@@ -3402,6 +3403,78 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,return ret; } +/* +* For refs and reflogs, they share a unified interface when scanning +* the whole directory. This function is used as the callback for each +* regular file or symlink in the directory. +*/ +typedef int (*files_fsck_refs_fn )(struct fsck_refs_options *o, +const char *gitdir, +const char *refs_check_dir, +struct dir_iterator *iter); + +static int files_fsck_refs_dir (struct ref_store *ref_store, +struct fsck_refs_options *o, +const char *refs_check_dir, +files_fsck_refs_fn *fsck_refs_fns) +{ +const char *gitdir = ref_store->gitdir; +struct strbuf sb =STRBUF_INIT ; +struct dir_iterator *iter; +int iter_status; +int ret =0 ; + +strbuf_addf (&sb,"%s/%s" , gitdir, refs_check_dir); + + iter =dir_iterator_begin (sb.buf,0 ); + +if (!iter) { + ret =error_errno ("cannot open directory %s" , sb.buf); + goto out; + } + +while ((iter_status =dir_iterator_advance (iter)) ==ITER_OK ) { +if (S_ISDIR (iter->st.st_mode)) { +continue ; + }else if (S_ISREG (iter->st.st_mode) || +S_ISLNK (iter->st.st_mode)) { +if (o->verbose) +fprintf_ln (stderr,"Checking %s/%s" , + refs_check_dir, iter->relative_path); +for (size_t i =0 ; fsck_refs_fns[i]; i++) { +if (fsck_refs_fns[i](o, gitdir, refs_check_dir, iter)) + ret =-1 ; + } + }else { + ret =error (_ ("unexpected file type for '%s'" ), + iter->basename); + } + } + +if (iter_status !=ITER_DONE ) + ret =error (_ ("failed to iterate over '%s'" ), sb.buf); + +out: +strbuf_release (&sb); +return ret; +} + +static int files_fsck_refs (struct ref_store *ref_store, +struct fsck_refs_options *o) +{ +int ret; +files_fsck_refs_fn fsck_refs_fns[]= { +NULL + }; + +if (o->verbose) +fprintf_ln (stderr,"Checking references consistency" ); + + ret =files_fsck_refs_dir (ref_store, o,"refs" , fsck_refs_fns); + +return ret; +} +static int files_fsck (struct ref_store *ref_store,struct fsck_refs_options *o) {
@@ -3410,6 +3483,8 @@ static int files_fsck(struct ref_store *ref_store,files_downcast (ref_store,REF_STORE_READ ,"fsck" ); ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o); + ret =files_fsck_refs (ref_store, o); +return ret; }
Signature · 1 line
2.45.2
Re: [GSoC][PATCH v2 5/7] files-backend: add unified interface for refs scanning
Quoted 5 lines · shejialuo
For refs and reflogs, we need to scan its corresponding directories to check every regular file or symbolic link which shares the same pattern. Introduce a unified interface for scanning directories for files-backend.
Here we talk about reflogs, but we only add a iterator for "$GIT_DIR/refs". Should we add something for reflogs too?
Quoted 84 lines
Mentored-by Patrick Steinhardt <[email protected]>Mentored-by Karthik Nayak <[email protected]>Signed-off-by shejialuo <[email protected]>--- refs/files-backend.c | 75 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+)refs/files-backend.c +62
-0@@ -4,6 +4,7 @@#include "../gettext.h" #include "../hash.h" #include "../hex.h" +#include "../fsck.h" #include "../refs.h" #include "refs-internal.h" #include "ref-cache.h" @@ -3402,6 +3403,78 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,return ret; } +/* +* For refs and reflogs, they share a unified interface when scanning +* the whole directory. This function is used as the callback for each +* regular file or symlink in the directory. +*/ +typedef int (*files_fsck_refs_fn )(struct fsck_refs_options *o, +const char *gitdir, +const char *refs_check_dir, +struct dir_iterator *iter); + +static int files_fsck_refs_dir (struct ref_store *ref_store, +struct fsck_refs_options *o, +const char *refs_check_dir, +files_fsck_refs_fn *fsck_refs_fns) +{ +const char *gitdir = ref_store->gitdir; +struct strbuf sb =STRBUF_INIT ; +struct dir_iterator *iter; +int iter_status; +int ret =0 ; + +strbuf_addf (&sb,"%s/%s" , gitdir, refs_check_dir); + + iter =dir_iterator_begin (sb.buf,0 ); + +if (!iter) { + ret =error_errno ("cannot open directory %s" , sb.buf); + goto out; + } + +while ((iter_status =dir_iterator_advance (iter)) ==ITER_OK ) { +if (S_ISDIR (iter->st.st_mode)) { +continue ; + }else if (S_ISREG (iter->st.st_mode) || +S_ISLNK (iter->st.st_mode)) { +if (o->verbose) +fprintf_ln (stderr,"Checking %s/%s" , + refs_check_dir, iter->relative_path); +for (size_t i =0 ; fsck_refs_fns[i]; i++) { +if (fsck_refs_fns[i](o, gitdir, refs_check_dir, iter)) + ret =-1 ; + } + }else { + ret =error (_ ("unexpected file type for '%s'" ), + iter->basename); + } + } + +if (iter_status !=ITER_DONE ) + ret =error (_ ("failed to iterate over '%s'" ), sb.buf); + +out: +strbuf_release (&sb); +return ret; +} + +static int files_fsck_refs (struct ref_store *ref_store, +struct fsck_refs_options *o) +{ +int ret; +files_fsck_refs_fn fsck_refs_fns[]= {
refs/files-backend.c static int files_ref_store_remove_on_disk(struct ref_store *ref_store, · line 3476
Quoted 10 lines
refs/files-backend.c +10
-0+ NULL + }; + +if (o->verbose) +fprintf_ln (stderr,"Checking references consistency" ); + + ret =files_fsck_refs_dir (ref_store, o,"refs" , fsck_refs_fns); + +return ret; +}
I'm not fully sure why this needs to be separate from files_fsck. Is that because we plan to also introduce files_fsck_reflogs?
Quoted 15 lines
refs/files-backend.c +3
-1+static int files_fsck (struct ref_store *ref_store,struct fsck_refs_options *o) {@@ -3410,6 +3483,8 @@ static int files_fsck(struct ref_store *ref_store,files_downcast (ref_store,REF_STORE_READ ,"fsck" ); ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o); + ret =files_fsck_refs (ref_store, o); +return ret; } --2.45.2
Re: [GSoC][PATCH v2 5/7] files-backend: add unified interface for refs scanning
Quoted 11 lines · Karthik Nayak
Quoted 5 lines · shejialuo
For refs and reflogs, we need to scan its corresponding directories to check every regular file or symbolic link which shares the same pattern. Introduce a unified interface for scanning directories for files-backend.
Here we talk about reflogs, but we only add a iterator for "$GIT_DIR/refs". Should we add something for reflogs too?
Here, we simply set up the basic structure. For both refs and reflogs, we will traverse the file system. What we concern about is the regular file and symlink file. The "refs_check_dir" parameter is used to indicate what we will check. Actually, we do not add any actual check operations for refs in this commit.
Quoted 89 lines
Quoted 89 lines
Mentored-by Patrick Steinhardt <[email protected]>Mentored-by Karthik Nayak <[email protected]>Signed-off-by shejialuo <[email protected]>--- refs/files-backend.c | 75 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+)refs/files-backend.c +67
-0@@ -4,6 +4,7 @@#include "../gettext.h" #include "../hash.h" #include "../hex.h" +#include "../fsck.h" #include "../refs.h" #include "refs-internal.h" #include "ref-cache.h" @@ -3402,6 +3403,78 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,return ret; } +/* +* For refs and reflogs, they share a unified interface when scanning +* the whole directory. This function is used as the callback for each +* regular file or symlink in the directory. +*/ +typedef int (*files_fsck_refs_fn )(struct fsck_refs_options *o, +const char *gitdir, +const char *refs_check_dir, +struct dir_iterator *iter); + +static int files_fsck_refs_dir (struct ref_store *ref_store, +struct fsck_refs_options *o, +const char *refs_check_dir, +files_fsck_refs_fn *fsck_refs_fns) +{ +const char *gitdir = ref_store->gitdir; +struct strbuf sb =STRBUF_INIT ; +struct dir_iterator *iter; +int iter_status; +int ret =0 ; + +strbuf_addf (&sb,"%s/%s" , gitdir, refs_check_dir); + + iter =dir_iterator_begin (sb.buf,0 ); + +if (!iter) { + ret =error_errno ("cannot open directory %s" , sb.buf); + goto out; + } + +while ((iter_status =dir_iterator_advance (iter)) ==ITER_OK ) { +if (S_ISDIR (iter->st.st_mode)) { +continue ; + }else if (S_ISREG (iter->st.st_mode) || +S_ISLNK (iter->st.st_mode)) { +if (o->verbose) +fprintf_ln (stderr,"Checking %s/%s" , + refs_check_dir, iter->relative_path); +for (size_t i =0 ; fsck_refs_fns[i]; i++) { +if (fsck_refs_fns[i](o, gitdir, refs_check_dir, iter)) + ret =-1 ; + } + }else { + ret =error (_ ("unexpected file type for '%s'" ), + iter->basename); + } + } + +if (iter_status !=ITER_DONE ) + ret =error (_ ("failed to iterate over '%s'" ), sb.buf); + +out: +strbuf_release (&sb); +return ret; +} + +static int files_fsck_refs (struct ref_store *ref_store, +struct fsck_refs_options *o) +{ +int ret; +files_fsck_refs_fn fsck_refs_fns[]= { +NULL + }; + +if (o->verbose) +fprintf_ln (stderr,"Checking references consistency" );
Quoted 9 lines
refs/files-backend.c static int files_ref_store_remove_on_disk(struct ref_store *ref_store, · line 3476
Quoted 5 lines
refs/files-backend.c +5
-0+ + ret =files_fsck_refs_dir (ref_store, o,"refs" , fsck_refs_fns); + +return ret; +}I'm not fully sure why this needs to be separate from
files_fsck. Is that because we plan to also introducefiles_fsck_reflogs?
Yes, exactly. Later we need to introduce "files_fsck_reflogs" which should define the different files_fsck_refs_fn here. So just introduce a function here.
Quoted 15 lines
Quoted 15 lines
refs/files-backend.c +3
-1+static int files_fsck (struct ref_store *ref_store,struct fsck_refs_options *o) {@@ -3410,6 +3483,8 @@ static int files_fsck(struct ref_store *ref_store,files_downcast (ref_store,REF_STORE_READ ,"fsck" ); ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o); + ret =files_fsck_refs (ref_store, o); +return ret; } --2.45.2
PATCH v2 6/7 fsck: add ref name check for files backend
In reply to shejialuoThe git-fsck(1) only implicitly checks the reference, it does not fully check refs with bad format name such as standalone "@" and name ending with ".lock".
In order to provide such checks, add a new fsck message id "badRefName" with default ERROR type. Use existing "check_refname_format" to explicit check the ref name. And add a new unit test to verify the functionality.
--- Documentation/fsck-msgids.txt | 3 + fsck.h | 1 + refs/files-backend.c | 20 +++++++ t/t0602-reffiles-fsck.sh | 101 ++++++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+) create mode 100755 t/t0602-reffiles-fsck.sh
Documentation/fsck-msgids.txt +3 -0
@@ -19,6 +19,9 @@ `badParentSha1`:: (ERROR) A commit object has a bad parent sha1. +`badRefName`:: + (ERROR) A ref has a bad name. + `badTagName`:: (INFO) A tag has an invalid format.
fsck.h +1 -0
@@ -31,6 +31,7 @@ enum fsck_msg_type { FUNC(BAD_NAME, ERROR) \ FUNC(BAD_OBJECT_SHA1, ERROR) \ FUNC(BAD_PARENT_SHA1, ERROR) \ + FUNC(BAD_REF_NAME, ERROR) \ FUNC(BAD_TIMEZONE, ERROR) \ FUNC(BAD_TREE, ERROR) \FUNC (BAD_TREE_SHA1 ,ERROR ) \
refs/files-backend.c +20 -0
@@ -3413,6 +3413,25 @@ typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o,const char *refs_check_dir, struct dir_iterator *iter); +static int files_fsck_refs_name (struct fsck_refs_options *o, + const char *gitdir UNUSED, +const char *refs_check_dir, +struct dir_iterator *iter) +{ +struct strbuf sb =STRBUF_INIT ; +int ret =0 ; + +if (check_refname_format (iter->basename,REFNAME_ALLOW_ONELEVEL )) { +strbuf_addf (&sb,"%s/%s" , refs_check_dir, iter->relative_path); + ret =fsck_refs_report (o, sb.buf, +FSCK_MSG_BAD_REF_NAME , +"invalid refname format" ); + } + +strbuf_release (&sb); +return ret; +} + static int files_fsck_refs_dir(struct ref_store *ref_store, struct fsck_refs_options *o, const char *refs_check_dir,
@@ -3464,6 +3483,7 @@ static int files_fsck_refs(struct ref_store *ref_store, {int ret;files_fsck_refs_fn fsck_refs_fns[]= { + files_fsck_refs_name,NULL };
t/t0602-reffiles-fsck.sh new file +101 -0
101 lines not on this page · raw message
Signature · 1 line
2.45.2
PATCH v2 7/7 fsck: add ref content check for files backend
In reply to shejialuoEnhance the git-fsck(1) command by adding a check for reference content in the files backend. The new functionality ensures that symrefs, real symbolic link and regular refs are validated correctly.
Add a new function "files_fsck_symref" to check whether the symrefs and symbolic link points to a valid object and a new function "files_fsck_refs_content" handles both regular refs and symbolic refs.
In order to check the trailing content, add a new parameter "trailing" to "parse_loose_ref_contents" function.
Last, add the following FSCK MESSAGEs:
1. "badRefSha(ERROR)": A ref has a bad sha.
2. "danglingSymre(WARN)": Found a dangling symref.
3. "trailingRefContent(WARN)": A ref content has trailing contents.
--- Documentation/fsck-msgids.txt | 9 +++ fsck.h | 3 + refs.c | 2 +- refs/files-backend.c | 124 +++++++++++++++++++++++++++++++++- refs/refs-internal.h | 5 +- t/t0602-reffiles-fsck.sh | 110 ++++++++++++++++++++++++++++++ 6 files changed, 248 insertions(+), 5 deletions(-)
Documentation/fsck-msgids.txt +9 -0
@@ -22,6 +22,9 @@ `badRefName`:: (ERROR) A ref has a bad name. +`badRefSha`:: + (ERROR) A ref has a bad sha. + `badTagName`:: (INFO) A tag has an invalid format.
@@ -37,6 +40,9 @@ `badType`:: (ERROR) Found an invalid object type. +`danglingSymref`:: + (WARN) Found a dangling symref. + `duplicateEntries`:: (ERROR) A tree contains duplicate file entries.
@@ -179,6 +185,9 @@ `symlinkTargetMissing`:: (ERROR) Unable to read symbolic link target's blob. +`trailingRefContent`:: + (WARN) A ref content has trailing contents. + `treeNotSorted`:: (ERROR) A tree is not properly sorted.
fsck.h +3 -0
@@ -32,6 +32,7 @@ enum fsck_msg_type { FUNC(BAD_OBJECT_SHA1, ERROR) \ FUNC(BAD_PARENT_SHA1, ERROR) \ FUNC(BAD_REF_NAME, ERROR) \ + FUNC(BAD_REF_SHA, ERROR) \ FUNC(BAD_TIMEZONE, ERROR) \ FUNC(BAD_TREE, ERROR) \FUNC (BAD_TREE_SHA1 ,ERROR ) \
@@ -69,11 +70,13 @@ enum fsck_msg_type {FUNC (SYMLINK_TARGET_BLOB ,ERROR ) \ /* warnings */ \ FUNC(EMPTY_NAME, WARN) \ + FUNC(DANGLING_SYMREF, WARN) \ FUNC(FULL_PATHNAME, WARN) \ FUNC(HAS_DOT, WARN) \ FUNC(HAS_DOTDOT, WARN) \ FUNC(HAS_DOTGIT, WARN) \ FUNC(NULL_SHA1, WARN) \ + FUNC(TRAILING_REF_CONTENT, WARN) \ FUNC(ZERO_PADDED_FILEMODE, WARN) \ FUNC(NUL_IN_COMMIT, WARN) \FUNC (LARGE_PATHNAME ,WARN ) \
refs.c +1 -1
@@ -1744,7 +1744,7 @@ static int refs_read_special_head(struct ref_store *ref_store, } result =parse_loose_ref_contents (content.buf, oid, referent, type, - failure_errno); + failure_errno,NULL ); done:strbuf_release (&full_path);
refs/files-backend.c +122 -2
154 lines not on this page · raw message
refs/refs-internal.h +3 -2
@@ -709,11 +709,12 @@ struct ref_store {/* * Parse contents of a loose ref file. *failure_errno maybe be set to EINVAL for -* invalid contents. +* invalid contents. Also trailing is set to 1 when there is any bytes after the +* hex. */ int parse_loose_ref_contents (const char *buf,struct object_id *oid,struct strbuf *referent,unsigned int *type, -int *failure_errno); +int *failure_errno,unsigned int *trailing);/* * Fill in the generic part of refs and add it to our collection of
t/t0602-reffiles-fsck.sh +110 -0
114 lines not on this page · raw message
Signature · 1 line
2.45.2
Re: [GSoC][PATCH v2 7/7] fsck: add ref content check for files backend
Quoted 2 lines · shejialuo
In order to check the trailing content, add a new parameter "trailing" to "
parse_loose_ref_contents" function.
About this one.
Quoted 8 lines
int parse_loose_ref_contents(const char *buf, struct
object_id*oid,refs/files-backend.c +1
-1struct strbuf *referent, unsigned int *type, - int *failure_errno) + int *failure_errno, unsigned int *trailing) { const char *p;if (skip_prefix (buf,"ref:" , &buf)) {
refs/files-backend.c int parse_loose_ref_contents(const char *buf, struct object_id *oid, · line 615
Quoted 9 lines
refs/files-backend.c +4
-0@@ -607,6 +607,10 @@ int parse_loose_ref_contents(const char *buf, struct object_id *oid, *failure_errno =EINVAL ;return -1 ; } + +if (trailing && (*p !='\0' && *p !='\n' )) + *trailing =1 ; +return 0 ; }
We know what the garbage looked like at this point. The caller owns the "buf" pointer and we are pointing into that buffer with the pointer p, and the garbage is right there.
So I am not sure if losing information by using "uint *" is a good idea. Wouldn't it make more sense to take "const char **trailing" as a parameter and tell the caller where the trailing junk begins?
refs/files-backend.c static int files_fsck_refs_name(struct fsck_refs_options *o, · line 3441
Quoted 3 lines
refs/files-backend.c +3
-0+static int files_fsck_symref (struct fsck_refs_options *o, +struct strbuf *refname, +struct strbuf *path)
This does not take things like HEAD or refs/remotes/origin/HEAD to validate. Instead, the caller is responsible for either doing a readlink on a symbolic link, or reading a textual symref and stripping "ref: " prefix from it, before calling this function. The "refname" parameter is not HEAD or refs/remotes/origin/HEAD but the pointee of the symref.
So I'd imagine that renaming it to fsck_symref_target or along that line to clarify that we are not checking the symref, but the target of a symref, would be a good idea.
refs/files-backend.c static int files_fsck_refs_name(struct fsck_refs_options *o, · line 3451
Quoted 10 lines
refs/files-backend.c +10
-0+{ +struct stat st; +int ret =0 ; + +if (lstat (path->buf, &st) <0 ) { + ret =fsck_refs_report (o, refname->buf, +FSCK_MSG_DANGLING_SYMREF , +"point to non-existent ref" ); + goto out; + }
Is that an error? Just like being on an unborn branch is not an error, it could be argued that a symref that points at a branch yet to be born wouldn't be an error, either, no?
refs/files-backend.c static int files_fsck_refs_name(struct fsck_refs_options *o, · line 3457
Quoted 5 lines
refs/files-backend.c +5
-0+if (!S_ISREG (st.st_mode) && !S_ISLNK (st.st_mode)) { + ret =fsck_refs_report (o, refname->buf, +FSCK_MSG_DANGLING_SYMREF , +"point to invalid object" ); + goto out;
The use of "object" here is highly misleading. Yes, you can call a filesystem entity like "a regular file", "a directory", etc. "an object", but the word can refer to many other kinds of "object". In fact, I originally read this to mean "we are referring to an object in the object database that is corrupt" or something, but of course that is not what we are complaining about. We are complaining that the symbolic link points at a file of wrong type (like a directory).
So, in short, missing is probably OK. Pointing at a wrong thing (like a directory or block device) is not. Where, if any, do we catch a symbolic ref that tries to escape the refs/* hierarchy (e.g. ".git/refs/heads/my-crazy-ref" that is a symbolic link that points at "../../../../else/where" that is not even part of the repository), by the way?
Thanks.
Re: [GSoC][PATCH v2 7/7] fsck: add ref content check for files backend
Quoted 29 lines · Junio C Hamano
Quoted 2 lines · shejialuo
In order to check the trailing content, add a new parameter "trailing" to "
parse_loose_ref_contents" function.About this one.
Quoted 8 lines
int parse_loose_ref_contents(const char *buf, struct
object_id*oid,refs/files-backend.c +1
-1struct strbuf *referent, unsigned int *type, - int *failure_errno) + int *failure_errno, unsigned int *trailing) { const char *p;if (skip_prefix (buf,"ref:" , &buf)) {refs/files-backend.c int parse_loose_ref_contents(const char *buf, struct object_id *oid, · line 615
Quoted 9 lines
refs/files-backend.c +4
-0@@ -607,6 +607,10 @@ int parse_loose_ref_contents(const char *buf, struct object_id *oid, *failure_errno =EINVAL ;return -1 ; } + +if (trailing && (*p !='\0' && *p !='\n' )) + *trailing =1 ; +return 0 ; }We know what the garbage looked like at this point. The caller owns the "buf" pointer and we are pointing into that buffer with the pointer p, and the garbage is right there.
Quoted 4 lines
So I am not sure if losing information by using "uint *" is a good idea. Wouldn't it make more sense to take "const char **trailing" as a parameter and tell the caller where the trailing junk begins?
Yes, I totally agree that using "uint *" will lose a lot of information here. Actually I have used the "const char **trailing", but I made a mistake to result the wild pointer. This is because when "parse_loose_ref_contents" handles symref, it will never handle *p. When the caller defines const char *trailing, it will be wild pointer. But I think we could set it to NULL when handling symref.
I will change the code in the next version.
Quoted 11 lines
refs/files-backend.c static int files_fsck_refs_name(struct fsck_refs_options *o, · line 3441
Quoted 3 lines
refs/files-backend.c +3
-0+static int files_fsck_symref (struct fsck_refs_options *o, +struct strbuf *refname, +struct strbuf *path)This does not take things like HEAD or refs/remotes/origin/HEAD to validate. Instead, the caller is responsible for either doing a readlink on a symbolic link, or reading a textual symref and stripping "ref: " prefix from it, before calling this function. The "refname" parameter is not HEAD or refs/remotes/origin/HEAD but the pointee of the symref.
Quoted 4 lines
So I'd imagine that renaming it to
fsck_symref_targetor along that line to clarify that we are not checking the symref, but the target of a symref, would be a good idea.
That's not correct. The "refname" parameter is EXACTLY the symref itself. What we do here is to check the "path" paramteter, there are two situations:
1. For symref we will strip "ref: " prefix, and combine the girdir and the stripped content to get the "path" parameter.
2. For symbolic we will get its actual path (here I made a mistake, I totally forget the situation when it points to absolute path, I will revise the code for handling it).
The design here is just check whether the symref points to the correct thing. It does not care about the pointee. The code will traverse every regular file under the "refs/" directory, eventually we will check the "pointee" status. For example, a symref "sym-branch" and a regular ref "branch".
sym-branch: "ref: refs/heads/branch".
branch: "xxxx"
The design will not report any error for "sym-branch". I think we should discuss here whether this design is OK.
Quoted 6 lines
Quoted 6 lines
refs/files-backend.c +6
-0+{ +struct stat st; +int ret =0 ; + +if (lstat (path->buf, &st) <0 ) { + ret = fsck_refs_report(o, refname->buf,
Quoted 9 lines
refs/files-backend.c static int files_fsck_refs_name(struct fsck_refs_options *o, · line 3451
Quoted 4 lines
refs/files-backend.c +4
-0+FSCK_MSG_DANGLING_SYMREF , + "point to non-existent ref"); + goto out; + }Is that an error? Just like being on an unborn branch is not an error, it could be argued that a symref that points at a branch yet to be born wouldn't be an error, either, no?
The reason why I choose "danglingSymref" and warn severity is that I let the code be align with "git checkout". When we use "git checkout" for a dangling symref. It would produce the following output:
$ git checkout branch-3 warning: ignoring dangling symref refs/heads/branch-3 error: pathspec 'branch-3' did not match any file(s) known to git
So I prefer to warn severity.
Quoted 6 lines
refs/files-backend.c static int files_fsck_refs_name(struct fsck_refs_options *o, · line 3457
Quoted 5 lines
refs/files-backend.c +5
-0+if (!S_ISREG (st.st_mode) && !S_ISLNK (st.st_mode)) { + ret =fsck_refs_report (o, refname->buf, +FSCK_MSG_DANGLING_SYMREF , +"point to invalid object" ); + goto out;
Quoted 8 lines
The use of "object" here is highly misleading. Yes, you can call a filesystem entity like "a regular file", "a directory", etc. "an object", but the word can refer to many other kinds of "object". In fact, I originally read this to mean "we are referring to an object in the object database that is corrupt" or something, but of course that is not what we are complaining about. We are complaining that the symbolic link points at a file of wrong type (like a directory).
Yes, it brings a lot of misleading here. I will clean the code and commit message (I also used object in commit message).
Quoted 7 lines
So, in short, missing is probably OK. Pointing at a wrong thing (like a directory or block device) is not. Where, if any, do we catch a symbolic ref that tries to escape the refs/* hierarchy (e.g. ".git/refs/heads/my-crazy-ref" that is a symbolic link that points at "../../../../else/where" that is not even part of the repository), by the way?
I intentionally ignored the "escape" situation. Actually, the path could be either absolute or relative. It may be a little complicated. I will find a way to support this in the next version.
Quoted 1 line
Thanks.
Thanks,
Jialuo
Re: [GSoC][PATCH v2 7/7] fsck: add ref content check for files backend
Quoted 11 lines · shejialuo
Quoted 11 lines
refs/files-backend.c static int files_fsck_refs_name(struct fsck_refs_options *o, · line 3441
Quoted 3 lines
refs/files-backend.c +3
-0+static int files_fsck_symref (struct fsck_refs_options *o, +struct strbuf *refname, +struct strbuf *path)This does not take things like HEAD or refs/remotes/origin/HEAD to validate. Instead, the caller is responsible for either doing a readlink on a symbolic link, or reading a textual symref and stripping "ref: " prefix from it, before calling this function. The "refname" parameter is not HEAD or refs/remotes/origin/HEAD but the pointee of the symref.
Quoted 6 lines
Quoted 3 lines
So I'd imagine that renaming it to
fsck_symref_targetor along that line to clarify that we are not checking the symref, but the target of a symref, would be a good idea.That's not correct. The "refname" parameter is EXACTLY the symref itself.
Yeah, but the story is the same. We are not really checking anything about the symref (i.e. the thing in "refname") being funny. We are checking what is in "path" (like "does it exist?") and the "refname" is there only for reporting purposes (i.e. "we have a symbolic ref REFNAME, that points at PATH which is a wrong thing").
Quoted 20 lines
Quoted 6 lines
Quoted 6 lines
refs/files-backend.c +6
-0+{ +struct stat st; +int ret =0 ; + +if (lstat (path->buf, &st) <0 ) { + ret = fsck_refs_report(o, refname->buf,Quoted 9 lines
refs/files-backend.c static int files_fsck_refs_name(struct fsck_refs_options *o, · line 3451
Quoted 4 lines
refs/files-backend.c +4
-0+FSCK_MSG_DANGLING_SYMREF , + "point to non-existent ref"); + goto out; + }Is that an error? Just like being on an unborn branch is not an error, it could be argued that a symref that points at a branch yet to be born wouldn't be an error, either, no?
The reason why I choose "danglingSymref" and warn severity is that I let the code be align with "git checkout". When we use "git checkout" for a dangling symref. It would produce the following output:
Quoted 5 lines
$ git checkout branch-3 warning: ignoring dangling symref refs/heads/branch-3 error: pathspec 'branch-3' did not match any file(s) known to gitSo I prefer to warn severity.
If you do this from that situation,
$ git branch branch-3 master
what happens is that the pointee of branch-3 is created at the commit pointed at by 'master'. No error. No warnings.
In a freshly created respository, HEAD is a dangling symbolic ref, and that is not an error. You can create a root commit from there just fine.
If there is anything that needs improvement in your example, it is that "checkout branch-3" should be taught to either (1) not warn about dangling symbolic link and just give the error, which is in line with how "git checkout HEAD" in a freshly created repository behaves, or (2) just like unborn 'master' pointed at by 'HEAD' is perfectly happy to be checked out, allow the unborn 'branch-3' to be pointed at by 'HEAD', and arrange the first commit (which will be a root commit) you create in that state to be pointed by the ref 'branch-3' points at.
So from all of these reasons, I do not think missing target should be treated as any error worthy event. Not even warning.
On the other hand, the target of the symref in "path" must be checked, even if it does not currently exist, for its validity, the same way an existing ref gets checked (lives inside refs/, passes check-ref-format, etc.).
Quoted 3 lines
I intentionally ignored the "escape" situation. Actually, the path could be either absolute or relative. It may be a little complicated. I will find a way to support this in the next version.
Yes, if this wants to claim to be part of "FSCK", it should catch all the errors the regular runtime would complain about, and "escape" thing is one of the first things that you need to get right. Whatever refs.c:read_ref_internal() for S_ISLNK(st.st_mode) case takes as legit should be considered legit, the "fallback - read through the symlink as if it were a non-symlink" case probably wants to be warned. refs.c:resolve_ref_unsafe(), which is used at the low level from object-name.c:get_oid_basic() via refs.c:repo_dwim_ref(), has further checks to see if a refname that a symref resolves to is valid and the runtime sanity relies on these checks.
Thanks.
Re: [GSoC][PATCH v2 7/7] fsck: add ref content check for files backend
Junio C Hamano <[email protected]> writes:
Quoted 18 lines
Quoted 15 lines · shejialuo
Quoted 14 lines
refs/files-backend.c static int files_fsck_refs_name(struct fsck_refs_options *o, · line 3441
Quoted 3 lines
refs/files-backend.c +3
-0+static int files_fsck_symref (struct fsck_refs_options *o, +struct strbuf *refname, +struct strbuf *path)This does not take things like HEAD or refs/remotes/origin/HEAD to validate. Instead, the caller is responsible for either doing a readlink on a symbolic link, or reading a textual symref and stripping "ref: " prefix from it, before calling this function. The "refname" parameter is not HEAD or refs/remotes/origin/HEAD but the pointee of the symref.
So I'd imagine that renaming it to
fsck_symref_targetor along that line to clarify that we are not checking the symref, but the target of a symref, would be a good idea.
Quoted 9 lines
Quoted 2 lines
That's not correct. The "refname" parameter is EXACTLY the symref itself.
Yeah, but the story is the same. We are not really checking anything about the symref (i.e. the thing in "refname") being funny. We are checking what is in "path" (like "does it exist?") and the "refname" is there only for reporting purposes (i.e. "we have a symbolic ref REFNAME, that points at PATH which is a wrong thing").
Yes, I agree, will rename the function in the next version.
Quoted 28 lines
Quoted 20 lines
Quoted 6 lines
Quoted 6 lines
refs/files-backend.c +6
-0+{ +struct stat st; +int ret =0 ; + +if (lstat (path->buf, &st) <0 ) { + ret = fsck_refs_report(o, refname->buf,Quoted 9 lines
refs/files-backend.c static int files_fsck_refs_name(struct fsck_refs_options *o, · line 3451
Quoted 4 lines
refs/files-backend.c +4
-0+FSCK_MSG_DANGLING_SYMREF , + "point to non-existent ref"); + goto out; + }Is that an error? Just like being on an unborn branch is not an error, it could be argued that a symref that points at a branch yet to be born wouldn't be an error, either, no?
The reason why I choose "danglingSymref" and warn severity is that I let the code be align with "git checkout". When we use "git checkout" for a dangling symref. It would produce the following output:
Quoted 5 lines
$ git checkout branch-3 warning: ignoring dangling symref refs/heads/branch-3 error: pathspec 'branch-3' did not match any file(s) known to gitSo I prefer to warn severity.
If you do this from that situation,
Quoted 9 lines
$ git branch branch-3 masterwhat happens is that the pointee of branch-3 is created at the commit pointed at by 'master'. No error. No warnings.
In a freshly created respository, HEAD is a dangling symbolic ref, and that is not an error. You can create a root commit from there just fine.
I am totally shocked by the fact that it will create the pointee file. I am a little curious about the design here. I know when creating a new repo, HEAD is a dangling symbolic ref. When we do first commit, it will create the pointee file. But HEAD is a special file, it should be treated differently. Why we treat the other symrefs the same way.
Quoted 10 lines
If there is anything that needs improvement in your example, it is that "checkout branch-3" should be taught to either (1) not warn about dangling symbolic link and just give the error, which is in line with how "git checkout HEAD" in a freshly created repository behaves, or (2) just like unborn 'master' pointed at by 'HEAD' is perfectly happy to be checked out, allow the unborn 'branch-3' to be pointed at by 'HEAD', and arrange the first commit (which will be a root commit) you create in that state to be pointed by the ref 'branch-3' points at.
Should we really improve this? From my perspective, a user will get more information when this warn message shows up. The "error: pathspec 'branch-3' did not match any file(s) known to git" is less informative.
Maybe the pro git users do not use git command to write to the git file system. However, this is a bad way to do this. We always want to the user to use the command to handle with git file system.
Instead, we should warn about the user when using
$ git branch branch-3 master
It will create a new pointee here, I suppose we should warn the user here except HEAD ref. The user could get the dangling symref by deleting the pointee ref. For example:
$ git symbolic-ref refs/heads/main master $ git checkout -b master-copy $ git branch --delete master
Should we warn this for user in git-fsck(1)?
Quoted 3 lines
So from all of these reasons, I do not think missing target should be treated as any error worthy event. Not even warning.
Yes, if git-branch implicitly creates the pointee ref file. We should not handle such case. But I think giving a warning here except HEAD ref is worthful.
Quoted 5 lines
On the other hand, the target of the symref in "path" must be checked, even if it does not currently exist, for its validity, the same way an existing ref gets checked (lives inside refs/, passes check-ref-format, etc.).
Yes!
Quoted 4 lines
Quoted 3 lines
I intentionally ignored the "escape" situation. Actually, the path could be either absolute or relative. It may be a little complicated. I will find a way to support this in the next version.
Quoted 11 lines
Yes, if this wants to claim to be part of "FSCK", it should catch all the errors the regular runtime would complain about, and "escape" thing is one of the first things that you need to get right. Whatever refs.c:read_ref_internal() for
S_ISLNK(st.st_mode)case takes as legit should be considered legit, the "fallback - read through the symlink as if it were a non-symlink" case probably wants to be warned. refs.c:resolve_ref_unsafe(), which is used at the low level from object-name.c:get_oid_basic() via refs.c:repo_dwim_ref(), has further checks to see if a refname that a symref resolves to is valid and the runtime sanity relies on these checks.
Thanks for your help! I will look at these codes to implement the "escape" check.
Quoted 2 lines
Thanks.
Re: [GSoC][PATCH v2 7/7] fsck: add ref content check for files backend
Quoted 6 lines · jialuo she
Instead, we should warn about the user when using
$ git branch branch-3 masterIt will create a new pointee here, I suppose we should warn the user here except HEAD ref.
The user created branch-3 as a symlink, fully expecting that the pointee will be updated.
By the way, a seemingly normal branch looking symref is very handy and that is what my "maint" branch looks like. Right now it is pointing at maint-2.45 (because 2.45.0 was the last feature release)
$ git symbolic-ref refs/heads/maint
refs/heads/maint-2.45and I would imagine many other users are using the trick to maintain multiple maintenance tracks. They would be upset if their next "Now 2.46.0 release is done, let's start the 2.46 maintenance track" mantra, which would look like so:
$ git symbolic-ref refs/heads/maint refs/heads/maint-2.46 $ git branch maint v2.46.0
starts to give an unnecessary warning.
Sorry for the confusion, I wanna emphasis what this series of patches replies on.
It is built upon on the this patch.
https://lore.kernel.org/git/[email protected]/T/#t
Thanks,
Jialuo
v3 cover ref consistency check infra setup
In reply to shejialuoThis is the third version. Mainly focuses on the following problems:
1. Add "escape" check for symrefs and symbolic links.
2. Resolve the reviews.
3. Fix some errors in version 2.
shejialuo (7):
fsck: add refs check interfaces to interface with fsck error levels
refs: set up ref consistency check infrastructure builtin/refs: add verify subcommand
builtin/fsck: add git-refs verify child process
files-backend: add unified interface for refs scanning
fsck: add ref name check for files backend
fsck: add ref content check for files backend
Documentation/fsck-msgids.txt | 12 ++ Documentation/git-refs.txt | 13 ++ builtin/fsck.c | 17 +++ builtin/refs.c | 45 ++++++ fsck.c | 69 +++++++++ fsck.h | 49 ++++++- refs.c | 7 +- refs.h | 8 ++ refs/debug.c | 11 ++ refs/files-backend.c | 255 +++++++++++++++++++++++++++++++++- refs/packed-backend.c | 8 ++ refs/refs-internal.h | 11 +- refs/reftable-backend.c | 8 ++ t/t0602-reffiles-fsck.sh | 211 ++++++++++++++++++++++++++++ 14 files changed, 716 insertions(+), 8 deletions(-) create mode 100755 t/t0602-reffiles-fsck.sh
Range-diff against v2:
1: a65de15dc7 ! 1: b8d20c4be9 fsck: add refs check interfaces to interface with fsck error levels
@@ fsck.h: int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o,
+struct fsck_refs_options {
+ enum fsck_msg_type *msg_type;
+ unsigned strict:1;
++
+ fsck_refs_error error_func;
+ unsigned verbose:1;
+};
@@ fsck.h: int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o,
+}
+
struct fsck_options {
++ /*
++ * Reorder the fields to allow `fsck_ref_options` to use
++ * the interfaces using `struct fsck_options`.
++ */
+ enum fsck_msg_type *msg_type;
+ unsigned strict:1;
++
fsck_walk_func walk;
fsck_error error_func;
- unsigned strict:1;
2: a87149069a ! 2: c7ed95d6b4 refs: set up ref consistency check infrastructure
@@ refs/debug.c: static int debug_reflog_expire(struct ref_store *ref_store, const
+static int debug_fsck(struct ref_store *ref_store,
+ struct fsck_refs_options *o)
+{
-+ trace_printf_key(&trace_refs, "fsck\n");
-+ return 0;
++ struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store;
++ int res = drefs->refs->be->fsck(drefs->refs, o);
++ trace_printf_key(&trace_refs, "fsck: %d\n", res);
++ return res;
+}
+
struct ref_storage_be refs_be_debug = {3: e1f1b77527 ! 3: 373f470fdc builtin/refs: add verify subcommand
@@ Documentation/git-refs.txt: include::ref-storage-format.txt[]
can be used to double check that the migration works as expected before performing the actual migration.
++The following options are specific to 'git refs verify':
++
+--strict::
+ Enable more strict checking, every WARN severity for the Fsck Messages
+ be seen as ERROR.
4: 2b0ee79e78 = 4: 4f8ebea573 builtin/fsck: add `git-refs verify` child process
5: 6d23eaa1d1 ! 5: 6062b3b453 files-backend: add unified interface for refs scanning
@@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
struct fsck_refs_options *o)
{
@@ refs/files-backend.c: static int files_fsck(struct ref_store *ref_store,
+ struct files_ref_store *refs =
files_downcast(ref_store, REF_STORE_READ, "fsck");
- ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o);
-+ ret = files_fsck_refs(ref_store, o);
+- ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o);
++ ret = refs->packed_ref_store->be->fsck(refs->packed_ref_store, o)
++ | files_fsck_refs(ref_store, o);
+
return ret;
}
6: 8c55ecdd88 = 6: 87a22f4c7f fsck: add ref name check for files backend
7: aba5f50940 ! 7: b945f61ef1 fsck: add ref content check for files backend
@@ Commit message
in the files backend. The new functionality ensures that symrefs, real
symbolic link and regular refs are validated correctly.
- Add a new function "files_fsck_symref" to check whether the symrefs and
- symbolic link points to a valid object and a new function
- "files_fsck_refs_content" handles both regular refs and symbolic refs.
+ In order to check the trailing content of the regular refs, add a new
+ parameter `trailing` to `parse_loose_ref_contents`.
- In order to check the trailing content, add a new parameter
- "trailing" to "parse_loose_ref_contents" function.
+ For symrefs, `parse_loose_ref_contents` will set the "referent".
+ However, symbolic link could be either absolute or relative. Use
+ "strbuf_add_real_path" to read the symbolic link and convert the
+ relative path to absolute path. Then use "skip_prefix" to make it align
+ with symref "referent".
+
+ Thus, the symrefs and symbolic links could share the same interface. Add
+ a new function "files_fsck_symref_target" which aims at checking the
+ following things:
+
+ 1. whether the pointee is under the `refs/` directory.
+ 2. whether the pointee name is correct.
+ 3. whether the pointee path is a wrong type in filesystem.
Last, add the following FSCK MESSAGEs:
- 1. "badRefSha(ERROR)": A ref has a bad sha.
- 2. "danglingSymre(WARN)": Found a dangling symref.
+ 1. "badRefContent(ERROR)": A ref has a bad content
+ 2. "badSymrefPointee(ERROR)": The pointee of a symref is bad.
3. "trailingRefContent(WARN)": A ref content has trailing contents.
Mentored-by: Patrick Steinhardt <[email protected]>
@@ Commit message
## Documentation/fsck-msgids.txt ##
@@
+ `badParentSha1`::
+ (ERROR) A commit object has a bad parent sha1.
+
++`badRefContent`::
++ (ERROR) A ref has a bad content.
++
`badRefName`::
(ERROR) A ref has a bad name.
-+`badRefSha`::
-+ (ERROR) A ref has a bad sha.
++`badSymrefPointee`::
++ (ERROR) The pointee of a symref is bad.
+
`badTagName`::
(INFO) A tag has an invalid format.
-@@
- `badType`::
- (ERROR) Found an invalid object type.
-
-+`danglingSymref`::
-+ (WARN) Found a dangling symref.
-+
- `duplicateEntries`::
- (ERROR) A tree contains duplicate file entries.
-
@@
`symlinkTargetMissing`::
(ERROR) Unable to read symbolic link target's blob.
@@ fsck.h: enum fsck_msg_type {
FUNC(BAD_OBJECT_SHA1, ERROR) \
FUNC(BAD_PARENT_SHA1, ERROR) \
FUNC(BAD_REF_NAME, ERROR) \
-+ FUNC(BAD_REF_SHA, ERROR) \
++ FUNC(BAD_REF_CONTENT, ERROR) \
++ FUNC(BAD_SYMREF_POINTEE, ERROR) \
FUNC(BAD_TIMEZONE, ERROR) \
FUNC(BAD_TREE, ERROR) \
FUNC(BAD_TREE_SHA1, ERROR) \
@@ fsck.h: enum fsck_msg_type {
- FUNC(SYMLINK_TARGET_BLOB, ERROR) \
- /* warnings */ \
- FUNC(EMPTY_NAME, WARN) \
-+ FUNC(DANGLING_SYMREF, WARN) \
- FUNC(FULL_PATHNAME, WARN) \
- FUNC(HAS_DOT, WARN) \
FUNC(HAS_DOTDOT, WARN) \
FUNC(HAS_DOTGIT, WARN) \
FUNC(NULL_SHA1, WARN) \
@@ refs.c: static int refs_read_special_head(struct ref_store *ref_store,
strbuf_release(&full_path);
## refs/files-backend.c ##
+@@
+ #include "../git-compat-util.h"
++#include "../abspath.h"
+ #include "../copy.h"
+ #include "../environment.h"
+ #include "../gettext.h"
@@ refs/files-backend.c: static int read_ref_internal(struct ref_store *ref_store, const char *refname,
strbuf_rtrim(&sb_contents);
buf = sb_contents.buf;
@@ refs/files-backend.c: static int files_read_symbolic_ref(struct ref_store *ref_s
int parse_loose_ref_contents(const char *buf, struct object_id *oid,
struct strbuf *referent, unsigned int *type,
- int *failure_errno)
-+ int *failure_errno, unsigned int *trailing)
++ int *failure_errno, const char **trailing)
{
const char *p;
if (skip_prefix(buf, "ref:", &buf)) {
@@ refs/files-backend.c: int parse_loose_ref_contents(const char *buf, struct objec
return -1;
}
+
-+ if (trailing && (*p != '\0' && *p != '\n'))
-+ *trailing = 1;
++ if (trailing)
++ *trailing = p;
+
return 0;
}
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
return ret;
}
-+static int files_fsck_symref(struct fsck_refs_options *o,
-+ struct strbuf *refname,
-+ struct strbuf *path)
++/*
++ * Check the symref "pointee_name" and "pointee_path". The caller should
++ * make sure that "pointee_path" is absolute. For symbolic ref, "pointee_name"
++ * would be the content after "refs:". For symblic link, "pointee_name" would
++ * be the relative path agaignst "gitdir".
++ */
++static int files_fsck_symref_target(struct fsck_refs_options *o,
++ const char *refname,
++ const char *pointee_name,
++ const char *pointee_path)
+{
++ const char *p = NULL;
+ struct stat st;
+ int ret = 0;
+
-+ if (lstat(path->buf, &st) < 0) {
-+ ret = fsck_refs_report(o, refname->buf,
-+ FSCK_MSG_DANGLING_SYMREF,
-+ "point to non-existent ref");
++ if (!skip_prefix(pointee_name, "refs/", &p)) {
++
++ ret = fsck_refs_report(o, refname,
++ FSCK_MSG_BAD_SYMREF_POINTEE,
++ "point to target out of refs hierarchy");
+ goto out;
+ }
+
++ if (check_refname_format(pointee_name, 0)) {
++ ret = fsck_refs_report(o, refname,
++ FSCK_MSG_BAD_SYMREF_POINTEE,
++ "point to invalid refname");
++ }
++
++ if (lstat(pointee_path, &st) < 0)
++ goto out;
++
+ if (!S_ISREG(st.st_mode) && !S_ISLNK(st.st_mode)) {
-+ ret = fsck_refs_report(o, refname->buf,
-+ FSCK_MSG_DANGLING_SYMREF,
-+ "point to invalid object");
++ ret = fsck_refs_report(o, refname,
++ FSCK_MSG_BAD_SYMREF_POINTEE,
++ "point to invalid target");
+ goto out;
+ }
+out:
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
+ const char *refs_check_dir,
+ struct dir_iterator *iter)
+{
-+ struct strbuf path = STRBUF_INIT,
-+ refname = STRBUF_INIT,
++ struct strbuf pointee_path = STRBUF_INIT,
+ ref_content = STRBUF_INIT,
-+ referent = STRBUF_INIT;
-+ unsigned int trailing = 0;
++ abs_gitdir = STRBUF_INIT,
++ referent = STRBUF_INIT,
++ refname = STRBUF_INIT;
++ const char *trailing = NULL;
+ int failure_errno = 0;
+ unsigned int type = 0;
+ struct object_id oid;
+ int ret = 0;
+
-+ strbuf_addbuf(&path, &iter->path);
+ strbuf_addf(&refname, "%s/%s", refs_check_dir, iter->relative_path);
+
+ /*
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
+ * of the destination object.
+ */
+ if (S_ISLNK(iter->st.st_mode)) {
-+ strbuf_strip_file_from_path(&path);
-+ ret = strbuf_readlink(&ref_content,
-+ iter->path.buf, iter->st.st_size);
-+ if (ret < 0) {
-+ ret = error_errno(_("could not read link '%s'"),
-+ iter->path.buf);
++ const char *pointee_name = NULL;
++
++ strbuf_add_real_path(&pointee_path, iter->path.buf);
++
++ strbuf_add_absolute_path(&abs_gitdir, gitdir);
++ strbuf_normalize_path(&abs_gitdir);
++ if (!is_dir_sep(abs_gitdir.buf[abs_gitdir.len - 1]))
++ strbuf_addch(&abs_gitdir, '/');
++
++ if (!skip_prefix(pointee_path.buf,
++ abs_gitdir.buf, &pointee_name)) {
++ ret = fsck_refs_report(o, refname.buf,
++ FSCK_MSG_BAD_SYMREF_POINTEE,
++ "point to target outside gitdir");
+ goto clean;
+ }
-+ strbuf_addbuf(&path, &ref_content);
-+ strbuf_reset(&ref_content);
+
-+ ret = files_fsck_symref(o, &refname, &path);
++ ret = files_fsck_symref_target(o, refname.buf, pointee_name,
++ pointee_path.buf);
+ goto clean;
+ }
+
-+ if (strbuf_read_file(&ref_content, path.buf, 0) < 0) {
++ if (strbuf_read_file(&ref_content, iter->path.buf, 0) < 0) {
+ ret = error_errno(_("%s/%s: unable to read the ref"),
+ refs_check_dir, iter->relative_path);
+ goto clean;
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
+ &referent, &type,
+ &failure_errno, &trailing)) {
+ ret = fsck_refs_report(o, refname.buf,
-+ FSCK_MSG_BAD_REF_SHA,
++ FSCK_MSG_BAD_REF_CONTENT,
+ "invalid ref content");
+ goto clean;
+ }
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
+ * connectivity.
+ */
+ if (referent.len && (type & REF_ISSYMREF)) {
-+ strbuf_reset(&path);
-+ strbuf_addf(&path, "%s/%s", gitdir, referent.buf);
-+
-+ if (check_refname_format(referent.buf, 0)) {
-+ ret = fsck_refs_report(o, refname.buf,
-+ FSCK_MSG_DANGLING_SYMREF,
-+ "point to invalid refname");
-+ goto clean;
-+ }
++ strbuf_addf(&pointee_path, "%s/%s", gitdir, referent.buf);
++ strbuf_rtrim(&referent);
+
-+ ret = files_fsck_symref(o, &refname, &path);
++ ret = files_fsck_symref_target(o, refname.buf, referent.buf,
++ pointee_path.buf);
+ goto clean;
+ } else {
+ /*
+ * Only regular refs could have a trailing garbage. Should
+ * be reported as a warning.
+ */
-+ if (trailing) {
++ if (trailing && (*trailing != '\0' && *trailing != '\n')) {
+ ret = fsck_refs_report(o, refname.buf,
+ FSCK_MSG_TRAILING_REF_CONTENT,
+ "trailing garbage in ref");
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
+ }
+
+clean:
-+ strbuf_release(&path);
++ strbuf_release(&abs_gitdir);
++ strbuf_release(&pointee_path);
+ strbuf_release(&refname);
+ strbuf_release(&ref_content);
+ strbuf_release(&referent);
@@ refs/refs-internal.h: struct ref_store {
/*
* Parse contents of a loose ref file. *failure_errno maybe be set to EINVAL for
- * invalid contents.
-+ * invalid contents. Also trailing is set to 1 when there is any bytes after the
-+ * hex.
++ * invalid contents. Also *trailing is set to the first character after the
++ * refname or NULL if the referent is not empty.
*/
int parse_loose_ref_contents(const char *buf, struct object_id *oid,
struct strbuf *referent, unsigned int *type,
- int *failure_errno);
-+ int *failure_errno, unsigned int *trailing);
++ int *failure_errno, const char **trailing);
/*
* Fill in the generic part of refs and add it to our collection of
@@ t/t0602-reffiles-fsck.sh: test_expect_success 'ref name check should be adapted
+ printf "xfsazqfxcadas" > $tag_dir_prefix/tag-2-bad &&
+ test_must_fail git refs verify 2>err &&
+ cat >expect <<-EOF &&
-+ error: refs/tags/tag-2-bad: badRefSha: invalid ref content
++ error: refs/tags/tag-2-bad: badRefContent: invalid ref content
+ EOF
+ rm $tag_dir_prefix/tag-2-bad &&
+ test_cmp expect err
@@ t/t0602-reffiles-fsck.sh: test_expect_success 'ref name check should be adapted
+ printf "xfsazqfxcadas" > $branch_dir_prefix/a/b/branch-2-bad &&
+ test_must_fail git refs verify 2>err &&
+ cat >expect <<-EOF &&
-+ error: refs/heads/a/b/branch-2-bad: badRefSha: invalid ref content
++ error: refs/heads/a/b/branch-2-bad: badRefContent: invalid ref content
+ EOF
+ rm $branch_dir_prefix/a/b/branch-2-bad &&
+ test_cmp expect err
@@ t/t0602-reffiles-fsck.sh: test_expect_success 'ref name check should be adapted
+ ) &&
+ (
+ cd repo &&
-+ printf "ref: refs/heads/branch-3" > $branch_dir_prefix/branch-2-bad &&
-+ git refs verify 2>err &&
++ printf "ref: refs/heads/.branch" > $branch_dir_prefix/branch-2-bad &&
++ test_must_fail git refs verify 2>err &&
+ cat >expect <<-EOF &&
-+ warning: refs/heads/branch-2-bad: danglingSymref: point to non-existent ref
++ error: refs/heads/branch-2-bad: badSymrefPointee: point to invalid refname
+ EOF
+ rm $branch_dir_prefix/branch-2-bad &&
+ test_cmp expect err
+ ) &&
+ (
+ cd repo &&
-+ printf "ref: refs/heads/.branch" > $branch_dir_prefix/branch-2-bad &&
-+ git refs verify 2>err &&
++ printf "ref: refs/heads" > $branch_dir_prefix/branch-2-bad &&
++ test_must_fail git refs verify 2>err &&
+ cat >expect <<-EOF &&
-+ warning: refs/heads/branch-2-bad: danglingSymref: point to invalid refname
++ error: refs/heads/branch-2-bad: badSymrefPointee: point to invalid target
+ EOF
+ rm $branch_dir_prefix/branch-2-bad &&
+ test_cmp expect err
+ ) &&
+ (
+ cd repo &&
-+ printf "ref: refs/heads" > $branch_dir_prefix/branch-2-bad &&
-+ git refs verify 2>err &&
++ printf "ref: logs/maint-v2.45" > $branch_dir_prefix/branch-2-bad &&
++ test_must_fail git refs verify 2>err &&
+ cat >expect <<-EOF &&
-+ warning: refs/heads/branch-2-bad: danglingSymref: point to invalid object
++ error: refs/heads/branch-2-bad: badSymrefPointee: point to target out of refs hierarchy
+ EOF
+ rm $branch_dir_prefix/branch-2-bad &&
+ test_cmp expect errSignature · 1 line
2.45.2
Quoted 6 lines · shejialuo
This is the third version. Mainly focuses on the following problems:
1. Add "escape" check for symrefs and symbolic links.
2. Resolve the reviews.
3. Fix some errors in version 2.
Quoted 8 lines
shejialuo (7):
fsck: add refs check interfaces to interface with fsck error levels
refs: set up ref consistency check infrastructure builtin/refs: add verify subcommand
builtin/fsck: addgit-refs verifychild process
files-backend: add unified interface for refs scanning
fsck: add ref name check for files backend
fsck: add ref content check for files backend
What commit is this series based on? It does not seem to apply cleanly on top of the same base as I queued an earlier iteration of this series (f931bfb2 (Merge branch 'ps/ref-storage-migration' into sj/ref-fsck, 2024-06-13)), or the current tip of 'master' that already merges that prerequisite topic. The early 6 applies on top of 'master' fine, but the last one. I wiggled the last one in (the patch to the fsck-msgids.txt documentation expects stale entries that were already removed in 2.45.2), but if you are basing your patches on an unusual commit, please describe it clearly here in the cover letter so that others can locate (or recreate as needed) it.
Quoted 18 lines · Junio C Hamano
Quoted 6 lines · shejialuo
This is the third version. Mainly focuses on the following problems:
1. Add "escape" check for symrefs and symbolic links.
2. Resolve the reviews.
3. Fix some errors in version 2.Quoted 8 lines
shejialuo (7):
fsck: add refs check interfaces to interface with fsck error levels
refs: set up ref consistency check infrastructure builtin/refs: add verify subcommand
builtin/fsck: addgit-refs verifychild process
files-backend: add unified interface for refs scanning
fsck: add ref name check for files backend
fsck: add ref content check for files backend
Quoted 10 lines
What commit is this series based on? It does not seem to apply cleanly on top of the same base as I queued an earlier iteration of this series (
f931bfb2(Merge branch 'ps/ref-storage-migration' into sj/ref-fsck, 2024-06-13)), or the current tip of 'master' that already merges that prerequisite topic. The early 6 applies on top of 'master' fine, but the last one. I wiggled the last one in (the patch to the fsck-msgids.txt documentation expects stale entries that were already removed in 2.45.2), but if you are basing your patches on an unusual commit, please describe it clearly here in the cover letter so that others can locate (or recreate as needed) it.
Well, I simply does "git checkout -b js-refs-fsck-v3" for the current branch "js-refs-fsck". And I use "git rebase" to change the code in this branch. So the base commit should be unchanged. However, maybe Patrick has changed the code and I forgot to update upstream.
And I have set up the base using the following commands:
$ git remote add junio https://github.com/gister/git.git $ git fetch junio $ git branch js-refs-fsck junio/ps/ref-storage-migration
I am sorry for the confusion. Next version, I will merge the master code and send the patch. I am sorry to make you so inconvenient.
PATCH v3 1/7 fsck: add refs check interfaces to interface with fsck error levels
In reply to shejialuoThe git-fsck(1) focuses on object database consistency check. It relies on the "fsck_options" to interact with fsck error levels. However, "fsck_options" aims at checking the object database which makes it unsuitable to change the semantics of it. Instead, create "fsck_refs_options" structure to handle refs consistency check.
The "git_fsck_config" sets up the "msg_type" and "skiplist" member of the "fsck_options". For refs, we just need the "msg_type". In order to allow setting up more refs-specific options easily later, add a separate function "git_fsck_refs_config" to initialize the refs-specific options.
Move the "msg_type" and "strict" member to the top of the "fsck_options" which allows us to convert "fsck_refs_options *" to "fsck_options *" to reuse the interfaces provided by "fsck.h" without changing the original code.
The static function "report" provided by "fsck.c" aims at reporting the problems related to object database which cannot be reused for refs. Provide "fsck_refs_report" function to integrate the fsck error levels into reference consistency check.
--- fsck.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ fsck.h | 45 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 112 insertions(+), 2 deletions(-)
fsck.c +69 -0
@@ -1249,6 +1249,20 @@ int fsck_buffer(const struct object_id *oid, enum object_type type, type); } +int fsck_refs_error_function(struct fsck_refs_options *o UNUSED, +const char *name, +enum fsck_msg_type msg_type, + enum fsck_msg_id msg_id UNUSED, +const char *message) +{ +if (msg_type ==FSCK_WARN ) { +warning ("%s: %s" , name, message); +return 0 ; + } +error ("%s: %s" , name, message); +return 1 ; +} + int fsck_error_function(struct fsck_options *o, const struct object_id *oid, enum object_type object_type UNUSED,
@@ -1323,6 +1337,61 @@ int fsck_finish(struct fsck_options *options)return ret; } +int fsck_refs_report (struct fsck_refs_options *o, +const char *name, +enum fsck_msg_id msg_id, +const char *fmt, ...) +{ +va_list ap; +struct strbuf sb =STRBUF_INIT ; +enum fsck_msg_type msg_type = +fsck_msg_type (msg_id, (struct fsck_options *)o); +int ret =0 ; + +if (msg_type ==FSCK_IGNORE ) +return 0 ; + +if (msg_type ==FSCK_FATAL ) + msg_type =FSCK_ERROR ; +else if (msg_type ==FSCK_INFO ) + msg_type =FSCK_WARN ; + +prepare_msg_ids (); +strbuf_addf (&sb,"%s: " , msg_id_info[msg_id].camelcased); + +va_start (ap, fmt); +strbuf_vaddf (&sb, fmt, ap); + ret = o->error_func (o, name, msg_type, msg_id, sb.buf); +strbuf_release (&sb); +va_end (ap); + +return ret; +} + +int git_fsck_refs_config (const char *var,const char *value, +const struct config_context *ctx,void *cb) +{ +struct fsck_refs_options *options = cb; +const char *msg_id; + +/* +* We don't check the value of fsck.skiplist here, because it +* is specific to object database, not reference database. +*/ +if (strcmp (var,"fsck.skiplist" ) ==0 ) { +return 0 ; + } + +if (skip_prefix (var,"fsck." , &msg_id)) { +if (!value) +return config_error_nonbool (var); +fsck_set_msg_type ((struct fsck_options *)options, msg_id, value); +return 0 ; + } + +return git_default_config (var, value, ctx, cb); +} +int git_fsck_config (const char *var,const char *value,const struct config_context *ctx,void *cb) {
fsck.h +43 -2
78 lines not on this page · raw message
Signature · 1 line
2.45.2
Re: [GSoC][PATCH v3 1/7] fsck: add refs check interfaces to interface with fsck error levels
shejialuo <[email protected]> writes:
[snip]
fsck.h int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o, · line 170
Quoted 5 lines
struct
fsck_options{fsck.h +4
-0+/* +* Reorder the fields to allow `fsck_ref_options` to use +* the interfaces using `struct fsck_options`. +*/
Why is this added? It makes sense to have it in the commit message because it talks about the change, but why make it persistent in the code?
[snip]
Re: [GSoC][PATCH v3 1/7] fsck: add refs check interfaces to interface with fsck error levels
Quoted 4 lines · Karthik Nayak
shejialuo <[email protected]> writes:
[snip]
Quoted 10 lines
fsck.h int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o, · line 170
Quoted 5 lines
struct
fsck_options{fsck.h +4
-0+/* +* Reorder the fields to allow `fsck_ref_options` to use +* the interfaces using `struct fsck_options`. +*/ Why is this added? It makes sense to have it in the commit message because it talks about the change, but why make it persistent in the code?
I explicitly add this comments due to the following reasons:
1. If someone needs to change the fsck_options, without this comment, he might be just add some new fields at top. Although the change will fail the tests here, I think we should mention this in code.
2. My later intention is that we should extract these common fields out of the fsck_options and fsck_ref_options.
Quoted 1 line
[snip]
Re: [GSoC][PATCH v3 1/7] fsck: add refs check interfaces to interface with fsck error levels
Quoted 16 lines · shejialuo
Quoted 4 lines · Karthik Nayak
shejialuo <[email protected]> writes:
[snip]
Quoted 10 lines
fsck.h int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o, · line 170
Quoted 5 lines
struct
fsck_options{fsck.h +4
-0+/* +* Reorder the fields to allow `fsck_ref_options` to use +* the interfaces using `struct fsck_options`. +*/ Why is this added? It makes sense to have it in the commit message because it talks about the change, but why make it persistent in the code?
Quoted 5 lines
I explicitly add this comments due to the following reasons:
1. If someone needs to change the
fsck_options, without this comment, he might be just add some new fields at top. Although the change will fail the tests here, I think we should mention this in code.
Do you mean you plan to take advantage of the fact that early members of two structures are the same? IOW, if there is a function that takes a pointer to smaller fsck_refs_options, you plan to pass a pointer to fsck_options from some callers, e.g.
extern void func(struct fsck_refs_options *);
void a_caller(struct fsck_options *o)
{
func((struct fsck_options *)o);
...If that is the case, then ...
Do not do that.
Your data structure design is broken. Instead you would do this:
struct fsck_options {
struct fsck_refs_options refs;
... other members ...
};
void a_caller(struct fsck_options *o)
{
func(&o->refs);
...Re: [GSoC][PATCH v3 1/7] fsck: add refs check interfaces to interface with fsck error levels
Quoted 30 lines · Junio C Hamano
Quoted 16 lines · shejialuo
Quoted 10 lines · Karthik Nayak
shejialuo <[email protected]> writes:
[snip]
fsck.h int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o, · line 170
Quoted 5 lines
struct
fsck_options{fsck.h +4
-0+/* +* Reorder the fields to allow `fsck_ref_options` to use +* the interfaces using `struct fsck_options`. +*/ Quoted 4 lines
Why is this added? It makes sense to have it in the commit message because it talks about the change, but why make it persistent in the code?
Quoted 5 lines
I explicitly add this comments due to the following reasons:
1. If someone needs to change the
fsck_options, without this comment, he might be just add some new fields at top. Although the change will fail the tests here, I think we should mention this in code.Do you mean you plan to take advantage of the fact that early members of two structures are the same? IOW, if there is a function that takes a pointer to smaller
fsck_refs_options, you plan to pass a pointer tofsck_optionsfrom some callers, e.g.
Quoted 6 lines
extern void func(struct fsck_refs_options *);
void a_caller(struct fsck_options *o)
{
func((struct fsck_options *)o);
...I do not want to convert "struct fsck_options*" to "struct fsck_refs_options*". Instead, I want to convert "struct fsck_refs_options*" to "struct fsck_options*" to reuse the functions which use the "struct fsck_options*" parameter.
Like the commit message said:
Move the "msg_type" and "strict" member to the top of the "fsck_options" which allows us to convert "fsck_refs_options *" to "fsck_options *" to reuse the interfaces provided by "fsck.h" without changing the original code.
It may seem we should add some fields into fsck_options, but I don't think it's a good idea. I will elaborate my design here:
The git-fsck(1) is highly relevant with the object database. I don't want to add some new fields into "fsck_options" due to the following reason:
The fields in fsck_options are all related to objects except "fsck_msg_type" and "strict". Adding some ref-related fields into "fsck_options" will break the semantics.
Actually, it may be perfect that I may abstract some interfaces here, however, it would be too complicated for this patch, many functions use "fsck_options *" as the parameter. I think we may do this later.
Re: [GSoC][PATCH v3 1/7] fsck: add refs check interfaces to interface with fsck error levels
In reply to Junio C HamanoQuoted 41 lines · Junio C Hamano
Quoted 16 lines · shejialuo
Quoted 4 lines · Karthik Nayak
shejialuo <[email protected]> writes:
[snip]
Quoted 10 lines
fsck.h int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o, · line 170
Quoted 5 lines
struct
fsck_options{fsck.h +4
-0+/* +* Reorder the fields to allow `fsck_ref_options` to use +* the interfaces using `struct fsck_options`. +*/ Why is this added? It makes sense to have it in the commit message because it talks about the change, but why make it persistent in the code?
Quoted 5 lines
I explicitly add this comments due to the following reasons:
1. If someone needs to change the
fsck_options, without this comment, he might be just add some new fields at top. Although the change will fail the tests here, I think we should mention this in code.Do you mean you plan to take advantage of the fact that early members of two structures are the same? IOW, if there is a function that takes a pointer to smaller
fsck_refs_options, you plan to pass a pointer tofsck_optionsfrom some callers, e.g.extern void func(struct fsck_refs_options *); void a_caller(struct fsck_options *o) { func((struct fsck_options *)o); ...If that is the case, then ...
Do not do that.
Your data structure design is broken. Instead you would do this:
Quoted 8 lines
struct fsck_options {
struct fsck_refs_options refs;
... other members ...
};
void a_caller(struct fsck_options *o)
{
func(&o->refs);
...Well, I totally agree with this. It's bad to convert the pointer type. I will change the code in this patch to make it OK.
Thanks for your advice.
PATCH v3 2/7 refs: set up ref consistency check infrastructure
In reply to shejialuoThe interfaces defined in the ref_storage_be are carefully structured in semantic. It's organized as the five parts:
1. The name and the initialization interfaces.
2. The ref transaction interfaces.
3. The ref internal interfaces (pack, rename and copy).
4. The ref filesystem interfaces.
5. The reflog related interfaces.
To keep consistent with the git-fsck(1), add a new interface named "fsck_refs_fn" to the end of "ref_storage_be". This semantic cannot be grouped into any above five categories. Explicitly add blank line to make it different from others.
--- refs.c | 5 +++++ refs.h | 8 ++++++++ refs/debug.c | 11 +++++++++++ refs/files-backend.c | 15 ++++++++++++++- refs/packed-backend.c | 8 ++++++++ refs/refs-internal.h | 6 ++++++ refs/reftable-backend.c | 8 ++++++++ 7 files changed, 60 insertions(+), 1 deletion(-)
refs.c +5 -0
@@ -316,6 +316,11 @@ int check_refname_format(const char *refname, int flags)return check_or_sanitize_refname (refname, flags,NULL ); } +int refs_fsck (struct ref_store *refs,struct fsck_refs_options *o) +{ +return refs->be->fsck (refs, o); +} +void sanitize_refname_component (const char *refname,struct strbuf *out) {if (check_or_sanitize_refname (refname,REFNAME_ALLOW_ONELEVEL , out))
refs.h +8 -0
@@ -3,6 +3,7 @@#include "commit.h" +struct fsck_refs_options ;struct object_id ;struct ref_store ;struct repository ;
@@ -547,6 +548,13 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat*/ int check_refname_format (const char *refname,int flags); +/* +* Check the reference database for consistency. Return 0 if refs and +* reflogs are consistent, and non-zero otherwise. The errors will be +* written to stderr. +*/ +int refs_fsck (struct ref_store *refs,struct fsck_refs_options *o); +/* * Apply the rules from check_refname_format, but mutate the result until it * is acceptable, and place the result in "out".
refs/debug.c +11 -0
@@ -419,6 +419,15 @@ static int debug_reflog_expire(struct ref_store *ref_store, const char *refname,return res; } +static int debug_fsck (struct ref_store *ref_store, +struct fsck_refs_options *o) +{ +struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store; +int res = drefs->refs->be->fsck (drefs->refs, o); +trace_printf_key (&trace_refs,"fsck: %d\n" , res); +return res; +} +struct ref_storage_be refs_be_debug = { .name ="debug" , .init =NULL ,
@@ -451,4 +460,6 @@ struct ref_storage_be refs_be_debug = { .create_reflog = debug_create_reflog, .delete_reflog = debug_delete_reflog, .reflog_expire = debug_reflog_expire, + + .fsck = debug_fsck, };
refs/files-backend.c +14 -1
@@ -3402,6 +3402,17 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,return ret; } +static int files_fsck (struct ref_store *ref_store, +struct fsck_refs_options *o) +{ +int ret; +struct files_ref_store *refs = +files_downcast (ref_store,REF_STORE_READ ,"fsck" ); + + ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o); +return ret; +} +struct ref_storage_be refs_be_files = { .name ="files" , .init = files_ref_store_init,
@@ -3428,5 +3439,7 @@ struct ref_storage_be refs_be_files = { .reflog_exists = files_reflog_exists, .create_reflog = files_create_reflog, .delete_reflog = files_delete_reflog, - .reflog_expire = files_reflog_expire + .reflog_expire = files_reflog_expire, + + .fsck = files_fsck, };
refs/packed-backend.c +8 -0
@@ -1733,6 +1733,12 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_sreturn empty_ref_iterator_begin (); } +static int packed_fsck (struct ref_store *ref_store, +struct fsck_refs_options *o) +{ +return 0 ; +} +struct ref_storage_be refs_be_packed = { .name ="packed" , .init = packed_ref_store_init,
@@ -1760,4 +1766,6 @@ struct ref_storage_be refs_be_packed = { .create_reflog = NULL, .delete_reflog = NULL, .reflog_expire = NULL, + + .fsck = packed_fsck, };
refs/refs-internal.h +6 -0
@@ -4,6 +4,7 @@#include "refs.h" #include "iterator.h" +struct fsck_refs_options ;struct ref_transaction ;/*
@@ -650,6 +651,9 @@ typedef int read_raw_ref_fn(struct ref_store *ref_store, const char *refname,typedef int read_symbolic_ref_fn (struct ref_store *ref_store,const char *refname,struct strbuf *referent); +typedef int fsck_fn (struct ref_store *ref_store, +struct fsck_refs_options *o); +struct ref_storage_be {const char *name;ref_store_init_fn *init;
@@ -677,6 +681,8 @@ struct ref_storage_be {create_reflog_fn *create_reflog;delete_reflog_fn *delete_reflog;reflog_expire_fn *reflog_expire; + +fsck_fn *fsck; };extern struct ref_storage_be refs_be_files;
refs/reftable-backend.c +8 -0
@@ -2242,6 +2242,12 @@ static int reftable_be_reflog_expire(struct ref_store *ref_store,return ret; } +static int reftable_be_fsck (struct ref_store *ref_store, +struct fsck_refs_options *o) +{ +return 0 ; +} +struct ref_storage_be refs_be_reftable = { .name ="reftable" , .init = reftable_be_init,
@@ -2269,4 +2275,6 @@ struct ref_storage_be refs_be_reftable = { .create_reflog = reftable_be_create_reflog, .delete_reflog = reftable_be_delete_reflog, .reflog_expire = reftable_be_reflog_expire, + + .fsck = reftable_be_fsck, };
Signature · 1 line
2.45.2
PATCH v3 3/7 builtin/refs: add verify subcommand
In reply to shejialuoIntroduce a new subcommand "verify" in git-refs(1) to allow the user to check the reference database consistency.
--- Documentation/git-refs.txt | 13 +++++++++++ builtin/refs.c | 45 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+)
Documentation/git-refs.txt +13 -0
@@ -10,6 +10,7 @@ SYNOPSIS -------- [verse] 'git refs migrate' --ref-format=<format> [--dry-run] +'git refs verify' [--strict] [--verbose] DESCRIPTION -----------
@@ -22,6 +23,9 @@ COMMANDS migrate:: Migrate ref store between different formats. +verify:: + Verify reference database consistency. + OPTIONS -------
@@ -39,6 +43,15 @@ include::ref-storage-format.txt[] can be used to double check that the migration works as expected before performing the actual migration. +The following options are specific to 'git refs verify': + +--strict:: + Enable more strict checking, every WARN severity for the `Fsck Messages` + be seen as ERROR. + +--verbose:: + When verifying the reference database consistency, be chatty. + KNOWN LIMITATIONS -----------------
builtin/refs.c +45 -0
70 lines not on this page · raw message
Signature · 1 line
2.45.2
Re: [PATCH v3 3/7] builtin/refs: add verify subcommand
Quoted 34 lines · shejialuo
Introduce a new subcommand "verify" in git-refs(1) to allow the user to check the reference database consistency.
Mentored-by Patrick Steinhardt <[email protected]>Mentored-by Karthik Nayak <[email protected]>Signed-off-by shejialuo <[email protected]>--- Documentation/git-refs.txt | 13 +++++++++++ builtin/refs.c | 45 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+)Documentation/git-refs.txt +4
-0@@ -10,6 +10,7 @@ SYNOPSIS -------- [verse] 'git refs migrate' --ref-format=<format> [--dry-run] +'git refs verify' [--strict] [--verbose] DESCRIPTION -----------@@ -22,6 +23,9 @@ COMMANDS migrate:: Migrate ref store between different formats. +verify:: + Verify reference database consistency. + OPTIONS -------
Documentation/git-refs.txt include::ref-storage-format.txt[] · line 51
Quoted 9 lines
Documentation/git-refs.txt +6
-0@@ -39,6 +43,15 @@ include::ref-storage-format.txt[] can be used to double check that the migration works as expected before performing the actual migration. +The following options are specific to 'git refs verify': + +--strict:: + Enable more strict checking, every WARN severity for the `Fsck Messages` + be seen as ERROR. +
Nit: It would be nice to redirect users to the fsck documentation to understand severity levels.
[snip]
Re: [PATCH v3 3/7] builtin/refs: add verify subcommand
Quoted 39 lines · Karthik Nayak
Quoted 37 lines · shejialuo
Introduce a new subcommand "verify" in git-refs(1) to allow the user to check the reference database consistency.
Mentored-by Patrick Steinhardt <[email protected]>Mentored-by Karthik Nayak <[email protected]>Signed-off-by shejialuo <[email protected]>--- Documentation/git-refs.txt | 13 +++++++++++ builtin/refs.c | 45 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+)Documentation/git-refs.txt +4
-0@@ -10,6 +10,7 @@ SYNOPSIS -------- [verse] 'git refs migrate' --ref-format=<format> [--dry-run] +'git refs verify' [--strict] [--verbose] DESCRIPTION -----------@@ -22,6 +23,9 @@ COMMANDS migrate:: Migrate ref store between different formats. +verify:: + Verify reference database consistency. + OPTIONS -------@@ -39,6 +43,15 @@ include::ref-storage-format.txt[] can be used to double check that the migration works as expected before performing the actual migration.
Quoted 10 lines
Documentation/git-refs.txt include::ref-storage-format.txt[] · line 51
Quoted 6 lines
Documentation/git-refs.txt +6
-0+The following options are specific to 'git refs verify': + +--strict:: + Enable more strict checking, every WARN severity for the `Fsck Messages` + be seen as ERROR. +Nit: It would be nice to redirect users to the fsck documentation to understand severity levels.
I agree. Will improve this in the next version.
Quoted 1 line
[snip]
PATCH v3 4/7 builtin/fsck: add `git-refs verify` child process
In reply to shejialuoIntroduce a new function "fsck_refs" that initializes and runs a child process to execute the "git-refs verify" command.
--- builtin/fsck.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+)
builtin/fsck.c +17 -0
@@ -896,6 +896,21 @@ static int check_pack_rev_indexes(struct repository *r, int show_progress)return res; } +static void fsck_refs (void ) +{ +struct child_process refs_verify =CHILD_PROCESS_INIT ; +child_process_init (&refs_verify); + refs_verify.git_cmd =1 ; +strvec_pushl (&refs_verify.args,"refs" ,"verify" ,NULL ); +if (verbose) +strvec_push (&refs_verify.args,"--verbose" ); +if (check_strict) +strvec_push (&refs_verify.args,"--strict" ); + +if (run_command (&refs_verify)) + errors_found |=ERROR_REFS ; +} +static char const *const fsck_usage[] = { N_("git fsck [--tags] [--root] [--unreachable] [--cache] [--no-reflogs]\n" " [--[no-]full] [--strict] [--verbose] [--lost-found]\n"
@@ -1065,6 +1080,8 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)check_connectivity (); +fsck_refs (); +if (the_repository->settings.core_commit_graph) {struct child_process commit_graph_verify =CHILD_PROCESS_INIT ;
Signature · 1 line
2.45.2
PATCH v3 5/7 files-backend: add unified interface for refs scanning
In reply to shejialuoFor refs and reflogs, we need to scan its corresponding directories to check every regular file or symbolic link which shares the same pattern. Introduce a unified interface for scanning directories for files-backend.
--- refs/files-backend.c | 77 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-)
refs/files-backend.c +76 -1
@@ -4,6 +4,7 @@#include "../gettext.h" #include "../hash.h" #include "../hex.h" +#include "../fsck.h" #include "../refs.h" #include "refs-internal.h" #include "ref-cache.h"
@@ -3402,6 +3403,78 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,return ret; } +/* +* For refs and reflogs, they share a unified interface when scanning +* the whole directory. This function is used as the callback for each +* regular file or symlink in the directory. +*/ +typedef int (*files_fsck_refs_fn )(struct fsck_refs_options *o, +const char *gitdir, +const char *refs_check_dir, +struct dir_iterator *iter); + +static int files_fsck_refs_dir (struct ref_store *ref_store, +struct fsck_refs_options *o, +const char *refs_check_dir, +files_fsck_refs_fn *fsck_refs_fns) +{ +const char *gitdir = ref_store->gitdir; +struct strbuf sb =STRBUF_INIT ; +struct dir_iterator *iter; +int iter_status; +int ret =0 ; + +strbuf_addf (&sb,"%s/%s" , gitdir, refs_check_dir); + + iter =dir_iterator_begin (sb.buf,0 ); + +if (!iter) { + ret =error_errno ("cannot open directory %s" , sb.buf); + goto out; + } + +while ((iter_status =dir_iterator_advance (iter)) ==ITER_OK ) { +if (S_ISDIR (iter->st.st_mode)) { +continue ; + }else if (S_ISREG (iter->st.st_mode) || +S_ISLNK (iter->st.st_mode)) { +if (o->verbose) +fprintf_ln (stderr,"Checking %s/%s" , + refs_check_dir, iter->relative_path); +for (size_t i =0 ; fsck_refs_fns[i]; i++) { +if (fsck_refs_fns[i](o, gitdir, refs_check_dir, iter)) + ret =-1 ; + } + }else { + ret =error (_ ("unexpected file type for '%s'" ), + iter->basename); + } + } + +if (iter_status !=ITER_DONE ) + ret =error (_ ("failed to iterate over '%s'" ), sb.buf); + +out: +strbuf_release (&sb); +return ret; +} + +static int files_fsck_refs (struct ref_store *ref_store, +struct fsck_refs_options *o) +{ +int ret; +files_fsck_refs_fn fsck_refs_fns[]= { +NULL + }; + +if (o->verbose) +fprintf_ln (stderr,"Checking references consistency" ); + + ret =files_fsck_refs_dir (ref_store, o,"refs" , fsck_refs_fns); + +return ret; +} +static int files_fsck (struct ref_store *ref_store,struct fsck_refs_options *o) {
@@ -3409,7 +3482,9 @@ static int files_fsck(struct ref_store *ref_store,struct files_ref_store *refs =files_downcast (ref_store,REF_STORE_READ ,"fsck" ); - ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o); + ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o) + |files_fsck_refs (ref_store, o); +return ret; }
Signature · 1 line
2.45.2
PATCH v3 6/7 fsck: add ref name check for files backend
In reply to shejialuoThe git-fsck(1) only implicitly checks the reference, it does not fully check refs with bad format name such as standalone "@" and name ending with ".lock".
In order to provide such checks, add a new fsck message id "badRefName" with default ERROR type. Use existing "check_refname_format" to explicit check the ref name. And add a new unit test to verify the functionality.
--- Documentation/fsck-msgids.txt | 3 + fsck.h | 1 + refs/files-backend.c | 20 +++++++ t/t0602-reffiles-fsck.sh | 101 ++++++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+) create mode 100755 t/t0602-reffiles-fsck.sh
Documentation/fsck-msgids.txt +3 -0
@@ -19,6 +19,9 @@ `badParentSha1`:: (ERROR) A commit object has a bad parent sha1. +`badRefName`:: + (ERROR) A ref has a bad name. + `badTagName`:: (INFO) A tag has an invalid format.
fsck.h +1 -0
@@ -31,6 +31,7 @@ enum fsck_msg_type { FUNC(BAD_NAME, ERROR) \ FUNC(BAD_OBJECT_SHA1, ERROR) \ FUNC(BAD_PARENT_SHA1, ERROR) \ + FUNC(BAD_REF_NAME, ERROR) \ FUNC(BAD_TIMEZONE, ERROR) \ FUNC(BAD_TREE, ERROR) \FUNC (BAD_TREE_SHA1 ,ERROR ) \
refs/files-backend.c +20 -0
@@ -3413,6 +3413,25 @@ typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o,const char *refs_check_dir, struct dir_iterator *iter); +static int files_fsck_refs_name (struct fsck_refs_options *o, + const char *gitdir UNUSED, +const char *refs_check_dir, +struct dir_iterator *iter) +{ +struct strbuf sb =STRBUF_INIT ; +int ret =0 ; + +if (check_refname_format (iter->basename,REFNAME_ALLOW_ONELEVEL )) { +strbuf_addf (&sb,"%s/%s" , refs_check_dir, iter->relative_path); + ret =fsck_refs_report (o, sb.buf, +FSCK_MSG_BAD_REF_NAME , +"invalid refname format" ); + } + +strbuf_release (&sb); +return ret; +} + static int files_fsck_refs_dir(struct ref_store *ref_store, struct fsck_refs_options *o, const char *refs_check_dir,
@@ -3464,6 +3483,7 @@ static int files_fsck_refs(struct ref_store *ref_store, {int ret;files_fsck_refs_fn fsck_refs_fns[]= { + files_fsck_refs_name,NULL };
t/t0602-reffiles-fsck.sh new file +101 -0
101 lines not on this page · raw message
Signature · 1 line
2.45.2
PATCH v3 7/7 fsck: add ref content check for files backend
In reply to shejialuoEnhance the git-fsck(1) command by adding a check for reference content in the files backend. The new functionality ensures that symrefs, real symbolic link and regular refs are validated correctly.
In order to check the trailing content of the regular refs, add a new parameter trailing to parse_loose_ref_contents.
For symrefs, parse_loose_ref_contents will set the "referent". However, symbolic link could be either absolute or relative. Use "strbuf_add_real_path" to read the symbolic link and convert the relative path to absolute path. Then use "skip_prefix" to make it align with symref "referent".
Thus, the symrefs and symbolic links could share the same interface. Add a new function "files_fsck_symref_target" which aims at checking the following things:
1. whether the pointee is under the refs/ directory.
2. whether the pointee name is correct.
3. whether the pointee path is a wrong type in filesystem.
Last, add the following FSCK MESSAGEs:
1. "badRefContent(ERROR)": A ref has a bad content
2. "badSymrefPointee(ERROR)": The pointee of a symref is bad.
3. "trailingRefContent(WARN)": A ref content has trailing contents.
--- Documentation/fsck-msgids.txt | 9 +++ fsck.h | 3 + refs.c | 2 +- refs/files-backend.c | 145 +++++++++++++++++++++++++++++++++- refs/refs-internal.h | 5 +- t/t0602-reffiles-fsck.sh | 110 ++++++++++++++++++++++++++ 6 files changed, 269 insertions(+), 5 deletions(-)
Documentation/fsck-msgids.txt +9 -0
@@ -19,9 +19,15 @@ `badParentSha1`:: (ERROR) A commit object has a bad parent sha1. +`badRefContent`:: + (ERROR) A ref has a bad content. + `badRefName`:: (ERROR) A ref has a bad name. +`badSymrefPointee`:: + (ERROR) The pointee of a symref is bad. + `badTagName`:: (INFO) A tag has an invalid format.
@@ -179,6 +185,9 @@ `symlinkTargetMissing`:: (ERROR) Unable to read symbolic link target's blob. +`trailingRefContent`:: + (WARN) A ref content has trailing contents. + `treeNotSorted`:: (ERROR) A tree is not properly sorted.
fsck.h +3 -0
@@ -32,6 +32,8 @@ enum fsck_msg_type { FUNC(BAD_OBJECT_SHA1, ERROR) \ FUNC(BAD_PARENT_SHA1, ERROR) \ FUNC(BAD_REF_NAME, ERROR) \ + FUNC(BAD_REF_CONTENT, ERROR) \ + FUNC(BAD_SYMREF_POINTEE, ERROR) \ FUNC(BAD_TIMEZONE, ERROR) \ FUNC(BAD_TREE, ERROR) \FUNC (BAD_TREE_SHA1 ,ERROR ) \
@@ -74,6 +76,7 @@ enum fsck_msg_type { FUNC(HAS_DOTDOT, WARN) \ FUNC(HAS_DOTGIT, WARN) \ FUNC(NULL_SHA1, WARN) \ + FUNC(TRAILING_REF_CONTENT, WARN) \ FUNC(ZERO_PADDED_FILEMODE, WARN) \ FUNC(NUL_IN_COMMIT, WARN) \FUNC (LARGE_PATHNAME ,WARN ) \
refs.c +1 -1
@@ -1744,7 +1744,7 @@ static int refs_read_special_head(struct ref_store *ref_store, } result =parse_loose_ref_contents (content.buf, oid, referent, type, - failure_errno); + failure_errno,NULL ); done:strbuf_release (&full_path);
refs/files-backend.c +143 -2
179 lines not on this page · raw message
refs/refs-internal.h +3 -2
@@ -709,11 +709,12 @@ struct ref_store {/* * Parse contents of a loose ref file. *failure_errno maybe be set to EINVAL for -* invalid contents. +* invalid contents. Also *trailing is set to the first character after the +* refname or NULL if the referent is not empty. */ int parse_loose_ref_contents (const char *buf,struct object_id *oid,struct strbuf *referent,unsigned int *type, -int *failure_errno); +int *failure_errno,const char **trailing);/* * Fill in the generic part of refs and add it to our collection of
t/t0602-reffiles-fsck.sh +110 -0
114 lines not on this page · raw message
Signature · 1 line
2.45.2
v4 cover ref consistency check infra setup
In reply to shejialuoHi All:
This version mainly focuses on solving following reviews:
1. As Junio said, it's a bad idea to convert the "fsck_refs_options *" to "fsck_options *" or convert the "fsck_options *" to "fsck_refs_options *". So this version, I changed a lot for the first patch. I refactor the "fsck_options *" which contains refs-specific, objs-specific and common settings. This designs allows us to use combination to make the code much more clean.
2. As Karthik said, I should enhance documents for "git refs verify". This version does this.
And Junio found that the commit the previous series based on is strange. In order to make our life easier, I change the based on commit. Now it is based on the top master. And I will always keep the code align with the master to avoid strange things.
66ac6e4bcd (The fourteenth batch, 2024-06-17)
Thanks
shejialuo (7):
fsck: add refs check interfaces to interact with fsck error levels
refs: set up ref consistency check infrastructure builtin/refs: add verify subcommand
builtin/fsck: add git-refs verify child process
files-backend: add unified interface for refs scanning
fsck: add ref name check for files backend
fsck: add ref content check for files backend
Documentation/fsck-msgids.txt | 12 ++ Documentation/git-refs.txt | 13 ++ builtin/fsck.c | 25 +++- builtin/index-pack.c | 2 +- builtin/mktag.c | 5 +- builtin/refs.c | 45 ++++++ builtin/unpack-objects.c | 2 +- fetch-pack.c | 8 +- fsck.c | 113 +++++++++++---- fsck.h | 83 ++++++++--- object-file.c | 2 +- refs.c | 7 +- refs.h | 8 ++ refs/debug.c | 11 ++ refs/files-backend.c | 255 +++++++++++++++++++++++++++++++++- refs/packed-backend.c | 8 ++ refs/refs-internal.h | 11 +- refs/reftable-backend.c | 8 ++ t/t0602-reffiles-fsck.sh | 211 ++++++++++++++++++++++++++++ 19 files changed, 765 insertions(+), 64 deletions(-) create mode 100755 t/t0602-reffiles-fsck.sh
Range-diff against v3:
1: b8d20c4be9 < -: ---------- fsck: add refs check interfaces to interface with fsck error levels
-: ---------- > 1: 2d2c1f2cd3 fsck: add refs check interfaces to interact with fsck error levels
2: c7ed95d6b4 ! 2: 4bacffe44e refs: set up ref consistency check infrastructure
@@ refs.c: int check_refname_format(const char *refname, int flags)
return check_or_sanitize_refname(refname, flags, NULL);
}
-+int refs_fsck(struct ref_store *refs, struct fsck_refs_options *o)
++int refs_fsck(struct ref_store *refs, struct fsck_options *o)
+{
+ return refs->be->fsck(refs, o);
+}
@@ refs.h
#include "commit.h"
-+struct fsck_refs_options;
++struct fsck_options;
struct object_id;
struct ref_store;
struct repository;
@@ refs.h: int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void
+ * reflogs are consistent, and non-zero otherwise. The errors will be
+ * written to stderr.
+ */
-+int refs_fsck(struct ref_store *refs, struct fsck_refs_options *o);
++int refs_fsck(struct ref_store *refs, struct fsck_options *o);
+
/*
* Apply the rules from check_refname_format, but mutate the result until it
@@ refs/debug.c: static int debug_reflog_expire(struct ref_store *ref_store, const
}
+static int debug_fsck(struct ref_store *ref_store,
-+ struct fsck_refs_options *o)
++ struct fsck_options *o)
+{
+ struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store;
+ int res = drefs->refs->be->fsck(drefs->refs, o);
@@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
}
+static int files_fsck(struct ref_store *ref_store,
-+ struct fsck_refs_options *o)
++ struct fsck_options *o)
+{
+ int ret;
+ struct files_ref_store *refs =
@@ refs/packed-backend.c: static struct ref_iterator *packed_reflog_iterator_begin(
}
+static int packed_fsck(struct ref_store *ref_store,
-+ struct fsck_refs_options *o)
++ struct fsck_options *o)
+{
+ return 0;
+}
@@ refs/refs-internal.h
#include "refs.h"
#include "iterator.h"
-+struct fsck_refs_options;
++struct fsck_options;
struct ref_transaction;
/*
@@ refs/refs-internal.h: typedef int read_raw_ref_fn(struct ref_store *ref_store, c
struct strbuf *referent);
+typedef int fsck_fn(struct ref_store *ref_store,
-+ struct fsck_refs_options *o);
++ struct fsck_options *o);
+
struct ref_storage_be {
const char *name;
@@ refs/reftable-backend.c: static int reftable_be_reflog_expire(struct ref_store *
}
+static int reftable_be_fsck(struct ref_store *ref_store,
-+ struct fsck_refs_options *o)
++ struct fsck_options *o)
+{
+ return 0;
+}
3: 373f470fdc ! 3: f00acab4d1 builtin/refs: add verify subcommand
@@ Documentation/git-refs.txt: include::ref-storage-format.txt[]
+
+--strict::
+ Enable more strict checking, every WARN severity for the `Fsck Messages`
-+ be seen as ERROR.
++ be seen as ERROR. See linkgit:git-fsck[1].
+
+--verbose::
+ When verifying the reference database consistency, be chatty.
@@ builtin/refs.c: static int cmd_refs_migrate(int argc, const char **argv, const c
+ };
+ int ret = 0;
+ unsigned int verbose = 0, strict = 0;
-+ struct fsck_refs_options fsck_refs_options = FSCK_REFS_OPTIONS_DEFAULT;
++ struct fsck_options fsck_options = FSCK_OPTIONS_DEFAULT;
+ struct option options[] = {
+ OPT__VERBOSE(&verbose, N_("be verbose")),
+ OPT_BOOL(0, "strict", &strict, N_("enable strict checking")),
@@ builtin/refs.c: static int cmd_refs_migrate(int argc, const char **argv, const c
+ usage(_("too many arguments"));
+
+ if (verbose)
-+ fsck_refs_options.verbose = 1;
++ fsck_options.verbose = 1;
+ if (strict)
-+ fsck_refs_options.strict = 1;
++ fsck_options.strict = 1;
+
-+ git_config(git_fsck_refs_config, &fsck_refs_options);
++ git_config(git_fsck_config, &fsck_options);
+ prepare_repo_settings(the_repository);
+
-+ ret = refs_fsck(get_main_ref_store(the_repository), &fsck_refs_options);
++ ret = refs_fsck(get_main_ref_store(the_repository), &fsck_options);
+
+ /*
-+ * Explicitly free the allocated array. This is necessary because
-+ * this program is executed as child process of git-fsck(1) and the
-+ * allocated array may not freed when git-fsck(1) aborts somewhere.
++ * Explicitly free the allocated array and object skiplist set. Because
++ * we reuse `git_fsck_config` here. It will still set the skiplist.
+ */
-+ free(fsck_refs_options.msg_type);
++ free(fsck_options.msg_type);
++ oidset_clear(&fsck_options.objs_options.skiplist);
+ return ret;
+}
+
4: 4f8ebea573 = 4: 6494f5b0c2 builtin/fsck: add `git-refs verify` child process
5: 6062b3b453 ! 5: 2c064f4f71 files-backend: add unified interface for refs scanning @@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
+ * the whole directory. This function is used as the callback for each
+ * regular file or symlink in the directory.
+ */
-+typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o, ++typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
+ const char *gitdir,
+ const char *refs_check_dir,
+ struct dir_iterator *iter);
+
+static int files_fsck_refs_dir(struct ref_store *ref_store,
-+ struct fsck_refs_options *o,
++ struct fsck_options *o,
+ const char *refs_check_dir,
+ files_fsck_refs_fn *fsck_refs_fns)
+{
@@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
+}
+
+static int files_fsck_refs(struct ref_store *ref_store,
-+ struct fsck_refs_options *o)
++ struct fsck_options *o)
+{
+ int ret;
+ files_fsck_refs_fn fsck_refs_fns[]= {
@@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
+}
+
static int files_fsck(struct ref_store *ref_store,
- struct fsck_refs_options *o)
+ struct fsck_options *o)
{
@@ refs/files-backend.c: static int files_fsck(struct ref_store *ref_store,
struct files_ref_store *refs =
6: 87a22f4c7f ! 6: 88b54b9e1a fsck: add ref name check for files backend
@@ fsck.h: enum fsck_msg_type {
FUNC(BAD_TREE_SHA1, ERROR) \
## refs/files-backend.c ##
-@@ refs/files-backend.c: typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o, +@@ refs/files-backend.c: typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
const char *refs_check_dir,
struct dir_iterator *iter);
-+static int files_fsck_refs_name(struct fsck_refs_options *o, ++static int files_fsck_refs_name(struct fsck_options *o,
+ const char *gitdir UNUSED,
+ const char *refs_check_dir,
+ struct dir_iterator *iter)
@@ refs/files-backend.c: typedef int (*files_fsck_refs_fn)(struct fsck_refs_options
+}
+
static int files_fsck_refs_dir(struct ref_store *ref_store,
- struct fsck_refs_options *o,
+ struct fsck_options *o,
const char *refs_check_dir,
@@ refs/files-backend.c: static int files_fsck_refs(struct ref_store *ref_store,
{
7: b945f61ef1 ! 7: 8309a4746a fsck: add ref content check for files backend
@@ refs/files-backend.c: int parse_loose_ref_contents(const char *buf, struct objec
return 0;
}
-@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *o, +@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
return ret;
}
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
+ * would be the content after "refs:". For symblic link, "pointee_name" would
+ * be the relative path agaignst "gitdir".
+ */
-+static int files_fsck_symref_target(struct fsck_refs_options *o, ++static int files_fsck_symref_target(struct fsck_options *o,
+ const char *refname,
+ const char *pointee_name,
+ const char *pointee_path)
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
+ return ret;
+}
+
-+static int files_fsck_refs_content(struct fsck_refs_options *o, ++static int files_fsck_refs_content(struct fsck_options *o,
+ const char *gitdir,
+ const char *refs_check_dir,
+ struct dir_iterator *iter)
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
+}
+
static int files_fsck_refs_dir(struct ref_store *ref_store,
- struct fsck_refs_options *o,
+ struct fsck_options *o,
const char *refs_check_dir,
@@ refs/files-backend.c: static int files_fsck_refs(struct ref_store *ref_store,
int ret;
Signature · 1 line
2.45.2
PATCH v4 1/7 fsck: add refs check interfaces to interact with fsck error levels
The git-fsck(1) focuses on object database consistency check. It relies on the "fsck_options" to interact with fsck error levels. However, "fsck_options" aims at checking the object database which contains a lot of fields only related to object database.
In order to add ref operations, create a new struct named "fsck_refs_options" and a new struct named "fsck_objs_options". Remove object-related fields from "fsck_options" to "fsck_objs_options". Change the "fsck_options" with three parts of members:
1. The "fsck_refs_options".
2. The "fsck_objs_options".
3. The common settings both for refs and objects. Because we leave common settings in "fsck_options". The setup process could be fully reused without any code changing.
Also add related macros to align with the current code. Because we remove some fields from "fsck_options" to "fsck_objs_options". Change the influenced code to use the "fsck_options.objs_options" instead of using "fsck_options" itself.
The static function "report" provided by "fsck.c" aims at reporting the problems related to object database which cannot be reused for refs. Provide "fsck_refs_report" function to integrate the fsck error levels into reference consistency check.
--- builtin/fsck.c | 8 +-- builtin/index-pack.c | 2 +- builtin/mktag.c | 5 +- builtin/unpack-objects.c | 2 +- fetch-pack.c | 8 +-- fsck.c | 113 +++++++++++++++++++++++++++++---------- fsck.h | 79 ++++++++++++++++++++------- object-file.c | 2 +- 8 files changed, 161 insertions(+), 58 deletions(-)
builtin/fsck.c +4 -4
@@ -233,7 +233,7 @@ static void mark_unreachable_referents(const struct object_id *oid)object_as_type (obj, type,0 ); } - options.walk = mark_used; + options.objs_options.walk = mark_used;fsck_walk (obj,NULL , &options);if (obj->type ==OBJ_TREE )free_tree_buffer ((struct tree *)obj);
@@ -936,9 +936,9 @@ int cmd_fsck(int argc, const char **argv, const char *prefix) argc =parse_options (argc, argv, prefix, fsck_opts, fsck_usage,0 ); - fsck_walk_options.walk = mark_object; - fsck_obj_options.walk = mark_used; - fsck_obj_options.error_func = fsck_error_func; + fsck_walk_options.objs_options.walk = mark_object; + fsck_obj_options.objs_options.walk = mark_used; + fsck_obj_options.objs_options.error_func = fsck_error_func;if (check_strict) fsck_obj_options.strict =1 ;
builtin/index-pack.c +1 -1
@@ -1746,7 +1746,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)usage (index_pack_usage);disable_replace_refs (); - fsck_options.walk = mark_link; + fsck_options.objs_options.walk = mark_link;reset_pack_idx_option (&opts); opts.flags |=WRITE_REV ;
builtin/mktag.c +3 -2
@@ -91,8 +91,9 @@ int cmd_mktag(int argc, const char **argv, const char *prefix)if (strbuf_read (&buf,0 ,0 ) <0 )die_errno (_ ("could not read from stdin" )); - fsck_options.error_func = mktag_fsck_error_func; -fsck_set_msg_type_from_ids (&fsck_options,FSCK_MSG_EXTRA_HEADER_ENTRY , + fsck_options.objs_options.error_func = mktag_fsck_error_func; +fsck_set_msg_type_from_ids (&fsck_options, +FSCK_MSG_EXTRA_HEADER_ENTRY ,FSCK_WARN );/* config might set fsck.extraHeaderEntry=* again */ git_config (git_fsck_config, &fsck_options);
builtin/unpack-objects.c +1 -1
@@ -239,7 +239,7 @@ static int check_object(struct object *obj, enum object_type type,die ("Whoops! Cannot find object '%s'" ,oid_to_hex (&obj->oid));if (fsck_object (obj, obj_buf->buffer, obj_buf->size, &fsck_options))die ("fsck error in packed object" ); - fsck_options.walk = check_object; + fsck_options.objs_options.walk = check_object;if (fsck_walk (obj,NULL , &fsck_options))die ("Error on reachable objects of %s" ,oid_to_hex (&obj->oid));write_cached_object (obj, obj_buf);
fetch-pack.c +5 -3
@@ -1220,7 +1220,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, }else alternate_shallow_file =NULL ;if (get_pack (args, fd, pack_lockfiles,NULL , sought, nr_sought, - &fsck_options.gitmodules_found)) + &fsck_options.objs_options.gitmodules_found))die (_ ("git fetch-pack: fetch failed." ));if (fsck_finish (&fsck_options))die ("fsck failed" );
@@ -1780,7 +1780,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,if (get_pack (args, fd, pack_lockfiles, packfile_uris.nr ? &index_pack_args :NULL , - sought, nr_sought, &fsck_options.gitmodules_found)) + sought, nr_sought, + &fsck_options.objs_options.gitmodules_found))die (_ ("git fetch-pack: fetch failed." ));do_check_stateless_delimiter (args->stateless_rpc, &reader);
@@ -1823,7 +1824,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, packname[the_hash_algo->hexsz] ='\0' ; -parse_gitmodules_oids (cmd.out, &fsck_options.gitmodules_found); +parse_gitmodules_oids (cmd.out, + &fsck_options.objs_options.gitmodules_found);close (cmd.out);
fsck.c +85 -28
275 lines not on this page · raw message
fsck.h +61 -18
123 lines not on this page · raw message
object-file.c +1 -1
@@ -2510,7 +2510,7 @@ static int index_mem(struct index_state *istate,struct fsck_options opts =FSCK_OPTIONS_DEFAULT ; opts.strict =1 ; - opts.error_func = hash_format_check_report; + opts.objs_options.error_func = hash_format_check_report;if (fsck_buffer (null_oid (), type, buf, size, &opts))die (_ ("refusing to create malformed object" ));fsck_finish (&opts);
Signature · 1 line
2.45.2
Re: [GSoC][PATCH v4 1/7] fsck: add refs check interfaces to interact with fsck error levels
Quoted 26 lines · shejialuo
The git-fsck(1) focuses on object database consistency check. It relies on the "
fsck_options" to interact with fsck error levels. However, "fsck_options" aims at checking the object database which contains a lot of fields only related to object database.In order to add ref operations, create a new struct named "
fsck_refs_options" and a new struct named "fsck_objs_options". Remove object-related fields from "fsck_options" to "fsck_objs_options". Change the "fsck_options" with three parts of members:1. The "
fsck_refs_options".
2. The "fsck_objs_options".
3. The common settings both for refs and objects. Because we leave common settings in "fsck_options". The setup process could be fully reused without any code changing.Also add related macros to align with the current code. Because we remove some fields from "
fsck_options" to "fsck_objs_options". Change the influenced code to use the "fsck_options.objs_options" instead of using "fsck_options" itself.The static function "report" provided by "
fsck.c" aims at reporting the problems related to object database which cannot be reused for refs. Provide "fsck_refs_report" function to integrate the fsck error levels into reference consistency check.
Quoted 4 lines
Mentored-by Patrick Steinhardt <[email protected]>Mentored-by Karthik Nayak <[email protected]>Signed-off-by shejialuo <[email protected]>---
This seems to be doing too many things at once, making the result a lot harder to review than necessary. At this point, nobody checks refs and reports problems with refs, so fsck_refs_report() has no callers and it is impossible to tell if the function signature of it, iow, the set of parameters it receives, is sufficient, for example.
Stepping back a bit, it is true that (1) all existing checks are about "objects", and (2) all checks we want to implement around "objects" and "refs" can be split cleanly into these two categories?
I am wondering if there are checks and reports that would benefit from having access to both objects and refs (e.g. when checking a ref, you may want to see both what the name of the ref is and what object the ref points at), in which case, being forced to implement such a check-and-report as "object" or "ref" that has access to only different subset of information may turn out to be too limiting.
Yes, I am OK with having substructure in fsck_options, but I am doubting if it is a good idea to have a separate fsck_refs_report() that can only take "name" that is different from fsck.c::report().
For example, how would we ensure that refs/heads/foo is allowed to point at a commit object and nothing else, and how would we report a violation when we find that ref/heads/foo is pointing at a tag, i.e., "refs/heads/foo points at f665776185ad074b236c00751d666da7d1977dbe which is a tag". The fsck_refs_report() function is not equipped to do that; neither is .refs_options.error_func() that only takes "name".
fsck.c static int fsck_blobs(struct oidset *blobs_found, struct oidset *blobs_done, · line 1322
Quoted 10 lines
fsck.c +9
-0+int fsck_refs_report(struct fsck_options *o, + const char *name, + enum fsck_msg_id msg_id, + const char *fmt, ...) + va_start(ap, fmt); +strbuf_vaddf (&sb, fmt, ap); + ret = o->refs_options.error_func (o, name, msg_type, msg_id, sb.buf); +strbuf_release (&sb); +va_end (ap);
Perhaps the code and data structure of the entire series may be capable of supporting such a check-and-report, but the primary point I am making is that among what [1/7] adds, we cannot sanely judge if these "refs" related additions are sensible by looking at [1/7].
Thanks.
Re: [GSoC][PATCH v4 1/7] fsck: add refs check interfaces to interact with fsck error levels
Quoted 1 line · Junio C Hamano
shejialuo <[email protected]> writes:
[snip]
Quoted 7 lines
This seems to be doing too many things at once, making the result a lot harder to review than necessary. At this point, nobody checks refs and reports problems with refs, so
fsck_refs_report()has no callers and it is impossible to tell if the function signature of it, iow, the set of parameters it receives, is sufficient, for example.
I will split this commit into multiple commits in the next version.
Quoted 4 lines
Stepping back a bit, it is true that (1) all existing checks are about "objects", and (2) all checks we want to implement around "objects" and "refs" can be split cleanly into these two categories?
I am sure that "objects" and "refs" checks cannot be split cleanly into two categories. Let me elaborate more about this. The git-fsck(1) has already implicitly checked refs. When executing git-fsck(1) with no arguments, it will call "builtin/fsck.c::get_default_heads()". It will traverse every raw ref and use "fsck_handle_ref" to check the refs.
struct obj *obj;
obj = parse_object(the_repository, oid);
if (!obj) {
...
errors_found |= ERROR_REACHABLE;
return 0;
}
if (obj->type != OBJ_COMMIT && is_branch(refname)) {
...
errors_found |= ERROR_REFS;
}
obj->flags |= USED;
mark_object_reachable(obj);
return 0;git-fsck(1) firstly scans the object directory to check every loose object. If everything is OK, it will set "obj->flags" like the following:
obj->flags &= ~(REACHABLE | SEEN); obj->flags |= HAS_OBJ;
When an object is marked with "UNREACHABLE" and "UNUSED", it will be dangling.
When checking objects, git-fsck(1) still checks the refs. But from my understanding, the main motivation here is mark the "USED" flag because we want to tell whether an object is dangling and BTY we check refs. So, we cannot really classify into these two categories. But we can classify into these two categories from action view. When checking refs, we look at the refs options; when checking objects, we look at the objects options. Although we may implicitly check some other things, this should be OK.
So, I think "fsck_options" is only about options. Because the nature of the connectivity between refs and commit objects, we cannot make the boundary so clear.
Quoted 7 lines
I am wondering if there are checks and reports that would benefit from having access to both objects and refs (e.g. when checking a ref, you may want to see both what the name of the ref is and what object the ref points at), in which case, being forced to implement such a check-and-report as "object" or "ref" that has access to only different subset of information may turn out to be too limiting.
Should "check-and-report" does this? I am doubt. There are some cases we do look at both object and ref. But "report" should not do this. The flow git-fsck(1) does is like the following:
if (some_thing_bad) {
ret = report(..., message);
}Here, "check" will provide dedicated message to report. So If we really both look at both object and ref, the caller should be responsible for that. Like the message "refs/heads/foo points at f665776185ad074b236c00751d666da7d1977dbe which is a tag". The caller should produce such message. It will call some APIs to get the information and pass it to the "report" function.
Quoted 4 lines
Yes, I am OK with having substructure in
fsck_options, but I am doubting if it is a good idea to have a separatefsck_refs_report()that can only take "name" that is different fromfsck.c::report().
Quoted 8 lines
For example, how would we ensure that refs/heads/foo is allowed to point at a commit object and nothing else, and how would we report a violation when we find that ref/heads/foo is pointing at a tag, i.e., "refs/heads/foo points at
f665776185ad074b236c00751d666da7d1977dbewhich is a tag". Thefsck_refs_report()function is not equipped to do that; neither is .refs_options.error_func() that only takes "name".
Yes, I am not satisfied with current design either. Actually, there are many redundant code in new "fsck_refs_report" function. Let's look at "fsck.c::report()" closely to see what it does.
static int report(struct fsck_options *options,
const struct object_id *oid, enum object_type object_type,
enum fsck_msg_id msg_id, const char *fmt, ...)
{
va_list ap;
struct strbuf sb = STRBUF_INIT;
enum fsck_msg_type msg_type = fsck_msg_type(msg_id, options);
int result;
if (msg_type == FSCK_IGNORE)
return 0;
if (object_on_skiplist(options, oid))
return 0;
if (msg_type == FSCK_FATAL)
msg_type = FSCK_ERROR;
else if (msg_type == FSCK_INFO)
msg_type = FSCK_WARN;
prepare_msg_ids();
strbuf_addf(&sb, "%s: ", msg_id_info[msg_id].camelcased);
va_start(ap, fmt);
strbuf_vaddf(&sb, fmt, ap);
result = options->objs_options.error_func(options, oid, object_type,
msg_type, msg_id, sb.buf);
strbuf_release(&sb);
va_end(ap);
return result;
}It will get the current message "msg_type" which would be configured and it will convert the "FSCK_FATAL" to "FSCK_ERROR", and "FSCK_INFO" to "FSCK_WARN". From above code, we can easily know what would the error (warn) message be the following:
'object: fsck_describe_object(o, oid): FSCK_MESSAGE: message'.
The "fsck_describe_object" aims to provide meaningful message about the oid. And "FSCK_MESSAGE" is "msg_id_info[msg_id]".The last field "message" is the caller-provided message.
So the hardest part to reuse this function is that it receives the parameters related to object such as oid and object_type. Actually, the "fsck_refs_report" could generate such information "refs/heads/foo points at f665776185ad074b236c00751d666da7d1977dbe which is a tag" like the following (I use string literal for simplicity here):
if (some_thing_bad) {
...
ret = fsck_refs_report(o, "refs/heads/foo", FSCK_BAD_REF_CONTENT,
"points at f665776185ad074b236c00751d666da7d1977dbe which is a tag")
}It will generate the following output in the current design:
error: refs/heads/foo: badRefContent: points at f665776185ad074b236c00751d666da7d1977dbe which is a tag
So my idea is that we just reuse the "report" function here. The bad part for current "report" function is that it is highly coupled with the object check. However, my idea is just like the "fsck_refs_report".
<name>: <FSCK_MESSAGE_STRING>: <message>
Thus, we may unify the report interface both for refs and objects. And we don't define two differtn interface:
1. "fsck_refs_error"
2. "fsck_objs_error"
Also, we just need to use "fsck_refs_report" and "fsck_objs_report" to wrap the "report" function here to avoid redundance.
Quoted 7 lines
Quoted 7 lines
fsck.c +6
-0+int fsck_refs_report(struct fsck_options *o, + const char *name, + enum fsck_msg_id msg_id, + const char *fmt, ...) + va_start(ap, fmt); +strbuf_vaddf (&sb, fmt, ap);
Quoted 9 lines
fsck.c static int fsck_blobs(struct oidset *blobs_found, struct oidset *blobs_done, · line 1322
Quoted 3 lines
fsck.c +3
-0+ ret = o->refs_options.error_func (o, name, msg_type, msg_id, sb.buf); +strbuf_release (&sb); +va_end (ap);Perhaps the code and data structure of the entire series may be capable of supporting such a check-and-report, but the primary point I am making is that among what [1/7] adds, we cannot sanely judge if these "refs" related additions are sensible by looking at [1/7].
From my above reply, I guess the current design gives the caller extensibility to provide customize message. All we do is make code more clean without redundance.
Quoted 1 line
Thanks.
Thanks,
Jialuo
PATCH v4 2/7 refs: set up ref consistency check infrastructure
In reply to shejialuoThe interfaces defined in the ref_storage_be are carefully structured in semantic. It's organized as the five parts:
1. The name and the initialization interfaces.
2. The ref transaction interfaces.
3. The ref internal interfaces (pack, rename and copy).
4. The ref filesystem interfaces.
5. The reflog related interfaces.
To keep consistent with the git-fsck(1), add a new interface named "fsck_refs_fn" to the end of "ref_storage_be". This semantic cannot be grouped into any above five categories. Explicitly add blank line to make it different from others.
--- refs.c | 5 +++++ refs.h | 8 ++++++++ refs/debug.c | 11 +++++++++++ refs/files-backend.c | 15 ++++++++++++++- refs/packed-backend.c | 8 ++++++++ refs/refs-internal.h | 6 ++++++ refs/reftable-backend.c | 8 ++++++++ 7 files changed, 60 insertions(+), 1 deletion(-)
refs.c +5 -0
@@ -316,6 +316,11 @@ int check_refname_format(const char *refname, int flags)return check_or_sanitize_refname (refname, flags,NULL ); } +int refs_fsck (struct ref_store *refs,struct fsck_options *o) +{ +return refs->be->fsck (refs, o); +} +void sanitize_refname_component (const char *refname,struct strbuf *out) {if (check_or_sanitize_refname (refname,REFNAME_ALLOW_ONELEVEL , out))
refs.h +8 -0
@@ -3,6 +3,7 @@#include "commit.h" +struct fsck_options ;struct object_id ;struct ref_store ;struct repository ;
@@ -547,6 +548,13 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat*/ int check_refname_format (const char *refname,int flags); +/* +* Check the reference database for consistency. Return 0 if refs and +* reflogs are consistent, and non-zero otherwise. The errors will be +* written to stderr. +*/ +int refs_fsck (struct ref_store *refs,struct fsck_options *o); +/* * Apply the rules from check_refname_format, but mutate the result until it * is acceptable, and place the result in "out".
refs/debug.c +11 -0
@@ -419,6 +419,15 @@ static int debug_reflog_expire(struct ref_store *ref_store, const char *refname,return res; } +static int debug_fsck (struct ref_store *ref_store, +struct fsck_options *o) +{ +struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store; +int res = drefs->refs->be->fsck (drefs->refs, o); +trace_printf_key (&trace_refs,"fsck: %d\n" , res); +return res; +} +struct ref_storage_be refs_be_debug = { .name ="debug" , .init =NULL ,
@@ -451,4 +460,6 @@ struct ref_storage_be refs_be_debug = { .create_reflog = debug_create_reflog, .delete_reflog = debug_delete_reflog, .reflog_expire = debug_reflog_expire, + + .fsck = debug_fsck, };
refs/files-backend.c +14 -1
@@ -3405,6 +3405,17 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,return ret; } +static int files_fsck (struct ref_store *ref_store, +struct fsck_options *o) +{ +int ret; +struct files_ref_store *refs = +files_downcast (ref_store,REF_STORE_READ ,"fsck" ); + + ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o); +return ret; +} +struct ref_storage_be refs_be_files = { .name ="files" , .init = files_ref_store_init,
@@ -3431,5 +3442,7 @@ struct ref_storage_be refs_be_files = { .reflog_exists = files_reflog_exists, .create_reflog = files_create_reflog, .delete_reflog = files_delete_reflog, - .reflog_expire = files_reflog_expire + .reflog_expire = files_reflog_expire, + + .fsck = files_fsck, };
refs/packed-backend.c +8 -0
@@ -1733,6 +1733,12 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_sreturn empty_ref_iterator_begin (); } +static int packed_fsck (struct ref_store *ref_store, +struct fsck_options *o) +{ +return 0 ; +} +struct ref_storage_be refs_be_packed = { .name ="packed" , .init = packed_ref_store_init,
@@ -1760,4 +1766,6 @@ struct ref_storage_be refs_be_packed = { .create_reflog = NULL, .delete_reflog = NULL, .reflog_expire = NULL, + + .fsck = packed_fsck, };
refs/refs-internal.h +6 -0
@@ -4,6 +4,7 @@#include "refs.h" #include "iterator.h" +struct fsck_options ;struct ref_transaction ;/*
@@ -650,6 +651,9 @@ typedef int read_raw_ref_fn(struct ref_store *ref_store, const char *refname,typedef int read_symbolic_ref_fn (struct ref_store *ref_store,const char *refname,struct strbuf *referent); +typedef int fsck_fn (struct ref_store *ref_store, +struct fsck_options *o); +struct ref_storage_be {const char *name;ref_store_init_fn *init;
@@ -677,6 +681,8 @@ struct ref_storage_be {create_reflog_fn *create_reflog;delete_reflog_fn *delete_reflog;reflog_expire_fn *reflog_expire; + +fsck_fn *fsck; };extern struct ref_storage_be refs_be_files;
refs/reftable-backend.c +8 -0
@@ -2281,6 +2281,12 @@ static int reftable_be_reflog_expire(struct ref_store *ref_store,return ret; } +static int reftable_be_fsck (struct ref_store *ref_store, +struct fsck_options *o) +{ +return 0 ; +} +struct ref_storage_be refs_be_reftable = { .name ="reftable" , .init = reftable_be_init,
@@ -2308,4 +2314,6 @@ struct ref_storage_be refs_be_reftable = { .create_reflog = reftable_be_create_reflog, .delete_reflog = reftable_be_delete_reflog, .reflog_expire = reftable_be_reflog_expire, + + .fsck = reftable_be_fsck, };
Signature · 1 line
2.45.2
PATCH v4 3/7 builtin/refs: add verify subcommand
In reply to shejialuoIntroduce a new subcommand "verify" in git-refs(1) to allow the user to check the reference database consistency.
--- Documentation/git-refs.txt | 13 +++++++++++ builtin/refs.c | 45 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+)
Documentation/git-refs.txt +13 -0
@@ -10,6 +10,7 @@ SYNOPSIS -------- [verse] 'git refs migrate' --ref-format=<format> [--dry-run] +'git refs verify' [--strict] [--verbose] DESCRIPTION -----------
@@ -22,6 +23,9 @@ COMMANDS migrate:: Migrate ref store between different formats. +verify:: + Verify reference database consistency. + OPTIONS -------
@@ -39,6 +43,15 @@ include::ref-storage-format.txt[] can be used to double check that the migration works as expected before performing the actual migration. +The following options are specific to 'git refs verify': + +--strict:: + Enable more strict checking, every WARN severity for the `Fsck Messages` + be seen as ERROR. See linkgit:git-fsck[1]. + +--verbose:: + When verifying the reference database consistency, be chatty. + KNOWN LIMITATIONS -----------------
builtin/refs.c +45 -0
70 lines not on this page · raw message
Signature · 1 line
2.45.2
PATCH v4 4/7 builtin/fsck: add `git-refs verify` child process
In reply to shejialuoIntroduce a new function "fsck_refs" that initializes and runs a child process to execute the "git-refs verify" command.
--- builtin/fsck.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+)
builtin/fsck.c +17 -0
@@ -896,6 +896,21 @@ static int check_pack_rev_indexes(struct repository *r, int show_progress)return res; } +static void fsck_refs (void ) +{ +struct child_process refs_verify =CHILD_PROCESS_INIT ; +child_process_init (&refs_verify); + refs_verify.git_cmd =1 ; +strvec_pushl (&refs_verify.args,"refs" ,"verify" ,NULL ); +if (verbose) +strvec_push (&refs_verify.args,"--verbose" ); +if (check_strict) +strvec_push (&refs_verify.args,"--strict" ); + +if (run_command (&refs_verify)) + errors_found |=ERROR_REFS ; +} +static char const *const fsck_usage[] = { N_("git fsck [--tags] [--root] [--unreachable] [--cache] [--no-reflogs]\n" " [--[no-]full] [--strict] [--verbose] [--lost-found]\n"
@@ -1065,6 +1080,8 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)check_connectivity (); +fsck_refs (); +if (the_repository->settings.core_commit_graph) {struct child_process commit_graph_verify =CHILD_PROCESS_INIT ;
Signature · 1 line
2.45.2
PATCH v4 5/7 files-backend: add unified interface for refs scanning
In reply to shejialuoFor refs and reflogs, we need to scan its corresponding directories to check every regular file or symbolic link which shares the same pattern. Introduce a unified interface for scanning directories for files-backend.
--- refs/files-backend.c | 77 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-)
refs/files-backend.c +76 -1
@@ -4,6 +4,7 @@#include "../gettext.h" #include "../hash.h" #include "../hex.h" +#include "../fsck.h" #include "../refs.h" #include "refs-internal.h" #include "ref-cache.h"
@@ -3405,6 +3406,78 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,return ret; } +/* +* For refs and reflogs, they share a unified interface when scanning +* the whole directory. This function is used as the callback for each +* regular file or symlink in the directory. +*/ +typedef int (*files_fsck_refs_fn )(struct fsck_options *o, +const char *gitdir, +const char *refs_check_dir, +struct dir_iterator *iter); + +static int files_fsck_refs_dir (struct ref_store *ref_store, +struct fsck_options *o, +const char *refs_check_dir, +files_fsck_refs_fn *fsck_refs_fns) +{ +const char *gitdir = ref_store->gitdir; +struct strbuf sb =STRBUF_INIT ; +struct dir_iterator *iter; +int iter_status; +int ret =0 ; + +strbuf_addf (&sb,"%s/%s" , gitdir, refs_check_dir); + + iter =dir_iterator_begin (sb.buf,0 ); + +if (!iter) { + ret =error_errno ("cannot open directory %s" , sb.buf); + goto out; + } + +while ((iter_status =dir_iterator_advance (iter)) ==ITER_OK ) { +if (S_ISDIR (iter->st.st_mode)) { +continue ; + }else if (S_ISREG (iter->st.st_mode) || +S_ISLNK (iter->st.st_mode)) { +if (o->verbose) +fprintf_ln (stderr,"Checking %s/%s" , + refs_check_dir, iter->relative_path); +for (size_t i =0 ; fsck_refs_fns[i]; i++) { +if (fsck_refs_fns[i](o, gitdir, refs_check_dir, iter)) + ret =-1 ; + } + }else { + ret =error (_ ("unexpected file type for '%s'" ), + iter->basename); + } + } + +if (iter_status !=ITER_DONE ) + ret =error (_ ("failed to iterate over '%s'" ), sb.buf); + +out: +strbuf_release (&sb); +return ret; +} + +static int files_fsck_refs (struct ref_store *ref_store, +struct fsck_options *o) +{ +int ret; +files_fsck_refs_fn fsck_refs_fns[]= { +NULL + }; + +if (o->verbose) +fprintf_ln (stderr,"Checking references consistency" ); + + ret =files_fsck_refs_dir (ref_store, o,"refs" , fsck_refs_fns); + +return ret; +} +static int files_fsck (struct ref_store *ref_store,struct fsck_options *o) {
@@ -3412,7 +3485,9 @@ static int files_fsck(struct ref_store *ref_store,struct files_ref_store *refs =files_downcast (ref_store,REF_STORE_READ ,"fsck" ); - ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o); + ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o) + |files_fsck_refs (ref_store, o); +return ret; }
Signature · 1 line
2.45.2
PATCH v4 6/7 fsck: add ref name check for files backend
In reply to shejialuoThe git-fsck(1) only implicitly checks the reference, it does not fully check refs with bad format name such as standalone "@" and name ending with ".lock".
In order to provide such checks, add a new fsck message id "badRefName" with default ERROR type. Use existing "check_refname_format" to explicit check the ref name. And add a new unit test to verify the functionality.
--- Documentation/fsck-msgids.txt | 3 + fsck.h | 1 + refs/files-backend.c | 20 +++++++ t/t0602-reffiles-fsck.sh | 101 ++++++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+) create mode 100755 t/t0602-reffiles-fsck.sh
Documentation/fsck-msgids.txt +3 -0
@@ -19,6 +19,9 @@ `badParentSha1`:: (ERROR) A commit object has a bad parent sha1. +`badRefName`:: + (ERROR) A ref has a bad name. + `badTagName`:: (INFO) A tag has an invalid format.
fsck.h +1 -0
@@ -31,6 +31,7 @@ enum fsck_msg_type { FUNC(BAD_NAME, ERROR) \ FUNC(BAD_OBJECT_SHA1, ERROR) \ FUNC(BAD_PARENT_SHA1, ERROR) \ + FUNC(BAD_REF_NAME, ERROR) \ FUNC(BAD_TIMEZONE, ERROR) \ FUNC(BAD_TREE, ERROR) \FUNC (BAD_TREE_SHA1 ,ERROR ) \
refs/files-backend.c +20 -0
@@ -3416,6 +3416,25 @@ typedef int (*files_fsck_refs_fn)(struct fsck_options *o,const char *refs_check_dir, struct dir_iterator *iter); +static int files_fsck_refs_name (struct fsck_options *o, + const char *gitdir UNUSED, +const char *refs_check_dir, +struct dir_iterator *iter) +{ +struct strbuf sb =STRBUF_INIT ; +int ret =0 ; + +if (check_refname_format (iter->basename,REFNAME_ALLOW_ONELEVEL )) { +strbuf_addf (&sb,"%s/%s" , refs_check_dir, iter->relative_path); + ret =fsck_refs_report (o, sb.buf, +FSCK_MSG_BAD_REF_NAME , +"invalid refname format" ); + } + +strbuf_release (&sb); +return ret; +} + static int files_fsck_refs_dir(struct ref_store *ref_store, struct fsck_options *o, const char *refs_check_dir,
@@ -3467,6 +3486,7 @@ static int files_fsck_refs(struct ref_store *ref_store, {int ret;files_fsck_refs_fn fsck_refs_fns[]= { + files_fsck_refs_name,NULL };
t/t0602-reffiles-fsck.sh new file +101 -0
101 lines not on this page · raw message
Signature · 1 line
2.45.2
PATCH v4 7/7 fsck: add ref content check for files backend
In reply to shejialuoEnhance the git-fsck(1) command by adding a check for reference content in the files backend. The new functionality ensures that symrefs, real symbolic link and regular refs are validated correctly.
In order to check the trailing content of the regular refs, add a new parameter trailing to parse_loose_ref_contents.
For symrefs, parse_loose_ref_contents will set the "referent". However, symbolic link could be either absolute or relative. Use "strbuf_add_real_path" to read the symbolic link and convert the relative path to absolute path. Then use "skip_prefix" to make it align with symref "referent".
Thus, the symrefs and symbolic links could share the same interface. Add a new function "files_fsck_symref_target" which aims at checking the following things:
1. whether the pointee is under the refs/ directory.
2. whether the pointee name is correct.
3. whether the pointee path is a wrong type in filesystem.
Last, add the following FSCK MESSAGEs:
1. "badRefContent(ERROR)": A ref has a bad content
2. "badSymrefPointee(ERROR)": The pointee of a symref is bad.
3. "trailingRefContent(WARN)": A ref content has trailing contents.
--- Documentation/fsck-msgids.txt | 9 +++ fsck.h | 3 + refs.c | 2 +- refs/files-backend.c | 145 +++++++++++++++++++++++++++++++++- refs/refs-internal.h | 5 +- t/t0602-reffiles-fsck.sh | 110 ++++++++++++++++++++++++++ 6 files changed, 269 insertions(+), 5 deletions(-)
Documentation/fsck-msgids.txt +9 -0
@@ -19,9 +19,15 @@ `badParentSha1`:: (ERROR) A commit object has a bad parent sha1. +`badRefContent`:: + (ERROR) A ref has a bad content. + `badRefName`:: (ERROR) A ref has a bad name. +`badSymrefPointee`:: + (ERROR) The pointee of a symref is bad. + `badTagName`:: (INFO) A tag has an invalid format.
@@ -167,6 +173,9 @@ `nullSha1`:: (WARN) Tree contains entries pointing to a null sha1. +`trailingRefContent`:: + (WARN) A ref content has trailing contents. + `treeNotSorted`:: (ERROR) A tree is not properly sorted.
fsck.h +3 -0
@@ -32,6 +32,8 @@ enum fsck_msg_type { FUNC(BAD_OBJECT_SHA1, ERROR) \ FUNC(BAD_PARENT_SHA1, ERROR) \ FUNC(BAD_REF_NAME, ERROR) \ + FUNC(BAD_REF_CONTENT, ERROR) \ + FUNC(BAD_SYMREF_POINTEE, ERROR) \ FUNC(BAD_TIMEZONE, ERROR) \ FUNC(BAD_TREE, ERROR) \FUNC (BAD_TREE_SHA1 ,ERROR ) \
@@ -72,6 +74,7 @@ enum fsck_msg_type { FUNC(HAS_DOTDOT, WARN) \ FUNC(HAS_DOTGIT, WARN) \ FUNC(NULL_SHA1, WARN) \ + FUNC(TRAILING_REF_CONTENT, WARN) \ FUNC(ZERO_PADDED_FILEMODE, WARN) \ FUNC(NUL_IN_COMMIT, WARN) \FUNC (LARGE_PATHNAME ,WARN ) \
refs.c +1 -1
@@ -1744,7 +1744,7 @@ static int refs_read_special_head(struct ref_store *ref_store, } result =parse_loose_ref_contents (content.buf, oid, referent, type, - failure_errno); + failure_errno,NULL ); done:strbuf_release (&full_path);
refs/files-backend.c +143 -2
179 lines not on this page · raw message
refs/refs-internal.h +3 -2
@@ -709,11 +709,12 @@ struct ref_store {/* * Parse contents of a loose ref file. *failure_errno maybe be set to EINVAL for -* invalid contents. +* invalid contents. Also *trailing is set to the first character after the +* refname or NULL if the referent is not empty. */ int parse_loose_ref_contents (const char *buf,struct object_id *oid,struct strbuf *referent,unsigned int *type, -int *failure_errno); +int *failure_errno,const char **trailing);/* * Fill in the generic part of refs and add it to our collection of
t/t0602-reffiles-fsck.sh +110 -0
114 lines not on this page · raw message
Signature · 1 line
2.45.2
v5 cover ref consistency check infra setup
In reply to shejialuoHi All, this version have been changed a lot compared with the v4. The mainly changed part is how to integrate the ref checks into the current git-fsck(1) which concentrates on the object database check.
In the previous version, I design the following structures:
struct fsck_options {
struct fsck_refs_options refs_options;
struct fsck_objs_options objs_options;
enum fsck_msg_type *msg_type;
unsigned strict:1,
verbose:1;
};Although Junio said It's OK that therea are sub-structures in "fsck_options", I decide not to use the above design. This is because in the current implementation, we either check refs or objects. The above design is a combination. It will give us the feeling that "fsck_options" is used to check both refs and objects.
So I decide to create two new structs "fsck_objects_options" and "fsck_refs_options". So we need to think about how to design this. In my implementation, I totally change the "fsck_options" to be a general options which "fsck_objects_options" and "fsck_refs_options" will incorporate.
However, there are some options related to configs such as "enum fsck_msg_type *msg_type" and "skiplist". I change them into the "fskc.c" as the static variable for singleton pattern.
And then extract the common options into "fsck_options". Thus we could reuse some interfaces when chekcing refs or objects.
At last, solve the review from Junio. Junio hopes that we could use a unified interface to report fsck error messages. This version solves this.
BTY, this patch is based on the commit:
1e1586e4ed (The sixteenth batch, 2024-06-24)
Thanks all
shejialuo (12):
fsck: rename "fsck_options" to "fsck_objects_options"
fsck: use "fsck_configs" to set up configs
fsck: abstract common options for reusing
fsck: add "fsck_refs_options" struct
fsck: add a unified interface for reporting fsck messages
fsck: add "fsck_refs_options" initialization macros
refs: set up ref consistency check infrastructure builtin/refs: add verify subcommand builtin/fsck: add git-refs verify child process
files-backend: add unified interface for refs scanning
fsck: add ref name check for files backend
fsck: add ref content check for files backend
Documentation/fsck-msgids.txt | 12 ++ Documentation/git-refs.txt | 13 ++ builtin/fsck.c | 38 +++-- builtin/index-pack.c | 18 +-- builtin/mktag.c | 15 +- builtin/refs.c | 36 +++++ builtin/unpack-objects.c | 14 +- fetch-pack.c | 12 +- fsck.c | 197 +++++++++++++++++++------- fsck.h | 122 ++++++++++++---- object-file.c | 20 +-- refs.c | 7 +- refs.h | 8 ++ refs/debug.c | 11 ++ refs/files-backend.c | 255 +++++++++++++++++++++++++++++++++- refs/packed-backend.c | 8 ++ refs/refs-internal.h | 11 +- refs/reftable-backend.c | 8 ++ t/t0602-reffiles-fsck.sh | 211 ++++++++++++++++++++++++++++ 19 files changed, 889 insertions(+), 127 deletions(-) create mode 100755 t/t0602-reffiles-fsck.sh
Range-diff against v4:
1: 2d2c1f2cd3 < -: ---------- fsck: add refs check interfaces to interact with fsck error levels
-: ---------- > 1: d83b5797cb fsck: rename "fsck_options" to "fsck_objects_options"
-: ---------- > 2: 88000b092a fsck: use "fsck_configs" to set up configs
-: ---------- > 3: 65093bae64 fsck: abstract common options for reusing
-: ---------- > 4: 403750da30 fsck: add "fsck_refs_options" struct
-: ---------- > 5: 8dea5654d5 fsck: add a unified interface for reporting fsck messages
-: ---------- > 6: 8fb62b54c6 fsck: add "fsck_refs_options" initialization macros
2: 4bacffe44e ! 7: 2ecd1f5407 refs: set up ref consistency check infrastructure
@@ refs.c: int check_refname_format(const char *refname, int flags)
return check_or_sanitize_refname(refname, flags, NULL);
}
-+int refs_fsck(struct ref_store *refs, struct fsck_options *o)
++int refs_fsck(struct ref_store *refs, struct fsck_refs_options *o)
+{
+ return refs->be->fsck(refs, o);
+}
@@ refs.h
#include "commit.h"
-+struct fsck_options;
++struct fsck_refs_options;
struct object_id;
struct ref_store;
struct repository;
@@ refs.h: int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void
+ * reflogs are consistent, and non-zero otherwise. The errors will be
+ * written to stderr.
+ */
-+int refs_fsck(struct ref_store *refs, struct fsck_options *o);
++int refs_fsck(struct ref_store *refs, struct fsck_refs_options *o);
+
/*
* Apply the rules from check_refname_format, but mutate the result until it
@@ refs/debug.c: static int debug_reflog_expire(struct ref_store *ref_store, const
}
+static int debug_fsck(struct ref_store *ref_store,
-+ struct fsck_options *o)
++ struct fsck_refs_options *o)
+{
+ struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store;
+ int res = drefs->refs->be->fsck(drefs->refs, o);
@@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
}
+static int files_fsck(struct ref_store *ref_store,
-+ struct fsck_options *o)
++ struct fsck_refs_options *o)
+{
+ int ret;
+ struct files_ref_store *refs =
@@ refs/packed-backend.c: static struct ref_iterator *packed_reflog_iterator_begin(
}
+static int packed_fsck(struct ref_store *ref_store,
-+ struct fsck_options *o)
++ struct fsck_refs_options *o)
+{
+ return 0;
+}
@@ refs/refs-internal.h: typedef int read_raw_ref_fn(struct ref_store *ref_store, c
struct strbuf *referent);
+typedef int fsck_fn(struct ref_store *ref_store,
-+ struct fsck_options *o);
++ struct fsck_refs_options *o);
+
struct ref_storage_be {
const char *name;
@@ refs/reftable-backend.c: static int reftable_be_reflog_expire(struct ref_store *
}
+static int reftable_be_fsck(struct ref_store *ref_store,
-+ struct fsck_options *o)
++ struct fsck_refs_options *o)
+{
+ return 0;
+}
3: f00acab4d1 ! 8: 534a3d00af builtin/refs: add verify subcommand
@@ builtin/refs.c: static int cmd_refs_migrate(int argc, const char **argv, const c
+static int cmd_refs_verify(int argc, const char **argv, const char *prefix)
+{
++ struct fsck_refs_options fsck_refs_options = FSCK_REFS_OPTIONS_DEFAULT;
+ const char * const verify_usage[] = {
+ REFS_VERIFY_USAGE,
+ NULL,
+ };
-+ int ret = 0;
+ unsigned int verbose = 0, strict = 0;
-+ struct fsck_options fsck_options = FSCK_OPTIONS_DEFAULT;
+ struct option options[] = {
+ OPT__VERBOSE(&verbose, N_("be verbose")),
+ OPT_BOOL(0, "strict", &strict, N_("enable strict checking")),
@@ builtin/refs.c: static int cmd_refs_migrate(int argc, const char **argv, const c
+ usage(_("too many arguments"));
+
+ if (verbose)
-+ fsck_options.verbose = 1;
++ fsck_refs_options.fsck_options.verbose = 1;
+ if (strict)
-+ fsck_options.strict = 1;
++ fsck_refs_options.fsck_options.strict = 1;
+
-+ git_config(git_fsck_config, &fsck_options);
++ git_config(git_fsck_config, &fsck_refs_options.fsck_options);
+ prepare_repo_settings(the_repository);
+
-+ ret = refs_fsck(get_main_ref_store(the_repository), &fsck_options);
-+
-+ /*
-+ * Explicitly free the allocated array and object skiplist set. Because
-+ * we reuse `git_fsck_config` here. It will still set the skiplist.
-+ */
-+ free(fsck_options.msg_type);
-+ oidset_clear(&fsck_options.objs_options.skiplist);
-+ return ret;
++ return refs_fsck(get_main_ref_store(the_repository), &fsck_refs_options);
+}
+
int cmd_refs(int argc, const char **argv, const char *prefix)
4: 6494f5b0c2 = 9: 054c8cea89 builtin/fsck: add `git-refs verify` child process
5: 2c064f4f71 ! 10: 7f87c6a26e files-backend: add unified interface for refs scanning
@@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
+ * the whole directory. This function is used as the callback for each
+ * regular file or symlink in the directory.
+ */
-+typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
++typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o,
+ const char *gitdir,
+ const char *refs_check_dir,
+ struct dir_iterator *iter);
+
+static int files_fsck_refs_dir(struct ref_store *ref_store,
-+ struct fsck_options *o,
++ struct fsck_refs_options *o,
+ const char *refs_check_dir,
+ files_fsck_refs_fn *fsck_refs_fns)
+{
@@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
+ continue;
+ } else if (S_ISREG(iter->st.st_mode) ||
+ S_ISLNK(iter->st.st_mode)) {
-+ if (o->verbose)
++ if (o->fsck_options.verbose)
+ fprintf_ln(stderr, "Checking %s/%s",
+ refs_check_dir, iter->relative_path);
+ for (size_t i = 0; fsck_refs_fns[i]; i++) {
@@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
+}
+
+static int files_fsck_refs(struct ref_store *ref_store,
-+ struct fsck_options *o)
++ struct fsck_refs_options *o)
+{
+ int ret;
+ files_fsck_refs_fn fsck_refs_fns[]= {
+ NULL
+ };
+
-+ if (o->verbose)
++ if (o->fsck_options.verbose)
+ fprintf_ln(stderr, "Checking references consistency");
+
+ ret = files_fsck_refs_dir(ref_store, o, "refs", fsck_refs_fns);
@@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
+}
+
static int files_fsck(struct ref_store *ref_store,
- struct fsck_options *o)
+ struct fsck_refs_options *o)
{
@@ refs/files-backend.c: static int files_fsck(struct ref_store *ref_store,
struct files_ref_store *refs =
6: 88b54b9e1a ! 11: 7d78014e5f fsck: add ref name check for files backend
@@ fsck.h: enum fsck_msg_type {
FUNC(BAD_TREE_SHA1, ERROR) \
## refs/files-backend.c ##
-@@ refs/files-backend.c: typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
+@@ refs/files-backend.c: typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o,
const char *refs_check_dir,
struct dir_iterator *iter);
-+static int files_fsck_refs_name(struct fsck_options *o,
++static int files_fsck_refs_name(struct fsck_refs_options *o,
+ const char *gitdir UNUSED,
+ const char *refs_check_dir,
+ struct dir_iterator *iter)
@@ refs/files-backend.c: typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
+
+ if (check_refname_format(iter->basename, REFNAME_ALLOW_ONELEVEL)) {
+ strbuf_addf(&sb, "%s/%s", refs_check_dir, iter->relative_path);
-+ ret = fsck_refs_report(o, sb.buf,
++ ret = fsck_refs_report(o, NULL, sb.buf,
+ FSCK_MSG_BAD_REF_NAME,
+ "invalid refname format");
+ }
@@ refs/files-backend.c: typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
+}
+
static int files_fsck_refs_dir(struct ref_store *ref_store,
- struct fsck_options *o,
+ struct fsck_refs_options *o,
const char *refs_check_dir,
@@ refs/files-backend.c: static int files_fsck_refs(struct ref_store *ref_store,
{
7: 8309a4746a ! 12: 16209a73ed fsck: add ref content check for files backend
@@ refs/files-backend.c: int parse_loose_ref_contents(const char *buf, struct objec
return 0;
}
-@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
+@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *o,
return ret;
}
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
+ * would be the content after "refs:". For symblic link, "pointee_name" would
+ * be the relative path agaignst "gitdir".
+ */
-+static int files_fsck_symref_target(struct fsck_options *o,
++static int files_fsck_symref_target(struct fsck_refs_options *o,
+ const char *refname,
+ const char *pointee_name,
+ const char *pointee_path)
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
+
+ if (!skip_prefix(pointee_name, "refs/", &p)) {
+
-+ ret = fsck_refs_report(o, refname,
++ ret = fsck_refs_report(o, NULL, refname,
+ FSCK_MSG_BAD_SYMREF_POINTEE,
+ "point to target out of refs hierarchy");
+ goto out;
+ }
+
+ if (check_refname_format(pointee_name, 0)) {
-+ ret = fsck_refs_report(o, refname,
++ ret = fsck_refs_report(o, NULL, refname,
+ FSCK_MSG_BAD_SYMREF_POINTEE,
+ "point to invalid refname");
+ }
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
+ goto out;
+
+ if (!S_ISREG(st.st_mode) && !S_ISLNK(st.st_mode)) {
-+ ret = fsck_refs_report(o, refname,
++ ret = fsck_refs_report(o, NULL, refname,
+ FSCK_MSG_BAD_SYMREF_POINTEE,
+ "point to invalid target");
+ goto out;
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
+ return ret;
+}
+
-+static int files_fsck_refs_content(struct fsck_options *o,
++static int files_fsck_refs_content(struct fsck_refs_options *o,
+ const char *gitdir,
+ const char *refs_check_dir,
+ struct dir_iterator *iter)
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
+
+ if (!skip_prefix(pointee_path.buf,
+ abs_gitdir.buf, &pointee_name)) {
-+ ret = fsck_refs_report(o, refname.buf,
++ ret = fsck_refs_report(o, NULL, refname.buf,
+ FSCK_MSG_BAD_SYMREF_POINTEE,
+ "point to target outside gitdir");
+ goto clean;
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
+ if (parse_loose_ref_contents(ref_content.buf, &oid,
+ &referent, &type,
+ &failure_errno, &trailing)) {
-+ ret = fsck_refs_report(o, refname.buf,
++ ret = fsck_refs_report(o, NULL, refname.buf,
+ FSCK_MSG_BAD_REF_CONTENT,
+ "invalid ref content");
+ goto clean;
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
+ * be reported as a warning.
+ */
+ if (trailing && (*trailing != '\0' && *trailing != '\n')) {
-+ ret = fsck_refs_report(o, refname.buf,
++ ret = fsck_refs_report(o, NULL, refname.buf,
+ FSCK_MSG_TRAILING_REF_CONTENT,
+ "trailing garbage in ref");
+ goto clean;
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
+}
+
static int files_fsck_refs_dir(struct ref_store *ref_store,
- struct fsck_options *o,
+ struct fsck_refs_options *o,
const char *refs_check_dir,
@@ refs/files-backend.c: static int files_fsck_refs(struct ref_store *ref_store,
int ret;Signature · 1 line
2.45.2
PATCH v5 1/12 fsck: rename "fsck_options" to "fsck_objects_options"
The git-fsck(1) focuses on object database consistency check. It relies on the "fsck_options" to interact with fsck error levels. However, "fsck_options" aims at checking the object database which contains a lot of fields only related to object database.
In order to provide git-fsck(1) with refs consistency check. Rename "fsck_options" to "fsck_objects_options" to explicitly indicate this structure is used to check the object database.
--- builtin/fsck.c | 13 ++++---- builtin/index-pack.c | 18 +++++------ builtin/mktag.c | 13 ++++---- builtin/unpack-objects.c | 14 ++++----- fetch-pack.c | 12 ++++---- fsck.c | 64 ++++++++++++++++++++++------------------ fsck.h | 43 ++++++++++++++------------- object-file.c | 4 +-- 8 files changed, 95 insertions(+), 86 deletions(-)
builtin/fsck.c +7 -6
@@ -42,8 +42,8 @@ static int check_full = 1;static int connectivity_only;static int check_strict;static int keep_cache_objects; -static struct fsck_options fsck_walk_options =FSCK_OPTIONS_DEFAULT ; -static struct fsck_options fsck_obj_options =FSCK_OPTIONS_DEFAULT ; +static struct fsck_objects_options fsck_walk_options =FSCK_OBJECTS_OPTIONS_DEFAULT ; +static struct fsck_objects_options fsck_obj_options =FSCK_OBJECTS_OPTIONS_DEFAULT ;static int errors_found;static int write_lost_and_found;static int verbose;
@@ -89,7 +89,7 @@ static int objerror(struct object *obj, const char *err)return -1 ; } -static int fsck_error_func(struct fsck_options *o UNUSED, +static int fsck_error_func(struct fsck_objects_options *o UNUSED, const struct object_id *oid, enum object_type object_type, enum fsck_msg_type msg_type,
@@ -118,7 +118,7 @@ static int fsck_error_func(struct fsck_options *o UNUSED,static struct object_array pending;static int mark_object (struct object *obj,enum object_type type, - void *data, struct fsck_options *options UNUSED) + void *data, struct fsck_objects_options *options UNUSED) {struct object *parent = data;
@@ -204,7 +204,8 @@ static int traverse_reachable(void) } static int mark_used(struct object *obj, enum object_type type UNUSED, - void *data UNUSED, struct fsck_options *options UNUSED) + void *data UNUSED, + struct fsck_objects_options *options UNUSED) {if (!obj)return 1 ;
@@ -214,7 +215,7 @@ static int mark_used(struct object *obj, enum object_type type UNUSED,static void mark_unreachable_referents (const struct object_id *oid) { -struct fsck_options options =FSCK_OPTIONS_DEFAULT ; +struct fsck_objects_options options =FSCK_OBJECTS_OPTIONS_DEFAULT ;struct object *obj =lookup_object (the_repository, oid);if (!obj || !(obj->flags &HAS_OBJ ))
builtin/index-pack.c +9 -9
@@ -127,7 +127,7 @@ static int nr_threads;static int from_stdin;static int strict;static int do_fsck_object; -static struct fsck_options fsck_options =FSCK_OPTIONS_MISSING_GITMODULES ; +static struct fsck_objects_options fsck_objects_options =FSCK_OBJECTS_OPTIONS_MISSING_GITMODULES ;static int verbose;static const char *progress_title;static int show_resolving_progress;
@@ -220,7 +220,7 @@ static void cleanup_thread(void)static int mark_link (struct object *obj,enum object_type type, void *data UNUSED, - struct fsck_options *options UNUSED) + struct fsck_objects_options *options UNUSED) {if (!obj)return -1 ;
@@ -852,7 +852,7 @@ static void sha1_object(const void *data, struct object_entry *obj_entry,else die (_ ("invalid blob object %s" ),oid_to_hex (oid));if (do_fsck_object && -fsck_object (&blob->object, (void *)data, size, &fsck_options)) +fsck_object (&blob->object, (void *)data, size, &fsck_objects_options))die (_ ("fsck error in packed object" )); }else {struct object *obj;
@@ -871,9 +871,9 @@ static void sha1_object(const void *data, struct object_entry *obj_entry,if (!obj)die (_ ("invalid %s" ),type_name (type));if (do_fsck_object && -fsck_object (obj, buf, size, &fsck_options)) +fsck_object (obj, buf, size, &fsck_objects_options))die (_ ("fsck error in packed object" )); -if (strict &&fsck_walk (obj,NULL , &fsck_options)) +if (strict &&fsck_walk (obj,NULL , &fsck_objects_options))die (_ ("Not all child objects of %s are reachable" ),oid_to_hex (&obj->oid));if (obj->type ==OBJ_TREE ) {
@@ -1746,7 +1746,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)usage (index_pack_usage);disable_replace_refs (); - fsck_options.walk = mark_link; + fsck_objects_options.walk = mark_link;reset_pack_idx_option (&opts); opts.flags |=WRITE_REV ;
@@ -1770,13 +1770,13 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix) }else if (skip_to_optional_arg (arg,"--strict" , &arg)) { strict =1 ; do_fsck_object =1 ; -fsck_set_msg_types (&fsck_options, arg); +fsck_set_msg_types (&fsck_objects_options, arg); }else if (!strcmp (arg,"--check-self-contained-and-connected" )) { strict =1 ; check_self_contained_and_connected =1 ; }else if (skip_to_optional_arg (arg,"--fsck-objects" , &arg)) { do_fsck_object =1 ; -fsck_set_msg_types (&fsck_options, arg); +fsck_set_msg_types (&fsck_objects_options, arg); }else if (!strcmp (arg,"--verify" )) { verify =1 ; }else if (!strcmp (arg,"--verify-stat" )) {
@@ -1943,7 +1943,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)else close (input_fd); -if (do_fsck_object &&fsck_finish (&fsck_options)) +if (do_fsck_object &&fsck_finish (&fsck_objects_options))die (_ ("fsck error in pack objects" ));free (opts.anomaly);
builtin/mktag.c +7 -6
@@ -15,9 +15,9 @@ static char const * const builtin_mktag_usage[] = { };static int option_strict =1 ; -static struct fsck_options fsck_options =FSCK_OPTIONS_STRICT ; +static struct fsck_objects_options fsck_objects_options =FSCK_OBJECTS_OPTIONS_STRICT ; -static int mktag_fsck_error_func(struct fsck_options *o UNUSED, +static int mktag_fsck_error_func(struct fsck_objects_options *o UNUSED, const struct object_id *oid UNUSED, enum object_type object_type UNUSED, enum fsck_msg_type msg_type,
@@ -91,12 +91,13 @@ int cmd_mktag(int argc, const char **argv, const char *prefix)if (strbuf_read (&buf,0 ,0 ) <0 )die_errno (_ ("could not read from stdin" )); - fsck_options.error_func = mktag_fsck_error_func; -fsck_set_msg_type_from_ids (&fsck_options,FSCK_MSG_EXTRA_HEADER_ENTRY , + fsck_objects_options.error_func = mktag_fsck_error_func; +fsck_set_msg_type_from_ids (&fsck_objects_options, +FSCK_MSG_EXTRA_HEADER_ENTRY ,FSCK_WARN );/* config might set fsck.extraHeaderEntry=* again */ -git_config (git_fsck_config, &fsck_options); -if (fsck_tag_standalone (NULL , buf.buf, buf.len, &fsck_options, +git_config (git_fsck_config, &fsck_objects_options); +if (fsck_tag_standalone (NULL , buf.buf, buf.len, &fsck_objects_options, &tagged_oid, &tagged_type))die (_ ("tag on stdin did not pass our strict fsck check" ));
builtin/unpack-objects.c +7 -7
@@ -25,7 +25,7 @@ static unsigned int offset, len;static off_t consumed_bytes;static off_t max_input_size;static git_hash_ctx ctx; -static struct fsck_options fsck_options =FSCK_OPTIONS_STRICT ; +static struct fsck_objects_options fsck_objects_options =FSCK_OBJECTS_OPTIONS_STRICT ;static struct progress *progress;/*
@@ -212,7 +212,7 @@ static void write_cached_object(struct object *obj, struct obj_buffer *obj_buf)*/ static int check_object (struct object *obj,enum object_type type, void *data UNUSED, - struct fsck_options *options UNUSED) + struct fsck_objects_options *options UNUSED) {struct obj_buffer *obj_buf;
@@ -237,10 +237,10 @@ static int check_object(struct object *obj, enum object_type type, obj_buf =lookup_object_buffer (obj);if (!obj_buf)die ("Whoops! Cannot find object '%s'" ,oid_to_hex (&obj->oid)); -if (fsck_object (obj, obj_buf->buffer, obj_buf->size, &fsck_options)) +if (fsck_object (obj, obj_buf->buffer, obj_buf->size, &fsck_objects_options))die ("fsck error in packed object" ); - fsck_options.walk = check_object; -if (fsck_walk (obj,NULL , &fsck_options)) + fsck_objects_options.walk = check_object; +if (fsck_walk (obj,NULL , &fsck_objects_options))die ("Error on reachable objects of %s" ,oid_to_hex (&obj->oid));write_cached_object (obj, obj_buf);return 0 ;
@@ -635,7 +635,7 @@ int cmd_unpack_objects(int argc, const char **argv, const char *prefix UNUSED) }if (skip_prefix (arg,"--strict=" , &arg)) { strict =1 ; -fsck_set_msg_types (&fsck_options, arg); +fsck_set_msg_types (&fsck_objects_options, arg);continue ; }if (starts_with (arg,"--pack_header=" )) {
@@ -671,7 +671,7 @@ int cmd_unpack_objects(int argc, const char **argv, const char *prefix UNUSED) the_hash_algo->final_oid_fn (&oid, &tmp_ctx);if (strict) {write_rest (); -if (fsck_finish (&fsck_options)) +if (fsck_finish (&fsck_objects_options))die (_ ("fsck error in pack objects" )); }if (!hasheq (fill (the_hash_algo->rawsz), oid.hash))
fetch-pack.c +6 -6
@@ -46,7 +46,7 @@ static int server_supports_filtering;static int advertise_sid;static struct shallow_lock shallow_lock;static const char *alternate_shallow_file; -static struct fsck_options fsck_options =FSCK_OPTIONS_MISSING_GITMODULES ; +static struct fsck_objects_options fsck_objects_options =FSCK_OBJECTS_OPTIONS_MISSING_GITMODULES ;static struct strbuf fsck_msg_types =STRBUF_INIT ;static struct string_list uri_protocols =STRING_LIST_INIT_DUP ;
@@ -1220,9 +1220,9 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, }else alternate_shallow_file =NULL ;if (get_pack (args, fd, pack_lockfiles,NULL , sought, nr_sought, - &fsck_options.gitmodules_found)) + &fsck_objects_options.gitmodules_found))die (_ ("git fetch-pack: fetch failed." )); -if (fsck_finish (&fsck_options)) +if (fsck_finish (&fsck_objects_options))die ("fsck failed" ); all_done:
@@ -1780,7 +1780,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,if (get_pack (args, fd, pack_lockfiles, packfile_uris.nr ? &index_pack_args :NULL , - sought, nr_sought, &fsck_options.gitmodules_found)) + sought, nr_sought, &fsck_objects_options.gitmodules_found))die (_ ("git fetch-pack: fetch failed." ));do_check_stateless_delimiter (args->stateless_rpc, &reader);
@@ -1823,7 +1823,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, packname[the_hash_algo->hexsz] ='\0' ; -parse_gitmodules_oids (cmd.out, &fsck_options.gitmodules_found); +parse_gitmodules_oids (cmd.out, &fsck_objects_options.gitmodules_found);close (cmd.out);
@@ -1844,7 +1844,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,string_list_clear (&packfile_uris,0 );strvec_clear (&index_pack_args); -if (fsck_finish (&fsck_options)) +if (fsck_finish (&fsck_objects_options))die ("fsck failed" );if (negotiator)
fsck.c +35 -29
237 lines not on this page · raw message
fsck.h +22 -21
133 lines not on this page · raw message
object-file.c +2 -2
@@ -2472,7 +2472,7 @@ int repo_has_object_file(struct repository *r,* report the minimal fsck error here, and rely on the caller to * give more context. */ -static int hash_format_check_report(struct fsck_options *opts UNUSED, +static int hash_format_check_report(struct fsck_objects_options *opts UNUSED, const struct object_id *oid UNUSED, enum object_type object_type UNUSED, enum fsck_msg_type msg_type UNUSED,
@@ -2507,7 +2507,7 @@ static int index_mem(struct index_state *istate, } }if (flags &HASH_FORMAT_CHECK ) { -struct fsck_options opts =FSCK_OPTIONS_DEFAULT ; +struct fsck_objects_options opts =FSCK_OBJECTS_OPTIONS_DEFAULT ; opts.strict =1 ; opts.error_func = hash_format_check_report;
Signature · 1 line
2.45.2
Re: [GSoC][PATCH v5 01/12] fsck: rename "fsck_options" to "fsck_objects_options"
builtin/fsck.c static int objerror(struct object *obj, const char *err) · line 95
Quoted 5 lines · shejialuo
builtin/fsck.c +1
-1-static int fsck_error_func(struct fsck_options *o UNUSED, +static int fsck_error_func(struct fsck_objects_options *o UNUSED, const struct object_id *oid, enum object_type object_type, enum fsck_msg_type msg_type,
It is curious that the addition/renaming of fsck_objects_options is presumably to allow fsck_${xyzzy}_options to be added for different $xyzzy (the first one being "refs"), and this function is only about fsck_objects_options. What name would the corresponding error function, called by checkers that take fsck_${xyzzy}_options, be given? fsck_${xyzzy}_error_func()? Shouldn't this be then become fsck_objects_error_func() or something?
Having said that.
Do we really need such a parallel system between "objects" and other kinds of things being checked that you are introducing with this step? What benefit are we getting from this additional complexity?
I would have expected that adding ref-related new members that object consistency checkers has no interest in to the fsck_options structure would be sufficient for the purpose of this series. Or if we really wanted to prepare for more complex future, use of the "union of variants, switched with a tag" pattern to arrange the data this way:
struct fsck_options {
enum fsck_type {
FSCK_OBJECTS,
FSCK_REFS,
...
} t;
union {
struct fsck_objects_options objects;
struct fsck_refs_options refs;
} u;
};would still allow functions like fsck_error_func(), and fsck_set_msg_types(), etc. to work on the common "fsck_options".
I dunno.
Re: [GSoC][PATCH v5 01/12] fsck: rename "fsck_options" to "fsck_objects_options"
Quoted 16 lines · Junio C Hamano
builtin/fsck.c static int objerror(struct object *obj, const char *err) · line 95
Quoted 5 lines · shejialuo
builtin/fsck.c +1
-1-static int fsck_error_func(struct fsck_options *o UNUSED, +static int fsck_error_func(struct fsck_objects_options *o UNUSED, const struct object_id *oid, enum object_type object_type, enum fsck_msg_type msg_type,It is curious that the addition/renaming of
fsck_objects_optionsis presumably to allow fsck_${xyzzy}_options to be added for different $xyzzy (the first one being "refs"), and this function is only aboutfsck_objects_options. What name would the corresponding error function, called by checkers that take fsck_${xyzzy}_options, be given? fsck_${xyzzy}_error_func()? Shouldn't this be then becomefsck_objects_error_func()or something?
Yes, it should be definitely changed here. Will improve in the next version.
Quoted 6 lines
Having said that.
Do we really need such a parallel system between "objects" and other kinds of things being checked that you are introducing with this step? What benefit are we getting from this additional complexity?
I am agree that the most simple way to handle for this series is add some ref-related new members. Thus, we can reuse existing code. However, it makes me feel so weird when implementing the code using this idea. For example,
struct fsck_options {
struct fsck_refs_options;
...
}When we create a new "fsck_options", it will be so misleading that the caller may think we will handle both refs and objects checks by using "fsck_options". So I just introduce this parallel system. When checking objects, caller should explicitly create "fsck_objects_options", when checking refs, caller should explicitly create "fsck_refs_options".
Because in semantics, we introduce a new check here. Combination means we will check the both. Although it is simple, but it will cause a lot of trouble in the future.
Quoted 19 lines
I would have expected that adding ref-related new members that object consistency checkers has no interest in to the
fsck_optionsstructure would be sufficient for the purpose of this series. Or if we really wanted to prepare for more complex future, use of the "union of variants, switched with a tag" pattern to arrange the data this way:struct fsck_options { enum fsck_type { FSCK_OBJECTS, FSCK_REFS, ... } t; union { struct fsck_objects_options objects; struct fsck_refs_options refs; } u; };
Quoted 3 lines
would still allow functions like
fsck_error_func(), andfsck_set_msg_types(), etc. to work on the common "fsck_options".
I agree that we could use this pattern, using union will make the semantics more clear.
Quoted 1 line
I dunno.
PATCH v5 2/12 fsck: use "fsck_configs" to set up configs
In reply to shejialuoSome fields such as "msg_type" and "skiplist" in "fsck_objects_options" are not options, these fields are related to "git-config(1)" which are initialized using "git_fsck_config" function. Create a static variable named "fsck_configs" in "fsck.c" which aims at handling configs. Thus we don't need to reply on the "fsck_objects_options" to set up the fsck error message severity.
--- fsck.c | 29 ++++++++++++++++++----------- fsck.h | 3 --- 2 files changed, 18 insertions(+), 14 deletions(-)
fsck.c +18 -11
@@ -24,6 +24,14 @@static ssize_t max_tree_entry_len =4096 ; +static struct { +enum fsck_msg_type *msg_type; +struct oidset oid_skiplist; +} fsck_configs = { + .msg_type =NULL , + .oid_skiplist =OIDSET_INIT +}; +#define STR (x) #x#define MSG_ID (id, msg_type) { STR(id), NULL, NULL, FSCK_##msg_type }, static struct {
@@ -103,7 +111,7 @@ static enum fsck_msg_type fsck_msg_type(enum fsck_msg_id msg_id, {assert (msg_id >=0 && msg_id <FSCK_MSG_MAX ); -if (!options->msg_type) { +if (!fsck_configs.msg_type) {enum fsck_msg_type msg_type = msg_id_info[msg_id].msg_type;if (options->strict && msg_type ==FSCK_WARN )
@@ -111,7 +119,7 @@ static enum fsck_msg_type fsck_msg_type(enum fsck_msg_id msg_id,return msg_type; } -return options->msg_type[msg_id]; +return fsck_configs.msg_type[msg_id]; }static enum fsck_msg_type parse_msg_type (const char *str)
@@ -138,16 +146,16 @@ void fsck_set_msg_type_from_ids(struct fsck_objects_options *options, enum fsck_msg_id msg_id, enum fsck_msg_type msg_type) { -if (!options->msg_type) { +if (!fsck_configs.msg_type) {int i;enum fsck_msg_type *severity;ALLOC_ARRAY (severity,FSCK_MSG_MAX );for (i =0 ; i <FSCK_MSG_MAX ; i++) severity[i] =fsck_msg_type (i, options); - options->msg_type = severity; + fsck_configs.msg_type = severity; } - options->msg_type[msg_id] = msg_type; + fsck_configs.msg_type[msg_id] = msg_type; } void fsck_set_msg_type(struct fsck_objects_options *options,
@@ -203,7 +211,7 @@ void fsck_set_msg_types(struct fsck_objects_options *options, const char *valuesif (!strcmp (buf,"skiplist" )) {if (equal == len)die ("skiplist requires a path" ); -oidset_parse_file (&options->skiplist, buf + equal +1 ); +oidset_parse_file (&fsck_configs.oid_skiplist, buf + equal +1 ); buf += len +1 ;continue ; }
@@ -217,10 +225,9 @@ void fsck_set_msg_types(struct fsck_objects_options *options, const char *valuesfree (to_free); } -static int object_on_skiplist (struct fsck_objects_options *opts, -const struct object_id *oid) +static int object_on_skiplist (const struct object_id *oid) { -return opts && oid &&oidset_contains (&opts->skiplist, oid); +return oid &&oidset_contains (&fsck_configs.oid_skiplist, oid); } __attribute__((format (printf, 5, 6)))
@@ -236,7 +243,7 @@ static int report(struct fsck_objects_options *options,if (msg_type ==FSCK_IGNORE )return 0 ; -if (object_on_skiplist (options, oid)) +if (object_on_skiplist (oid))return 0 ;if (msg_type ==FSCK_FATAL )
@@ -1109,7 +1116,7 @@ static int fsck_blob(const struct object_id *oid, const char *buf, {int ret =0 ; -if (object_on_skiplist (options, oid)) +if (object_on_skiplist (oid))return 0 ;if (oidset_contains (&options->gitmodules_found, oid)) {
fsck.h +0 -3
@@ -135,8 +135,6 @@ struct fsck_objects_options {fsck_walk_func walk;fsck_error error_func; unsigned strict:1; -enum fsck_msg_type *msg_type; -struct oidset skiplist;struct oidset gitmodules_found;struct oidset gitmodules_done;struct oidset gitattributes_found;
@@ -145,7 +143,6 @@ struct fsck_objects_options { };#define FSCK_OBJECTS_OPTIONS_DEFAULT { \ - .skiplist = OIDSET_INIT, \ .gitmodules_found = OIDSET_INIT, \ .gitmodules_done = OIDSET_INIT, \ .gitattributes_found = OIDSET_INIT, \
Signature · 1 line
2.45.2
Re: [GSoC][PATCH v5 02/12] fsck: use "fsck_configs" to set up configs
Quoted 6 lines · shejialuo
Some fields such as "
msg_type" and "skiplist" in "fsck_objects_options" are not options, these fields are related to "git-config(1)" which are initialized using "git_fsck_config" function. Create a static variable named "fsck_configs" in "fsck.c" which aims at handling configs. Thus we don't need to reply on the "fsck_objects_options" to set up the fsck error message severity.
reply???
As configuration often is used to prepopulate options, I need a lot stonger justification to split these into a different structure than "'config' is a noun that is different from a noun 'option'".
If we intend to have many "option" instances but what these two members store will be the same across these "option" instances, for example, that would be a lot better reason why we may want to separate these two members out of it, but I have a suspicion that if we were to use the "union with tags" approach, these two would become members of the common part shared between "objects' and "refs", i.e. the overall data structure might look more like this:
struct fsck_options {
enum fsck_msg_type *msg_type;
struct oidset oid_skiplist;
enum fsck_what_check { O, R } tag;
union {
struct fsck_object_options o;
struct fsck_ref_options r;
} u;
};by moving these two members out of fsck_object_options and moving them to the shared part.
I dunno. It is unclear what the real reason is for these two things to be extracted out and made to appear totally independent from the "options" thing to begin with, and I am not sure if I agree with the reason when it is known.
Re: [GSoC][PATCH v5 02/12] fsck: use "fsck_configs" to set up configs
Quoted 11 lines · Junio C Hamano
Quoted 6 lines · shejialuo
Some fields such as "
msg_type" and "skiplist" in "fsck_objects_options" are not options, these fields are related to "git-config(1)" which are initialized using "git_fsck_config" function. Create a static variable named "fsck_configs" in "fsck.c" which aims at handling configs. Thus we don't need to reply on the "fsck_objects_options" to set up the fsck error message severity.reply???
Sorry, I often make mistake to type "rely" as "reply".
Quoted 4 lines
As configuration often is used to prepopulate options, I need a lot stonger justification to split these into a different structure than "'config' is a noun that is different from a noun 'option'".
Quoted 8 lines
If we intend to have many "option" instances but what these two members store will be the same across these "option" instances, for example, that would be a lot better reason why we may want to separate these two members out of it, but I have a suspicion that if we were to use the "union with tags" approach, these two would become members of the common part shared between "objects' and "refs", i.e. the overall data structure might look more like this:
Actually, I feel really wired for this part. Let me elaborate on this. "fsck.c::git_fsck_config()" is used to set up the configs. It will eventually call the "fsck.c::fsck_set_msg_type_from_ids" like the following:
void fsck_set_msg_type_from_ids(struct fsck_options *options,
enum fsck_msg_id msg_id,
enum fsck_msg_type msg_type)
{
if (!options->msg_type) {
int i;
enum fsck_msg_type *severity.
ALLOC_ARRAY(severity, FSCK_MSG_MAX);
for (i = 0; i < FSCK_MSG_MAX; i++)
severity[i] = fsck_msg_type(i, options);
options->msg_type = severity;
}options->msg_type[msg_id] = msg_type; }
In the current codebase, the caller will simply create a "fsck_options" and setup the fsck error message severity. However, let's see "builtin/fskc.c", it creates the following two "fsck_options" and it only uses
static struct fsck_options fsck_walk_options = FSCK_OPTIONS_DEFAULT; static struct fsck_options fsck_obj_options = FSCK_OPTIONS_DEFAULT;
However, the code only uses "fsck_obj_options" to setup the configs. So it makes me feel so strange. So I just want to make it separation. Maybe a little wrong here.
Quoted 17 lines
struct fsck_options { enum fsck_msg_type *msg_type; struct oidset oid_skiplist; enum fsck_what_check { O, R } tag; union { struct fsck_object_options o; struct fsck_ref_options r; } u; };by moving these two members out of
fsck_object_optionsand moving them to the shared part.I dunno. It is unclear what the real reason is for these two things to be extracted out and made to appear totally independent from the "options" thing to begin with, and I am not sure if I agree with the reason when it is known.
PATCH v5 3/12 fsck: abstract common options for reusing
In reply to shejialuoCreate "fsck_options" struct to contain the general fields. Change "fsck_objects_options" to incorporate the general "fsck_options" and object-speicifc options.
The fsck message will use "strict" option to set the corresponding "msg_type". Rename the parameter name from "struct fsck_objects_options *" to "struct fsck_options *".
--- builtin/fsck.c | 6 +++--- builtin/index-pack.c | 4 ++-- builtin/mktag.c | 6 +++--- builtin/unpack-objects.c | 2 +- fsck.c | 25 ++++++++++++++++--------- fsck.h | 32 ++++++++++++++++++++++---------- object-file.c | 4 ++-- 7 files changed, 49 insertions(+), 30 deletions(-)
builtin/fsck.c +3 -3
@@ -939,9 +939,9 @@ int cmd_fsck(int argc, const char **argv, const char *prefix) fsck_walk_options.walk = mark_object; fsck_obj_options.walk = mark_used; - fsck_obj_options.error_func = fsck_error_func; + fsck_obj_options.fsck_options.error_func = fsck_error_func;if (check_strict) - fsck_obj_options.strict =1 ; + fsck_obj_options.fsck_options.strict =1 ;if (show_progress ==-1 ) show_progress =isatty (2 );
@@ -956,7 +956,7 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)if (name_objects)fsck_enable_object_names (&fsck_walk_options); -git_config (git_fsck_config, &fsck_obj_options); +git_config (git_fsck_config, &fsck_obj_options.fsck_options);prepare_repo_settings (the_repository);if (connectivity_only) {
builtin/index-pack.c +2 -2
@@ -1770,13 +1770,13 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix) }else if (skip_to_optional_arg (arg,"--strict" , &arg)) { strict =1 ; do_fsck_object =1 ; -fsck_set_msg_types (&fsck_objects_options, arg); +fsck_set_msg_types (&fsck_objects_options.fsck_options, arg); }else if (!strcmp (arg,"--check-self-contained-and-connected" )) { strict =1 ; check_self_contained_and_connected =1 ; }else if (skip_to_optional_arg (arg,"--fsck-objects" , &arg)) { do_fsck_object =1 ; -fsck_set_msg_types (&fsck_objects_options, arg); +fsck_set_msg_types (&fsck_objects_options.fsck_options, arg); }else if (!strcmp (arg,"--verify" )) { verify =1 ; }else if (!strcmp (arg,"--verify-stat" )) {
builtin/mktag.c +3 -3
@@ -91,12 +91,12 @@ int cmd_mktag(int argc, const char **argv, const char *prefix)if (strbuf_read (&buf,0 ,0 ) <0 )die_errno (_ ("could not read from stdin" )); - fsck_objects_options.error_func = mktag_fsck_error_func; -fsck_set_msg_type_from_ids (&fsck_objects_options, + fsck_objects_options.fsck_options.error_func = mktag_fsck_error_func; +fsck_set_msg_type_from_ids (&fsck_objects_options.fsck_options,FSCK_MSG_EXTRA_HEADER_ENTRY ,FSCK_WARN );/* config might set fsck.extraHeaderEntry=* again */ -git_config (git_fsck_config, &fsck_objects_options); +git_config (git_fsck_config, &fsck_objects_options.fsck_options);if (fsck_tag_standalone (NULL , buf.buf, buf.len, &fsck_objects_options, &tagged_oid, &tagged_type))die (_ ("tag on stdin did not pass our strict fsck check" ));
builtin/unpack-objects.c +1 -1
@@ -635,7 +635,7 @@ int cmd_unpack_objects(int argc, const char **argv, const char *prefix UNUSED) }if (skip_prefix (arg,"--strict=" , &arg)) { strict =1 ; -fsck_set_msg_types (&fsck_objects_options, arg); +fsck_set_msg_types (&fsck_objects_options.fsck_options, arg);continue ; }if (starts_with (arg,"--pack_header=" )) {
fsck.c +16 -9
@@ -107,7 +107,7 @@ void list_config_fsck_msg_ids(struct string_list *list, const char *prefix) }static enum fsck_msg_type fsck_msg_type (enum fsck_msg_id msg_id, -struct fsck_objects_options *options) +struct fsck_options *options) {assert (msg_id >=0 && msg_id <FSCK_MSG_MAX );
@@ -142,7 +142,7 @@ int is_valid_msg_type(const char *msg_id, const char *msg_type)return 1 ; } -void fsck_set_msg_type_from_ids (struct fsck_objects_options *options, +void fsck_set_msg_type_from_ids (struct fsck_options *options,enum fsck_msg_id msg_id,enum fsck_msg_type msg_type) {
@@ -158,7 +158,7 @@ void fsck_set_msg_type_from_ids(struct fsck_objects_options *options, fsck_configs.msg_type[msg_id] = msg_type; } -void fsck_set_msg_type (struct fsck_objects_options *options, +void fsck_set_msg_type (struct fsck_options *options,const char *msg_id_str,const char *msg_type_str) {int msg_id =parse_msg_id (msg_id_str);
@@ -187,7 +187,7 @@ void fsck_set_msg_type(struct fsck_objects_options *options,free (to_free); } -void fsck_set_msg_types (struct fsck_objects_options *options,const char *values) +void fsck_set_msg_types (struct fsck_options *options,const char *values) {char *buf =xstrdup (values), *to_free = buf;int done =0 ;
@@ -237,9 +237,16 @@ static int report(struct fsck_objects_options *options, {va_list ap;struct strbuf sb =STRBUF_INIT ; -enum fsck_msg_type msg_type =fsck_msg_type (msg_id, options); +struct fsck_options *fsck_options; +enum fsck_msg_type msg_type;int result; +if (options) + fsck_options = &options->fsck_options; +else +BUG ("fsck_options is not set" ); + + msg_type =fsck_msg_type (msg_id, fsck_options);if (msg_type ==FSCK_IGNORE )return 0 ;
@@ -256,8 +263,8 @@ static int report(struct fsck_objects_options *options,va_start (ap, fmt);strbuf_vaddf (&sb, fmt, ap); - result = options->error_func (options, oid, object_type, - msg_type, msg_id, sb.buf); + result = fsck_options->error_func (options, oid, object_type, + msg_type, msg_id, sb.buf);strbuf_release (&sb);va_end (ap);
@@ -711,7 +718,7 @@ static int fsck_tree(const struct object_id *tree_oid,* bits.. */ case S_IFREG |0664 : -if (!options->strict) +if (!options->fsck_options.strict)break ;/* fallthrough */ default :
@@ -1283,7 +1290,7 @@ int fsck_finish(struct fsck_objects_options *options)int git_fsck_config (const char *var,const char *value,const struct config_context *ctx,void *cb) { -struct fsck_objects_options *options = cb; +struct fsck_options *options = cb;const char *msg_id;if (strcmp (var,"fsck.skiplist" ) ==0 ) {
fsck.h +22 -10
@@ -92,15 +92,16 @@ enum fsck_msg_id { };#undef MSG_ID +struct fsck_options ;struct fsck_objects_options ;struct object ; -void fsck_set_msg_type_from_ids (struct fsck_objects_options *options, +void fsck_set_msg_type_from_ids (struct fsck_options *options,enum fsck_msg_id msg_id,enum fsck_msg_type msg_type); -void fsck_set_msg_type (struct fsck_objects_options *options, +void fsck_set_msg_type (struct fsck_options *options,const char *msg_id,const char *msg_type); -void fsck_set_msg_types (struct fsck_objects_options *options,const char *values); +void fsck_set_msg_types (struct fsck_options *options,const char *values);int is_valid_msg_type (const char *msg_id,const char *msg_type);/*
@@ -131,10 +132,15 @@ int fsck_error_cb_print_missing_gitmodules(struct fsck_objects_options *o,enum fsck_msg_id msg_id, const char *message); +struct fsck_options { +fsck_error error_func; +unsigned verbose:1 , + strict:1 ; +}; +struct fsck_objects_options { +struct fsck_options fsck_options;fsck_walk_func walk; -fsck_error error_func; -unsigned strict:1 ;struct oidset gitmodules_found;struct oidset gitmodules_done;struct oidset gitattributes_found;
@@ -143,27 +149,33 @@ struct fsck_objects_options { };#define FSCK_OBJECTS_OPTIONS_DEFAULT { \ + .fsck_options = { \ + .error_func = fsck_error_function, \ + }, \ .gitmodules_found = OIDSET_INIT, \ .gitmodules_done = OIDSET_INIT, \ .gitattributes_found = OIDSET_INIT, \ .gitattributes_done = OIDSET_INIT, \ - .error_func = fsck_error_function \ }#define FSCK_OBJECTS_OPTIONS_STRICT { \ - .strict = 1, \ + .fsck_options = { \ + .error_func = fsck_error_function, \ + .strict = 1, \ + }, \ .gitmodules_found = OIDSET_INIT, \ .gitmodules_done = OIDSET_INIT, \ .gitattributes_found = OIDSET_INIT, \ .gitattributes_done = OIDSET_INIT, \ - .error_func = fsck_error_function, \ }#define FSCK_OBJECTS_OPTIONS_MISSING_GITMODULES { \ - .strict = 1, \ + .fsck_options = { \ + .error_func = fsck_error_cb_print_missing_gitmodules, \ + .strict = 1, \ + }, \ .gitmodules_found = OIDSET_INIT, \ .gitmodules_done = OIDSET_INIT, \ .gitattributes_found = OIDSET_INIT, \ .gitattributes_done = OIDSET_INIT, \ - .error_func = fsck_error_cb_print_missing_gitmodules, \ }/* descend in all linked child objects
object-file.c +2 -2
@@ -2509,8 +2509,8 @@ static int index_mem(struct index_state *istate,if (flags &HASH_FORMAT_CHECK ) {struct fsck_objects_options opts =FSCK_OBJECTS_OPTIONS_DEFAULT ; - opts.strict =1 ; - opts.error_func = hash_format_check_report; + opts.fsck_options.strict =1 ; + opts.fsck_options.error_func = hash_format_check_report;if (fsck_buffer (null_oid (), type, buf, size, &opts))die (_ ("refusing to create malformed object" ));fsck_finish (&opts);
Signature · 1 line
2.45.2
PATCH v5 4/12 fsck: add "fsck_refs_options" struct
In reply to shejialuoAdd "fsck_refs_options" to support ref consistency checks.
--- fsck.h | 4 ++++ 1 file changed, 4 insertions(+)
fsck.h +4 -0
@@ -138,6 +138,10 @@ struct fsck_options { strict:1 ; }; +struct fsck_refs_options { +struct fsck_options fsck_options; +}; +struct fsck_objects_options {struct fsck_options fsck_options;fsck_walk_func walk;
Signature · 1 line
2.45.2
PATCH v5 5/12 fsck: add a unified interface for reporting fsck messages
In reply to shejialuoThe static function "report" provided by "fsck.c" aims at reporting the problems related to object database which cannot be reused for refs. In order to provide a unified interface which can report either objects or refs, create a new function "fsck_report" by adding two parameters "refs_options" and "checked_ref_name" following the "report" prototype. However, instead of using "...", provide "va_list" to allow more flexibility.
The "vfsck_report" function will use "error_func" registered in "fsck_options" function to report customized messages. Change "error_func" prototype to align with the "vfsck_report".
Change "report" function to make it use "vfsck_report" to report objects-related messages. Add a new function called "fsck_refs_report" to use "vfsck_report" to report refs-related messages. Also, create a general function "fsck_report" for future use.
--- builtin/fsck.c | 4 ++- builtin/mktag.c | 4 ++- fsck.c | 88 ++++++++++++++++++++++++++++++++++++++++--------- fsck.h | 39 +++++++++++++++++++--- object-file.c | 14 ++++---- 5 files changed, 122 insertions(+), 27 deletions(-)
builtin/fsck.c +3 -1
@@ -89,9 +89,11 @@ static int objerror(struct object *obj, const char *err)return -1 ; } -static int fsck_error_func(struct fsck_objects_options *o UNUSED, +static int fsck_error_func(struct fsck_objects_options *objects_options UNUSED, + struct fsck_refs_options *refs_options UNUSED,const struct object_id *oid,enum object_type object_type, + const char *checked_ref_name UNUSED,enum fsck_msg_type msg_type, enum fsck_msg_id msg_id UNUSED,const char *message)
builtin/mktag.c +3 -1
@@ -17,9 +17,11 @@ static int option_strict = 1;static struct fsck_objects_options fsck_objects_options =FSCK_OBJECTS_OPTIONS_STRICT ; -static int mktag_fsck_error_func(struct fsck_objects_options *o UNUSED, +static int mktag_fsck_error_func(struct fsck_objects_options *objects_options UNUSED, + struct fsck_refs_options *refs_options UNUSED, const struct object_id *oid UNUSED, enum object_type object_type UNUSED, + const char *checked_ref_name UNUSED,enum fsck_msg_type msg_type, enum fsck_msg_id msg_id UNUSED,const char *message)
fsck.c +73 -15
139 lines not on this page · raw message
fsck.h +35 -4
@@ -93,6 +93,7 @@ enum fsck_msg_id {#undef MSG_IDstruct fsck_options ; +struct fsck_refs_options ;struct fsck_objects_options ;struct object ;
@@ -115,19 +116,27 @@ int is_valid_msg_type(const char *msg_id, const char *msg_type);typedef int (*fsck_walk_func )(struct object *obj,enum object_type object_type,void *data,struct fsck_objects_options *options); -/* callback for fsck_object, type is FSCK_ERROR or FSCK_WARN */ -typedef int (*fsck_error )(struct fsck_objects_options *o, +/* +* callback function for reporting errors when checking either objects or refs +*/ +typedef int (*fsck_error )(struct fsck_objects_options *objects_options, +struct fsck_refs_options *refs_options,const struct object_id *oid,enum object_type object_type, +const char *checked_ref_name,enum fsck_msg_type msg_type,enum fsck_msg_id msg_id,const char *message); -int fsck_error_function (struct fsck_objects_options *o, +int fsck_error_function (struct fsck_objects_options *objects_options, +struct fsck_refs_options *refs_options,const struct object_id *oid,enum object_type object_type, +const char *checked_ref_name,enum fsck_msg_type msg_type,enum fsck_msg_id msg_id,const char *message); -int fsck_error_cb_print_missing_gitmodules (struct fsck_objects_options *o, +int fsck_error_cb_print_missing_gitmodules (struct fsck_objects_options *objects_options, +struct fsck_refs_options *refs_options,const struct object_id *oid,enum object_type object_type, +const char *checked_ref_name,enum fsck_msg_type msg_type,enum fsck_msg_id msg_id,const char *message);
@@ -223,6 +232,28 @@ int fsck_tag_standalone(const struct object_id *oid, const char *buffer,*/ int fsck_finish (struct fsck_objects_options *options); +/* +* Provide a unified interface for either fscking refs or objects. +* It will get the current msg error type and call the error_func callback +* which is registered in the "fsck_options" struct. For refs, the caller +* should pass NULL for "objs_options". For objects, the caller should pass +* NULL for "refs_options". +*/ +__attribute__((format (printf,7 ,8 ))) +int fsck_report (struct fsck_objects_options *objects_options, +struct fsck_refs_options *refs_options, +const struct object_id *oid, +enum object_type object_type, +const char *checked_ref_name, +enum fsck_msg_id msg_id,const char *fmt, ...); + +__attribute__((format (printf,5 ,6 ))) +int fsck_refs_report (struct fsck_refs_options *refs_options, +const struct object_id *oid, +const char *checked_ref_name, +enum fsck_msg_id msg_id, +const char *fmt, ...); +/* * Subsystem for storing human-readable names for each object. *
object-file.c +8 -6
@@ -2472,12 +2472,14 @@ int repo_has_object_file(struct repository *r,* report the minimal fsck error here, and rely on the caller to * give more context. */ -static int hash_format_check_report(struct fsck_objects_options *opts UNUSED, - const struct object_id *oid UNUSED, - enum object_type object_type UNUSED, - enum fsck_msg_type msg_type UNUSED, - enum fsck_msg_id msg_id UNUSED, -const char *message) +static int hash_format_check_report(struct fsck_objects_options *objects_options UNUSED, + struct fsck_refs_options *refs_options UNUSED, + const struct object_id *oid UNUSED, + enum object_type object_type UNUSED, + const char *chekced_ref_name UNUSED, + enum fsck_msg_type msg_type UNUSED, + enum fsck_msg_id msg_id UNUSED, +const char *message) {error (_ ("object fails fsck: %s" ), message);return 1 ;
Signature · 1 line
2.45.2
PATCH v5 6/12 fsck: add "fsck_refs_options" initialization macros
In reply to shejialuoAdd "FSCK_REFS_OPTIONS_DEFAULT" and "FSCK_REFS_OPTIONS_STRICT" macros to create the "fsck_refs_options" easily. Add refs-specific "error_func" callback "fsck_refs_error_function".
"fsck_refs_error_function" will use the "oid" parameter. When the caller passed the oid, it will use "oid_to_hex" to get the corresponding hex value to report to the user.
--- fsck.c | 23 +++++++++++++++++++++++ fsck.h | 19 +++++++++++++++++++ 2 files changed, 42 insertions(+)
fsck.c +23 -0
@@ -1285,6 +1285,29 @@ int fsck_error_function(struct fsck_objects_options *objects_options,return 1 ; } +int fsck_refs_error_function(struct fsck_objects_options *objects_options UNUSED, +struct fsck_refs_options *refs_options, +const struct object_id *oid, + enum object_type object_type UNUSED, +const char *checked_ref_name, +enum fsck_msg_type msg_type, + enum fsck_msg_id msg_id UNUSED, +const char *message) +{ +static struct strbuf sb =STRBUF_INIT ; + +strbuf_addstr (&sb, checked_ref_name); +if (oid) +strbuf_addf (&sb," -> (%s)" ,oid_to_hex (oid)); + +if (msg_type ==FSCK_WARN ) { +warning ("%s: %s" , sb.buf, message); +return 0 ; + } +error ("%s: %s" , sb.buf, message); +return 1 ; +} + static int fsck_blobs(struct oidset *blobs_found, struct oidset *blobs_done, enum fsck_msg_id msg_missing, enum fsck_msg_id msg_type, struct fsck_objects_options *options,
fsck.h +19 -0
@@ -140,6 +140,14 @@ int fsck_error_cb_print_missing_gitmodules(struct fsck_objects_options *objects_enum fsck_msg_type msg_type, enum fsck_msg_id msg_id, const char *message); +int fsck_refs_error_function (struct fsck_objects_options *objects_options, +struct fsck_refs_options *refs_options, +const struct object_id *oid, +enum object_type object_type, +const char *checked_ref_name, +enum fsck_msg_type msg_type, +enum fsck_msg_id msg_id, +const char *message);struct fsck_options {fsck_error error_func;
@@ -150,6 +158,17 @@ struct fsck_options {struct fsck_refs_options {struct fsck_options fsck_options; }; +#define FSCK_REFS_OPTIONS_DEFAULT { \ + .fsck_options = { \ + .error_func = fsck_refs_error_function, \ + }, \ +} +#define FSCK_REFS_OPTIONS_STRICT { \ + .fsck_options = { \ + .error_func = fsck_refs_error_function, \ + .strict = 1, \ + }, \ +}struct fsck_objects_options {struct fsck_options fsck_options;
Signature · 1 line
2.45.2
PATCH v5 7/12 refs: set up ref consistency check infrastructure
In reply to shejialuoThe interfaces defined in the ref_storage_be are carefully structured in semantic. It's organized as the five parts:
1. The name and the initialization interfaces.
2. The ref transaction interfaces.
3. The ref internal interfaces (pack, rename and copy).
4. The ref filesystem interfaces.
5. The reflog related interfaces.
To keep consistent with the git-fsck(1), add a new interface named "fsck_refs_fn" to the end of "ref_storage_be". This semantic cannot be grouped into any above five categories. Explicitly add blank line to make it different from others.
--- refs.c | 5 +++++ refs.h | 8 ++++++++ refs/debug.c | 11 +++++++++++ refs/files-backend.c | 15 ++++++++++++++- refs/packed-backend.c | 8 ++++++++ refs/refs-internal.h | 6 ++++++ refs/reftable-backend.c | 8 ++++++++ 7 files changed, 60 insertions(+), 1 deletion(-)
refs.c +5 -0
@@ -316,6 +316,11 @@ int check_refname_format(const char *refname, int flags)return check_or_sanitize_refname (refname, flags,NULL ); } +int refs_fsck (struct ref_store *refs,struct fsck_refs_options *o) +{ +return refs->be->fsck (refs, o); +} +void sanitize_refname_component (const char *refname,struct strbuf *out) {if (check_or_sanitize_refname (refname,REFNAME_ALLOW_ONELEVEL , out))
refs.h +8 -0
@@ -3,6 +3,7 @@#include "commit.h" +struct fsck_refs_options ;struct object_id ;struct ref_store ;struct repository ;
@@ -547,6 +548,13 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat*/ int check_refname_format (const char *refname,int flags); +/* +* Check the reference database for consistency. Return 0 if refs and +* reflogs are consistent, and non-zero otherwise. The errors will be +* written to stderr. +*/ +int refs_fsck (struct ref_store *refs,struct fsck_refs_options *o); +/* * Apply the rules from check_refname_format, but mutate the result until it * is acceptable, and place the result in "out".
refs/debug.c +11 -0
@@ -419,6 +419,15 @@ static int debug_reflog_expire(struct ref_store *ref_store, const char *refname,return res; } +static int debug_fsck (struct ref_store *ref_store, +struct fsck_refs_options *o) +{ +struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store; +int res = drefs->refs->be->fsck (drefs->refs, o); +trace_printf_key (&trace_refs,"fsck: %d\n" , res); +return res; +} +struct ref_storage_be refs_be_debug = { .name ="debug" , .init =NULL ,
@@ -451,4 +460,6 @@ struct ref_storage_be refs_be_debug = { .create_reflog = debug_create_reflog, .delete_reflog = debug_delete_reflog, .reflog_expire = debug_reflog_expire, + + .fsck = debug_fsck, };
refs/files-backend.c +14 -1
@@ -3406,6 +3406,17 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,return ret; } +static int files_fsck (struct ref_store *ref_store, +struct fsck_refs_options *o) +{ +int ret; +struct files_ref_store *refs = +files_downcast (ref_store,REF_STORE_READ ,"fsck" ); + + ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o); +return ret; +} +struct ref_storage_be refs_be_files = { .name ="files" , .init = files_ref_store_init,
@@ -3432,5 +3443,7 @@ struct ref_storage_be refs_be_files = { .reflog_exists = files_reflog_exists, .create_reflog = files_create_reflog, .delete_reflog = files_delete_reflog, - .reflog_expire = files_reflog_expire + .reflog_expire = files_reflog_expire, + + .fsck = files_fsck, };
refs/packed-backend.c +8 -0
@@ -1733,6 +1733,12 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_sreturn empty_ref_iterator_begin (); } +static int packed_fsck (struct ref_store *ref_store, +struct fsck_refs_options *o) +{ +return 0 ; +} +struct ref_storage_be refs_be_packed = { .name ="packed" , .init = packed_ref_store_init,
@@ -1760,4 +1766,6 @@ struct ref_storage_be refs_be_packed = { .create_reflog = NULL, .delete_reflog = NULL, .reflog_expire = NULL, + + .fsck = packed_fsck, };
refs/refs-internal.h +6 -0
@@ -4,6 +4,7 @@#include "refs.h" #include "iterator.h" +struct fsck_options ;struct ref_transaction ;/*
@@ -650,6 +651,9 @@ typedef int read_raw_ref_fn(struct ref_store *ref_store, const char *refname,typedef int read_symbolic_ref_fn (struct ref_store *ref_store,const char *refname,struct strbuf *referent); +typedef int fsck_fn (struct ref_store *ref_store, +struct fsck_refs_options *o); +struct ref_storage_be {const char *name;ref_store_init_fn *init;
@@ -677,6 +681,8 @@ struct ref_storage_be {create_reflog_fn *create_reflog;delete_reflog_fn *delete_reflog;reflog_expire_fn *reflog_expire; + +fsck_fn *fsck; };extern struct ref_storage_be refs_be_files;
refs/reftable-backend.c +8 -0
@@ -2292,6 +2292,12 @@ static int reftable_be_reflog_expire(struct ref_store *ref_store,return ret; } +static int reftable_be_fsck (struct ref_store *ref_store, +struct fsck_refs_options *o) +{ +return 0 ; +} +struct ref_storage_be refs_be_reftable = { .name ="reftable" , .init = reftable_be_init,
@@ -2319,4 +2325,6 @@ struct ref_storage_be refs_be_reftable = { .create_reflog = reftable_be_create_reflog, .delete_reflog = reftable_be_delete_reflog, .reflog_expire = reftable_be_reflog_expire, + + .fsck = reftable_be_fsck, };
Signature · 1 line
2.45.2
PATCH v5 8/12 builtin/refs: add verify subcommand
In reply to shejialuoIntroduce a new subcommand "verify" in git-refs(1) to allow the user to check the reference database consistency.
--- Documentation/git-refs.txt | 13 +++++++++++++ builtin/refs.c | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+)
Documentation/git-refs.txt +13 -0
@@ -10,6 +10,7 @@ SYNOPSIS -------- [verse] 'git refs migrate' --ref-format=<format> [--dry-run] +'git refs verify' [--strict] [--verbose] DESCRIPTION -----------
@@ -22,6 +23,9 @@ COMMANDS migrate:: Migrate ref store between different formats. +verify:: + Verify reference database consistency. + OPTIONS -------
@@ -39,6 +43,15 @@ include::ref-storage-format.txt[] can be used to double check that the migration works as expected before performing the actual migration. +The following options are specific to 'git refs verify': + +--strict:: + Enable more strict checking, every WARN severity for the `Fsck Messages` + be seen as ERROR. See linkgit:git-fsck[1]. + +--verbose:: + When verifying the reference database consistency, be chatty. + KNOWN LIMITATIONS -----------------
builtin/refs.c +36 -0
@@ -1,4 +1,6 @@#include "builtin.h" +#include "config.h" +#include "fsck.h" #include "parse-options.h" #include "refs.h" #include "repository.h"
@@ -7,6 +9,9 @@#define REFS_MIGRATE_USAGE \ N_("git refs migrate --ref-format=<format> [--dry-run]") +#define REFS_VERIFY_USAGE \ + N_("git refs verify [--strict] [--verbose]") +static int cmd_refs_migrate (int argc,const char **argv,const char *prefix) {const char *const migrate_usage[] = {
@@ -58,15 +63,46 @@ static int cmd_refs_migrate(int argc, const char **argv, const char *prefix)return err; } +static int cmd_refs_verify (int argc,const char **argv,const char *prefix) +{ +struct fsck_refs_options fsck_refs_options =FSCK_REFS_OPTIONS_DEFAULT ; +const char *const verify_usage[] = { +REFS_VERIFY_USAGE , +NULL , + }; +unsigned int verbose =0 , strict =0 ; +struct option options[] = { +OPT__VERBOSE (&verbose,N_ ("be verbose" )), +OPT_BOOL (0 ,"strict" , &strict,N_ ("enable strict checking" )), +OPT_END (), + }; + + argc =parse_options (argc, argv, prefix, options, verify_usage,0 ); +if (argc) +usage (_ ("too many arguments" )); + +if (verbose) + fsck_refs_options.fsck_options.verbose =1 ; +if (strict) + fsck_refs_options.fsck_options.strict =1 ; + +git_config (git_fsck_config, &fsck_refs_options.fsck_options); +prepare_repo_settings (the_repository); + +return refs_fsck (get_main_ref_store (the_repository), &fsck_refs_options); +} +int cmd_refs (int argc,const char **argv,const char *prefix) {const char *const refs_usage[] = {REFS_MIGRATE_USAGE , +REFS_VERIFY_USAGE ,NULL , };parse_opt_subcommand_fn *fn =NULL ;struct option opts[] = {OPT_SUBCOMMAND ("migrate" , &fn, cmd_refs_migrate), +OPT_SUBCOMMAND ("verify" , &fn, cmd_refs_verify),OPT_END (), };
Signature · 1 line
2.45.2
PATCH v5 9/12 builtin/fsck: add `git-refs verify` child process
In reply to shejialuoIntroduce a new function "fsck_refs" that initializes and runs a child process to execute the "git-refs verify" command.
--- builtin/fsck.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+)
builtin/fsck.c +17 -0
@@ -899,6 +899,21 @@ static int check_pack_rev_indexes(struct repository *r, int show_progress)return res; } +static void fsck_refs (void ) +{ +struct child_process refs_verify =CHILD_PROCESS_INIT ; +child_process_init (&refs_verify); + refs_verify.git_cmd =1 ; +strvec_pushl (&refs_verify.args,"refs" ,"verify" ,NULL ); +if (verbose) +strvec_push (&refs_verify.args,"--verbose" ); +if (check_strict) +strvec_push (&refs_verify.args,"--strict" ); + +if (run_command (&refs_verify)) + errors_found |=ERROR_REFS ; +} +static char const *const fsck_usage[] = { N_("git fsck [--tags] [--root] [--unreachable] [--cache] [--no-reflogs]\n" " [--[no-]full] [--strict] [--verbose] [--lost-found]\n"
@@ -1068,6 +1083,8 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)check_connectivity (); +fsck_refs (); +if (the_repository->settings.core_commit_graph) {struct child_process commit_graph_verify =CHILD_PROCESS_INIT ;
Signature · 1 line
2.45.2
PATCH v5 10/12 files-backend: add unified interface for refs scanning
In reply to shejialuoFor refs and reflogs, we need to scan its corresponding directories to check every regular file or symbolic link which shares the same pattern. Introduce a unified interface for scanning directories for files-backend.
--- refs/files-backend.c | 77 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-)
refs/files-backend.c +76 -1
@@ -4,6 +4,7 @@#include "../gettext.h" #include "../hash.h" #include "../hex.h" +#include "../fsck.h" #include "../refs.h" #include "refs-internal.h" #include "ref-cache.h"
@@ -3406,6 +3407,78 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,return ret; } +/* +* For refs and reflogs, they share a unified interface when scanning +* the whole directory. This function is used as the callback for each +* regular file or symlink in the directory. +*/ +typedef int (*files_fsck_refs_fn )(struct fsck_refs_options *o, +const char *gitdir, +const char *refs_check_dir, +struct dir_iterator *iter); + +static int files_fsck_refs_dir (struct ref_store *ref_store, +struct fsck_refs_options *o, +const char *refs_check_dir, +files_fsck_refs_fn *fsck_refs_fns) +{ +const char *gitdir = ref_store->gitdir; +struct strbuf sb =STRBUF_INIT ; +struct dir_iterator *iter; +int iter_status; +int ret =0 ; + +strbuf_addf (&sb,"%s/%s" , gitdir, refs_check_dir); + + iter =dir_iterator_begin (sb.buf,0 ); + +if (!iter) { + ret =error_errno ("cannot open directory %s" , sb.buf); + goto out; + } + +while ((iter_status =dir_iterator_advance (iter)) ==ITER_OK ) { +if (S_ISDIR (iter->st.st_mode)) { +continue ; + }else if (S_ISREG (iter->st.st_mode) || +S_ISLNK (iter->st.st_mode)) { +if (o->fsck_options.verbose) +fprintf_ln (stderr,"Checking %s/%s" , + refs_check_dir, iter->relative_path); +for (size_t i =0 ; fsck_refs_fns[i]; i++) { +if (fsck_refs_fns[i](o, gitdir, refs_check_dir, iter)) + ret =-1 ; + } + }else { + ret =error (_ ("unexpected file type for '%s'" ), + iter->basename); + } + } + +if (iter_status !=ITER_DONE ) + ret =error (_ ("failed to iterate over '%s'" ), sb.buf); + +out: +strbuf_release (&sb); +return ret; +} + +static int files_fsck_refs (struct ref_store *ref_store, +struct fsck_refs_options *o) +{ +int ret; +files_fsck_refs_fn fsck_refs_fns[]= { +NULL + }; + +if (o->fsck_options.verbose) +fprintf_ln (stderr,"Checking references consistency" ); + + ret =files_fsck_refs_dir (ref_store, o,"refs" , fsck_refs_fns); + +return ret; +} +static int files_fsck (struct ref_store *ref_store,struct fsck_refs_options *o) {
@@ -3413,7 +3486,9 @@ static int files_fsck(struct ref_store *ref_store,struct files_ref_store *refs =files_downcast (ref_store,REF_STORE_READ ,"fsck" ); - ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o); + ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o) + |files_fsck_refs (ref_store, o); +return ret; }
Signature · 1 line
2.45.2
PATCH v5 11/12 fsck: add ref name check for files backend
In reply to shejialuoThe git-fsck(1) only implicitly checks the reference, it does not fully check refs with bad format name such as standalone "@" and name ending with ".lock".
In order to provide such checks, add a new fsck message id "badRefName" with default ERROR type. Use existing "check_refname_format" to explicit check the ref name. And add a new unit test to verify the functionality.
--- Documentation/fsck-msgids.txt | 3 + fsck.h | 1 + refs/files-backend.c | 20 +++++++ t/t0602-reffiles-fsck.sh | 101 ++++++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+) create mode 100755 t/t0602-reffiles-fsck.sh
Documentation/fsck-msgids.txt +3 -0
@@ -19,6 +19,9 @@ `badParentSha1`:: (ERROR) A commit object has a bad parent sha1. +`badRefName`:: + (ERROR) A ref has a bad name. + `badTagName`:: (INFO) A tag has an invalid format.
fsck.h +1 -0
@@ -31,6 +31,7 @@ enum fsck_msg_type { FUNC(BAD_NAME, ERROR) \ FUNC(BAD_OBJECT_SHA1, ERROR) \ FUNC(BAD_PARENT_SHA1, ERROR) \ + FUNC(BAD_REF_NAME, ERROR) \ FUNC(BAD_TIMEZONE, ERROR) \ FUNC(BAD_TREE, ERROR) \FUNC (BAD_TREE_SHA1 ,ERROR ) \
refs/files-backend.c +20 -0
@@ -3417,6 +3417,25 @@ typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o,const char *refs_check_dir, struct dir_iterator *iter); +static int files_fsck_refs_name (struct fsck_refs_options *o, + const char *gitdir UNUSED, +const char *refs_check_dir, +struct dir_iterator *iter) +{ +struct strbuf sb =STRBUF_INIT ; +int ret =0 ; + +if (check_refname_format (iter->basename,REFNAME_ALLOW_ONELEVEL )) { +strbuf_addf (&sb,"%s/%s" , refs_check_dir, iter->relative_path); + ret =fsck_refs_report (o,NULL , sb.buf, +FSCK_MSG_BAD_REF_NAME , +"invalid refname format" ); + } + +strbuf_release (&sb); +return ret; +} + static int files_fsck_refs_dir(struct ref_store *ref_store, struct fsck_refs_options *o, const char *refs_check_dir,
@@ -3468,6 +3487,7 @@ static int files_fsck_refs(struct ref_store *ref_store, {int ret;files_fsck_refs_fn fsck_refs_fns[]= { + files_fsck_refs_name,NULL };
t/t0602-reffiles-fsck.sh new file +101 -0
101 lines not on this page · raw message
Signature · 1 line
2.45.2
PATCH v5 12/12 fsck: add ref content check for files backend
In reply to shejialuoEnhance the git-fsck(1) command by adding a check for reference content in the files backend. The new functionality ensures that symrefs, real symbolic link and regular refs are validated correctly.
In order to check the trailing content of the regular refs, add a new parameter trailing to parse_loose_ref_contents.
For symrefs, parse_loose_ref_contents will set the "referent". However, symbolic link could be either absolute or relative. Use "strbuf_add_real_path" to read the symbolic link and convert the relative path to absolute path. Then use "skip_prefix" to make it align with symref "referent".
Thus, the symrefs and symbolic links could share the same interface. Add a new function "files_fsck_symref_target" which aims at checking the following things:
1. whether the pointee is under the refs/ directory.
2. whether the pointee name is correct.
3. whether the pointee path is a wrong type in filesystem.
Last, add the following FSCK MESSAGEs:
1. "badRefContent(ERROR)": A ref has a bad content
2. "badSymrefPointee(ERROR)": The pointee of a symref is bad.
3. "trailingRefContent(WARN)": A ref content has trailing contents.
--- Documentation/fsck-msgids.txt | 9 +++ fsck.h | 3 + refs.c | 2 +- refs/files-backend.c | 145 +++++++++++++++++++++++++++++++++- refs/refs-internal.h | 5 +- t/t0602-reffiles-fsck.sh | 110 ++++++++++++++++++++++++++ 6 files changed, 269 insertions(+), 5 deletions(-)
Documentation/fsck-msgids.txt +9 -0
@@ -19,9 +19,15 @@ `badParentSha1`:: (ERROR) A commit object has a bad parent sha1. +`badRefContent`:: + (ERROR) A ref has a bad content. + `badRefName`:: (ERROR) A ref has a bad name. +`badSymrefPointee`:: + (ERROR) The pointee of a symref is bad. + `badTagName`:: (INFO) A tag has an invalid format.
@@ -167,6 +173,9 @@ `nullSha1`:: (WARN) Tree contains entries pointing to a null sha1. +`trailingRefContent`:: + (WARN) A ref content has trailing contents. + `treeNotSorted`:: (ERROR) A tree is not properly sorted.
fsck.h +3 -0
@@ -32,6 +32,8 @@ enum fsck_msg_type { FUNC(BAD_OBJECT_SHA1, ERROR) \ FUNC(BAD_PARENT_SHA1, ERROR) \ FUNC(BAD_REF_NAME, ERROR) \ + FUNC(BAD_REF_CONTENT, ERROR) \ + FUNC(BAD_SYMREF_POINTEE, ERROR) \ FUNC(BAD_TIMEZONE, ERROR) \ FUNC(BAD_TREE, ERROR) \FUNC (BAD_TREE_SHA1 ,ERROR ) \
@@ -72,6 +74,7 @@ enum fsck_msg_type { FUNC(HAS_DOTDOT, WARN) \ FUNC(HAS_DOTGIT, WARN) \ FUNC(NULL_SHA1, WARN) \ + FUNC(TRAILING_REF_CONTENT, WARN) \ FUNC(ZERO_PADDED_FILEMODE, WARN) \ FUNC(NUL_IN_COMMIT, WARN) \FUNC (LARGE_PATHNAME ,WARN ) \
refs.c +1 -1
@@ -1758,7 +1758,7 @@ static int refs_read_special_head(struct ref_store *ref_store, } result =parse_loose_ref_contents (content.buf, oid, referent, type, - failure_errno); + failure_errno,NULL ); done:strbuf_release (&full_path);
refs/files-backend.c +143 -2
179 lines not on this page · raw message
refs/refs-internal.h +3 -2
@@ -709,11 +709,12 @@ struct ref_store {/* * Parse contents of a loose ref file. *failure_errno maybe be set to EINVAL for -* invalid contents. +* invalid contents. Also *trailing is set to the first character after the +* refname or NULL if the referent is not empty. */ int parse_loose_ref_contents (const char *buf,struct object_id *oid,struct strbuf *referent,unsigned int *type, -int *failure_errno); +int *failure_errno,const char **trailing);/* * Fill in the generic part of refs and add it to our collection of
t/t0602-reffiles-fsck.sh +110 -0
114 lines not on this page · raw message
Signature · 1 line
2.45.2
v6 cover ref consistency check infra setup
In reply to shejialuoHi All:
This version follows the Junio's advice. Instead of creating the following data structure:
struct fsck_options {
enum fsck_type {
FSCK_OBJECTS,
FSCK_REFS,
...
} t;
union {
struct fsck_objects_options objects;
struct fsck_refs_options refs;
} u;
};I simply use the combination idea where "fsck_options" will incorporate "fsck_objects_options" and "fsck_refs_options". Karthik has told me that I should balance the job I should does and the extensibility for future.
Quoted 3 lines · So I use the most clear way to do this. Also Junio has
I would have expected that adding ref-related new members that object consistency checkers has no interest in to the
fsck_optionsstructure would be sufficient for the purpose of this series.
So this patch makes the design simple here.
Thanks for every reviewer.
Jialuo.
shejialuo (11):
fsck: add "fsck_objects_options" to hold objects-related options
fsck: rename "skiplist" to "oid_skiplist"
fsck: add "fsck_refs_options" into "fsck_options"
fsck: add a unified interface for reporting fsck messages
fsck: add "fsck_refs_options" initialization macros
refs: set up ref consistency check infrastructure builtin/refs: add verify subcommand
builtin/fsck: add git-refs verify child process
files-backend: add unified interface for refs scanning
fsck: add ref name check for files backend
fsck: add ref content check for files backend
Documentation/fsck-msgids.txt | 12 ++ Documentation/git-refs.txt | 13 ++ builtin/fsck.c | 44 ++++-- builtin/index-pack.c | 4 +- builtin/mktag.c | 3 +- builtin/refs.c | 44 ++++++ builtin/unpack-objects.c | 4 +- fetch-pack.c | 10 +- fsck.c | 158 +++++++++++++++------ fsck.h | 115 ++++++++++----- object-file.c | 13 +- refs.c | 7 +- refs.h | 8 ++ refs/debug.c | 11 ++ refs/files-backend.c | 255 +++++++++++++++++++++++++++++++++- refs/packed-backend.c | 8 ++ refs/refs-internal.h | 11 +- refs/reftable-backend.c | 8 ++ t/t0602-reffiles-fsck.sh | 211 ++++++++++++++++++++++++++++ 19 files changed, 824 insertions(+), 115 deletions(-) create mode 100755 t/t0602-reffiles-fsck.sh
Range-diff against v5:
1: d83b5797cb < -: ---------- fsck: rename "fsck_options" to "fsck_objects_options" 2: 88000b092a < -: ---------- fsck: use "fsck_configs" to set up configs 3: 65093bae64 < -: ---------- fsck: abstract common options for reusing 4: 403750da30 < -: ---------- fsck: add "fsck_refs_options" struct 5: 8dea5654d5 < -: ---------- fsck: add a unified interface for reporting fsck messages -: ---------- > 1: d32ae41a4d fsck: add "fsck_objects_options" to hold objects-related options -: ---------- > 2: 589a7a6fac fsck: rename "skiplist" to "oid_skiplist" -: ---------- > 3: bab97d7c82 fsck: add "fsck_refs_options" into "fsck_options" -: ---------- > 4: 276da52a6b fsck: add a unified interface for reporting fsck messages 6: 8fb62b54c6 ! 5: e93940c50c fsck: add "fsck_refs_options" initialization macros
@@ Commit message
fsck: add "fsck_refs_options" initialization macros
Add "FSCK_REFS_OPTIONS_DEFAULT" and "FSCK_REFS_OPTIONS_STRICT" macros to
- create the "fsck_refs_options" easily. Add refs-specific "error_func"
- callback "fsck_refs_error_function".
+ create the refs options easily. Add refs-specific "error_func" callback
+ "fsck_refs_error_function".
"fsck_refs_error_function" will use the "oid" parameter. When the caller passed the oid, it will use "oid_to_hex" to get the corresponding hex
@@ Commit message
## fsck.c ##
-@@ fsck.c: int fsck_error_function(struct fsck_objects_options *objects_options, +@@ fsck.c: int fsck_objects_error_function(struct fsck_options *o,
return 1;
}
-+int fsck_refs_error_function(struct fsck_objects_options *objects_options UNUSED,
-+ struct fsck_refs_options *refs_options,
++int fsck_refs_error_function(struct fsck_options *options UNUSED,
+ const struct object_id *oid,
+ enum object_type object_type UNUSED,
+ const char *checked_ref_name,
@@ fsck.c: int fsck_error_function(struct fsck_objects_options *objects_options,
+
static int fsck_blobs(struct oidset *blobs_found, struct oidset *blobs_done,
enum fsck_msg_id msg_missing, enum fsck_msg_id msg_type,
- struct fsck_objects_options *options,
+ struct fsck_options *options, const char *blob_type)
## fsck.h ##
-@@ fsck.h: int fsck_error_cb_print_missing_gitmodules(struct fsck_objects_options *objects_
- enum fsck_msg_type msg_type,
- enum fsck_msg_id msg_id,
- const char *message);
-+int fsck_refs_error_function(struct fsck_objects_options *objects_options,
-+ struct fsck_refs_options *refs_options,
+@@ fsck.h: int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o,
+ enum fsck_msg_type msg_type,
+ enum fsck_msg_id msg_id,
+ const char *message);
++int fsck_refs_error_function(struct fsck_options *options,
+ const struct object_id *oid,
+ enum object_type object_type,
+ const char *checked_ref_name,
@@ fsck.h: int fsck_error_cb_print_missing_gitmodules(struct fsck_objects_options *
+ enum fsck_msg_id msg_id,
+ const char *message);
- struct fsck_options {
- fsck_error error_func;
-@@ fsck.h: struct fsck_options {
struct fsck_refs_options {
- struct fsck_options fsck_options;
- };
+ unsigned verbose:1;
+@@ fsck.h: struct fsck_options {
+ .gitattributes_done = OIDSET_INIT, \
+ } \
+ }
+#define FSCK_REFS_OPTIONS_DEFAULT { \
-+ .fsck_options = { \
-+ .error_func = fsck_refs_error_function, \
-+ }, \
++ .error_func = fsck_refs_error_function, \
+}
+#define FSCK_REFS_OPTIONS_STRICT { \
-+ .fsck_options = { \
-+ .error_func = fsck_refs_error_function, \
-+ .strict = 1, \
-+ }, \
++ .strict = 1, \
++ .error_func = fsck_refs_error_function, \
+}
- struct fsck_objects_options {
- struct fsck_options fsck_options;
+ /* descend in all linked child objects
+ * the return value is:
7: 2ecd1f5407 ! 6: 85aa953f6d refs: set up ref consistency check infrastructure
@@ refs.c: int check_refname_format(const char *refname, int flags)
return check_or_sanitize_refname(refname, flags, NULL);
}
-+int refs_fsck(struct ref_store *refs, struct fsck_refs_options *o)
++int refs_fsck(struct ref_store *refs, struct fsck_options *o)
+{
+ return refs->be->fsck(refs, o);
+}
@@ refs.h
#include "commit.h"
-+struct fsck_refs_options;
++struct fsck_options;
struct object_id;
struct ref_store;
struct repository;
@@ refs.h: int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void
+ * reflogs are consistent, and non-zero otherwise. The errors will be
+ * written to stderr.
+ */
-+int refs_fsck(struct ref_store *refs, struct fsck_refs_options *o);
++int refs_fsck(struct ref_store *refs, struct fsck_options *o);
+
/*
* Apply the rules from check_refname_format, but mutate the result until it
@@ refs/debug.c: static int debug_reflog_expire(struct ref_store *ref_store, const
}
+static int debug_fsck(struct ref_store *ref_store,
-+ struct fsck_refs_options *o)
++ struct fsck_options *o)
+{
+ struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store;
+ int res = drefs->refs->be->fsck(drefs->refs, o);
@@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
}
+static int files_fsck(struct ref_store *ref_store,
-+ struct fsck_refs_options *o)
++ struct fsck_options *o)
+{
+ int ret;
+ struct files_ref_store *refs =
@@ refs/packed-backend.c: static struct ref_iterator *packed_reflog_iterator_begin(
}
+static int packed_fsck(struct ref_store *ref_store,
-+ struct fsck_refs_options *o)
++ struct fsck_options *o)
+{
+ return 0;
+}
@@ refs/refs-internal.h: typedef int read_raw_ref_fn(struct ref_store *ref_store, c
struct strbuf *referent);
+typedef int fsck_fn(struct ref_store *ref_store,
-+ struct fsck_refs_options *o);
++ struct fsck_options *o);
+
struct ref_storage_be {
const char *name;
@@ refs/reftable-backend.c: static int reftable_be_reflog_expire(struct ref_store *
}
+static int reftable_be_fsck(struct ref_store *ref_store,
-+ struct fsck_refs_options *o)
++ struct fsck_options *o)
+{
+ return 0;
+}
8: 534a3d00af ! 7: ce7adc7372 builtin/refs: add verify subcommand
@@ builtin/refs.c: static int cmd_refs_migrate(int argc, const char **argv, const c
+static int cmd_refs_verify(int argc, const char **argv, const char *prefix)
+{
-+ struct fsck_refs_options fsck_refs_options = FSCK_REFS_OPTIONS_DEFAULT;
++ struct fsck_options fsck_refs_options = FSCK_REFS_OPTIONS_DEFAULT;
+ const char * const verify_usage[] = {
+ REFS_VERIFY_USAGE,
+ NULL,
@@ builtin/refs.c: static int cmd_refs_migrate(int argc, const char **argv, const c
+ OPT_BOOL(0, "strict", &strict, N_("enable strict checking")),
+ OPT_END(),
+ };
++ int ret = 0;
+
+ argc = parse_options(argc, argv, prefix, options, verify_usage, 0);
+ if (argc)
+ usage(_("too many arguments"));
+
+ if (verbose)
-+ fsck_refs_options.fsck_options.verbose = 1;
++ fsck_refs_options.refs_options.verbose = 1;
+ if (strict)
-+ fsck_refs_options.fsck_options.strict = 1;
++ fsck_refs_options.strict = 1;
+
-+ git_config(git_fsck_config, &fsck_refs_options.fsck_options);
++ git_config(git_fsck_config, &fsck_refs_options);
+ prepare_repo_settings(the_repository);
+
-+ return refs_fsck(get_main_ref_store(the_repository), &fsck_refs_options);
++ ret = refs_fsck(get_main_ref_store(the_repository), &fsck_refs_options);
++
++ /*
++ * Explicitly free the allocated array and "oid_skiplist"
++ */
++ free(fsck_refs_options.msg_type);
++ oidset_clear(&fsck_refs_options.oid_skiplist);
++ return ret;
+}
+
int cmd_refs(int argc, const char **argv, const char *prefix)
9: 054c8cea89 = 8: 035eafe10b builtin/fsck: add `git-refs verify` child process
10: 7f87c6a26e ! 9: 9398bf3f0d files-backend: add unified interface for refs scanning
@@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
+ * the whole directory. This function is used as the callback for each
+ * regular file or symlink in the directory.
+ */
-+typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o,
++typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
+ const char *gitdir,
+ const char *refs_check_dir,
+ struct dir_iterator *iter);
+
+static int files_fsck_refs_dir(struct ref_store *ref_store,
-+ struct fsck_refs_options *o,
++ struct fsck_options *o,
+ const char *refs_check_dir,
+ files_fsck_refs_fn *fsck_refs_fns)
+{
@@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
+ continue;
+ } else if (S_ISREG(iter->st.st_mode) ||
+ S_ISLNK(iter->st.st_mode)) {
-+ if (o->fsck_options.verbose)
++ if (o->refs_options.verbose)
+ fprintf_ln(stderr, "Checking %s/%s",
+ refs_check_dir, iter->relative_path);
+ for (size_t i = 0; fsck_refs_fns[i]; i++) {
@@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
+}
+
+static int files_fsck_refs(struct ref_store *ref_store,
-+ struct fsck_refs_options *o)
++ struct fsck_options *o)
+{
+ int ret;
+ files_fsck_refs_fn fsck_refs_fns[]= {
+ NULL
+ };
+
-+ if (o->fsck_options.verbose)
++ if (o->refs_options.verbose)
+ fprintf_ln(stderr, "Checking references consistency");
+
+ ret = files_fsck_refs_dir(ref_store, o, "refs", fsck_refs_fns);
@@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
+}
+
static int files_fsck(struct ref_store *ref_store,
- struct fsck_refs_options *o)
+ struct fsck_options *o)
{
@@ refs/files-backend.c: static int files_fsck(struct ref_store *ref_store,
struct files_ref_store *refs =11: 7d78014e5f ! 10: 046773e35c fsck: add ref name check for files backend
@@ fsck.h: enum fsck_msg_type {
FUNC(BAD_TREE_SHA1, ERROR) \
## refs/files-backend.c ##
-@@ refs/files-backend.c: typedef int (*files_fsck_refs_fn)(struct fsck_refs_options *o, +@@ refs/files-backend.c: typedef int (*files_fsck_refs_fn)(struct fsck_options *o,
const char *refs_check_dir,
struct dir_iterator *iter);
-+static int files_fsck_refs_name(struct fsck_refs_options *o, ++static int files_fsck_refs_name(struct fsck_options *o,
+ const char *gitdir UNUSED,
+ const char *refs_check_dir,
+ struct dir_iterator *iter)
@@ refs/files-backend.c: typedef int (*files_fsck_refs_fn)(struct fsck_refs_options
+}
+
static int files_fsck_refs_dir(struct ref_store *ref_store,
- struct fsck_refs_options *o,
+ struct fsck_options *o,
const char *refs_check_dir,
@@ refs/files-backend.c: static int files_fsck_refs(struct ref_store *ref_store,
{
12: 16209a73ed ! 11: c9b9599589 fsck: add ref content check for files backend
@@ refs/files-backend.c: int parse_loose_ref_contents(const char *buf, struct objec
return 0;
}
-@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *o, +@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_options *o,
return ret;
}
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
+ * would be the content after "refs:". For symblic link, "pointee_name" would
+ * be the relative path agaignst "gitdir".
+ */
-+static int files_fsck_symref_target(struct fsck_refs_options *o, ++static int files_fsck_symref_target(struct fsck_options *o,
+ const char *refname,
+ const char *pointee_name,
+ const char *pointee_path)
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
+ return ret;
+}
+
-+static int files_fsck_refs_content(struct fsck_refs_options *o, ++static int files_fsck_refs_content(struct fsck_options *o,
+ const char *gitdir,
+ const char *refs_check_dir,
+ struct dir_iterator *iter)
@@ refs/files-backend.c: static int files_fsck_refs_name(struct fsck_refs_options *
+}
+
static int files_fsck_refs_dir(struct ref_store *ref_store,
- struct fsck_refs_options *o,
+ struct fsck_options *o,
const char *refs_check_dir,
@@ refs/files-backend.c: static int files_fsck_refs(struct ref_store *ref_store,
int ret;
Signature · 1 line
2.45.2
PATCH v6 1/11 fsck: add "fsck_objects_options" to hold objects-related options
The git-fsck(1) focuses on object database consistency check. It relies on the "fsck_options" to interact with fsck error levels. However, "fsck_options" aims at checking the object database which contains a lot of fields only related to object database.
In order to add ref operations, remove the options related to objects in "fsck_options" to "fsck_objects_options" and let the "fsck_options" incorporate "fsck_objects_options". Change the macros and influenced code to adapt into the above design.
--- builtin/fsck.c | 12 +++++----- builtin/index-pack.c | 4 ++-- builtin/mktag.c | 2 +- builtin/unpack-objects.c | 4 ++-- fetch-pack.c | 10 ++++---- fsck.c | 51 +++++++++++++++++++++------------------ fsck.h | 52 ++++++++++++++++++++++++---------------- object-file.c | 2 +- 8 files changed, 77 insertions(+), 60 deletions(-)
builtin/fsck.c +6 -6
@@ -42,8 +42,8 @@ static int check_full = 1;static int connectivity_only;static int check_strict;static int keep_cache_objects; -static struct fsck_options fsck_walk_options =FSCK_OPTIONS_DEFAULT ; -static struct fsck_options fsck_obj_options =FSCK_OPTIONS_DEFAULT ; +static struct fsck_options fsck_walk_options =FSCK_OBJECTS_OPTIONS_DEFAULT ; +static struct fsck_options fsck_obj_options =FSCK_OBJECTS_OPTIONS_DEFAULT ;static int errors_found;static int write_lost_and_found;static int verbose;
@@ -214,7 +214,7 @@ static int mark_used(struct object *obj, enum object_type type UNUSED,static void mark_unreachable_referents (const struct object_id *oid) { -struct fsck_options options =FSCK_OPTIONS_DEFAULT ; +struct fsck_options options =FSCK_OBJECTS_OPTIONS_DEFAULT ;struct object *obj =lookup_object (the_repository, oid);if (!obj || !(obj->flags &HAS_OBJ ))
@@ -233,7 +233,7 @@ static void mark_unreachable_referents(const struct object_id *oid)object_as_type (obj, type,0 ); } - options.walk = mark_used; + options.objects_options.walk = mark_used;fsck_walk (obj,NULL , &options);if (obj->type ==OBJ_TREE )free_tree_buffer ((struct tree *)obj);
@@ -936,8 +936,8 @@ int cmd_fsck(int argc, const char **argv, const char *prefix) argc =parse_options (argc, argv, prefix, fsck_opts, fsck_usage,0 ); - fsck_walk_options.walk = mark_object; - fsck_obj_options.walk = mark_used; + fsck_walk_options.objects_options.walk = mark_object; + fsck_obj_options.objects_options.walk = mark_used; fsck_obj_options.error_func = fsck_error_func;if (check_strict) fsck_obj_options.strict =1 ;
builtin/index-pack.c +2 -2
@@ -127,7 +127,7 @@ static int nr_threads;static int from_stdin;static int strict;static int do_fsck_object; -static struct fsck_options fsck_options =FSCK_OPTIONS_MISSING_GITMODULES ; +static struct fsck_options fsck_options =FSCK_OBJECTS_OPTIONS_MISSING_GITMODULES ;static int verbose;static const char *progress_title;static int show_resolving_progress;
@@ -1746,7 +1746,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)usage (index_pack_usage);disable_replace_refs (); - fsck_options.walk = mark_link; + fsck_options.objects_options.walk = mark_link;reset_pack_idx_option (&opts); opts.flags |=WRITE_REV ;
builtin/mktag.c +1 -1
@@ -15,7 +15,7 @@ static char const * const builtin_mktag_usage[] = { };static int option_strict =1 ; -static struct fsck_options fsck_options =FSCK_OPTIONS_STRICT ; +static struct fsck_options fsck_options =FSCK_OBJECTS_OPTIONS_STRICT ; static int mktag_fsck_error_func(struct fsck_options *o UNUSED, const struct object_id *oid UNUSED,
builtin/unpack-objects.c +2 -2
@@ -25,7 +25,7 @@ static unsigned int offset, len;static off_t consumed_bytes;static off_t max_input_size;static git_hash_ctx ctx; -static struct fsck_options fsck_options =FSCK_OPTIONS_STRICT ; +static struct fsck_options fsck_options =FSCK_OBJECTS_OPTIONS_STRICT ;static struct progress *progress;/*
@@ -239,7 +239,7 @@ static int check_object(struct object *obj, enum object_type type,die ("Whoops! Cannot find object '%s'" ,oid_to_hex (&obj->oid));if (fsck_object (obj, obj_buf->buffer, obj_buf->size, &fsck_options))die ("fsck error in packed object" ); - fsck_options.walk = check_object; + fsck_options.objects_options.walk = check_object;if (fsck_walk (obj,NULL , &fsck_options))die ("Error on reachable objects of %s" ,oid_to_hex (&obj->oid));write_cached_object (obj, obj_buf);
fetch-pack.c +6 -4
@@ -46,7 +46,7 @@ static int server_supports_filtering;static int advertise_sid;static struct shallow_lock shallow_lock;static const char *alternate_shallow_file; -static struct fsck_options fsck_options =FSCK_OPTIONS_MISSING_GITMODULES ; +static struct fsck_options fsck_options =FSCK_OBJECTS_OPTIONS_MISSING_GITMODULES ;static struct strbuf fsck_msg_types =STRBUF_INIT ;static struct string_list uri_protocols =STRING_LIST_INIT_DUP ;
@@ -1222,7 +1222,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, }else alternate_shallow_file =NULL ;if (get_pack (args, fd, pack_lockfiles,NULL , sought, nr_sought, - &fsck_options.gitmodules_found)) + &fsck_options.objects_options.gitmodules_found))die (_ ("git fetch-pack: fetch failed." ));if (fsck_finish (&fsck_options))die ("fsck failed" );
@@ -1782,7 +1782,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,if (get_pack (args, fd, pack_lockfiles, packfile_uris.nr ? &index_pack_args :NULL , - sought, nr_sought, &fsck_options.gitmodules_found)) + sought, nr_sought, + &fsck_options.objects_options.gitmodules_found))die (_ ("git fetch-pack: fetch failed." ));do_check_stateless_delimiter (args->stateless_rpc, &reader);
@@ -1825,7 +1826,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, packname[the_hash_algo->hexsz] ='\0' ; -parse_gitmodules_oids (cmd.out, &fsck_options.gitmodules_found); +parse_gitmodules_oids (cmd.out, + &fsck_options.objects_options.gitmodules_found);close (cmd.out);
fsck.c +28 -23
158 lines not on this page · raw message
fsck.h +31 -21
72 lines not on this page · raw message
object-file.c +1 -1
@@ -2507,7 +2507,7 @@ static int index_mem(struct index_state *istate, } }if (flags &HASH_FORMAT_CHECK ) { -struct fsck_options opts =FSCK_OPTIONS_DEFAULT ; +struct fsck_options opts =FSCK_OBJECTS_OPTIONS_DEFAULT ; opts.strict =1 ; opts.error_func = hash_format_check_report;
Signature · 1 line
2.45.2
PATCH v6 2/11 fsck: rename "skiplist" to "oid_skiplist"
In reply to shejialuoBecause we introduce ref consistency check. The original "skiplist" is a common option which is set up during handling user configs. To avoid causing ambiguity, rename "skiplist" to "oid_skiplist".
--- fsck.c | 4 ++-- fsck.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-)
fsck.c +2 -2
@@ -203,7 +203,7 @@ void fsck_set_msg_types(struct fsck_options *options, const char *values)if (!strcmp (buf,"skiplist" )) {if (equal == len)die ("skiplist requires a path" ); -oidset_parse_file (&options->skiplist, buf + equal +1 ); +oidset_parse_file (&options->oid_skiplist, buf + equal +1 ); buf += len +1 ;continue ; }
@@ -220,7 +220,7 @@ void fsck_set_msg_types(struct fsck_options *options, const char *values)static int object_on_skiplist (struct fsck_options *opts,const struct object_id *oid) { -return opts && oid &&oidset_contains (&opts->skiplist, oid); +return opts && oid &&oidset_contains (&opts->oid_skiplist, oid); } __attribute__((format (printf, 5, 6)))
fsck.h +2 -2
@@ -143,13 +143,13 @@ struct fsck_objects_options {struct fsck_options {unsigned strict:1 ;enum fsck_msg_type *msg_type; -struct oidset skiplist; +struct oidset oid_skiplist;fsck_error error_func;struct fsck_objects_options objects_options; };#define FSCK_OBJECTS_OPTIONS_DEFAULT { \ - .skiplist = OIDSET_INIT, \ + .oid_skiplist = OIDSET_INIT, \ .error_func = fsck_error_function, \ .objects_options = { \ .gitmodules_found = OIDSET_INIT, \
Signature · 1 line
2.45.2
PATCH v6 3/11 fsck: add "fsck_refs_options" into "fsck_options"
In reply to shejialuoIntroduce "fsck_refs_options" to represent the refs-related options.
--- fsck.h | 5 +++++ 1 file changed, 5 insertions(+)
fsck.h +5 -0
@@ -131,6 +131,10 @@ int fsck_error_cb_print_missing_gitmodules(struct fsck_options *o,enum fsck_msg_id msg_id, const char *message); +struct fsck_refs_options { +unsigned verbose:1 ; +}; +struct fsck_objects_options {fsck_walk_func walk;struct oidset gitmodules_found;
@@ -146,6 +150,7 @@ struct fsck_options {struct oidset oid_skiplist;fsck_error error_func;struct fsck_objects_options objects_options; +struct fsck_refs_options refs_options; };#define FSCK_OBJECTS_OPTIONS_DEFAULT { \
Signature · 1 line
2.45.2
PATCH v6 4/11 fsck: add a unified interface for reporting fsck messages
In reply to shejialuoThe static function "report" provided by "fsck.c" aims at reporting the problems related to object database which cannot be reused for refs. In order to provide a unified interface which can report either objects or refs, create a new function "vfsck_report" by adding "checked_ref_name" parameter following the "report" prototype. However, instead of using "...", provide "va_list" to allow more flexibility.
The "vfsck_report" function will use "error_func" registered in "fsck_options" function to report customized messages. Change "error_func" prototype to align with the "vfsck_report".
Change "report" function to make it use "vfsck_report" to report objects-related messages. Add a new function called "fsck_refs_report" to use "vfsck_report" to report refs-related messages.
--- builtin/fsck.c | 15 ++++----- builtin/mktag.c | 1 + fsck.c | 81 ++++++++++++++++++++++++++++++++++++------------- fsck.h | 40 +++++++++++++++--------- object-file.c | 11 ++++--- 5 files changed, 101 insertions(+), 47 deletions(-)
builtin/fsck.c +8 -7
@@ -89,12 +89,13 @@ static int objerror(struct object *obj, const char *err)return -1 ; } -static int fsck_error_func(struct fsck_options *o UNUSED, -const struct object_id *oid, -enum object_type object_type, -enum fsck_msg_type msg_type, - enum fsck_msg_id msg_id UNUSED, -const char *message) +static int fsck_objects_error_func(struct fsck_options *o UNUSED, +const struct object_id *oid, +enum object_type object_type, + const char *checked_ref_name UNUSED, +enum fsck_msg_type msg_type, + enum fsck_msg_id msg_id UNUSED, +const char *message) {switch (msg_type) {case FSCK_WARN :
@@ -938,7 +939,7 @@ int cmd_fsck(int argc, const char **argv, const char *prefix) fsck_walk_options.objects_options.walk = mark_object; fsck_obj_options.objects_options.walk = mark_used; - fsck_obj_options.error_func = fsck_error_func; + fsck_obj_options.error_func = fsck_objects_error_func;if (check_strict) fsck_obj_options.strict =1 ;
builtin/mktag.c +1 -0
@@ -20,6 +20,7 @@ static struct fsck_options fsck_options = FSCK_OBJECTS_OPTIONS_STRICT; static int mktag_fsck_error_func(struct fsck_options *o UNUSED, const struct object_id *oid UNUSED, enum object_type object_type UNUSED, + const char *checked_ref_name UNUSED,enum fsck_msg_type msg_type, enum fsck_msg_id msg_id UNUSED,const char *message)
fsck.c +60 -21
115 lines not on this page · raw message
fsck.h +26 -14
@@ -114,22 +114,27 @@ int is_valid_msg_type(const char *msg_id, const char *msg_type);typedef int (*fsck_walk_func )(struct object *obj,enum object_type object_type,void *data,struct fsck_options *options); -/* callback for fsck_object, type is FSCK_ERROR or FSCK_WARN */ +/* +* callback function for reporting errors when checking either objects or refs +*/ typedef int (*fsck_error )(struct fsck_options *o,const struct object_id *oid,enum object_type object_type, +const char *checked_ref_name,enum fsck_msg_type msg_type,enum fsck_msg_id msg_id,const char *message); -int fsck_error_function (struct fsck_options *o, -const struct object_id *oid,enum object_type object_type, -enum fsck_msg_type msg_type,enum fsck_msg_id msg_id, -const char *message); -int fsck_error_cb_print_missing_gitmodules (struct fsck_options *o, -const struct object_id *oid, -enum object_type object_type, -enum fsck_msg_type msg_type, -enum fsck_msg_id msg_id, -const char *message); +int fsck_objects_error_function (struct fsck_options *o, +const struct object_id *oid,enum object_type object_type, +const char *checked_ref_name, +enum fsck_msg_type msg_type,enum fsck_msg_id msg_id, +const char *message); +int fsck_objects_error_cb_print_missing_gitmodules (struct fsck_options *o, +const struct object_id *oid, +enum object_type object_type, +const char *checked_ref_name, +enum fsck_msg_type msg_type, +enum fsck_msg_id msg_id, +const char *message);struct fsck_refs_options {unsigned verbose:1 ;
@@ -155,7 +160,7 @@ struct fsck_options {#define FSCK_OBJECTS_OPTIONS_DEFAULT { \ .oid_skiplist = OIDSET_INIT, \ - .error_func = fsck_error_function, \ + .error_func = fsck_objects_error_function, \ .objects_options = { \ .gitmodules_found = OIDSET_INIT, \ .gitmodules_done = OIDSET_INIT, \
@@ -165,7 +170,7 @@ struct fsck_options { }#define FSCK_OBJECTS_OPTIONS_STRICT { \ .strict = 1, \ - .error_func = fsck_error_function, \ + .error_func = fsck_objects_error_function, \ .objects_options = { \ .gitmodules_found = OIDSET_INIT, \ .gitmodules_done = OIDSET_INIT, \
@@ -175,7 +180,7 @@ struct fsck_options { }#define FSCK_OBJECTS_OPTIONS_MISSING_GITMODULES { \ .strict = 1, \ - .error_func = fsck_error_cb_print_missing_gitmodules, \ + .error_func = fsck_objects_error_cb_print_missing_gitmodules, \ .objects_options = { \ .gitmodules_found = OIDSET_INIT, \ .gitmodules_done = OIDSET_INIT, \
@@ -224,6 +229,13 @@ int fsck_tag_standalone(const struct object_id *oid, const char *buffer,*/ int fsck_finish (struct fsck_options *options); +__attribute__((format (printf,5 ,6 ))) +int fsck_refs_report (struct fsck_options *options, +const struct object_id *oid, +const char *checked_ref_name, +enum fsck_msg_id msg_id, +const char *fmt, ...); +/* * Subsystem for storing human-readable names for each object. *
object-file.c +6 -5
@@ -2473,11 +2473,12 @@ int repo_has_object_file(struct repository *r,* give more context. */ static int hash_format_check_report(struct fsck_options *opts UNUSED, - const struct object_id *oid UNUSED, - enum object_type object_type UNUSED, - enum fsck_msg_type msg_type UNUSED, - enum fsck_msg_id msg_id UNUSED, -const char *message) + const struct object_id *oid UNUSED, + enum object_type object_type UNUSED, + const char *ref_checked_name UNUSED, + enum fsck_msg_type msg_type UNUSED, + enum fsck_msg_id msg_id UNUSED, +const char *message) {error (_ ("object fails fsck: %s" ), message);return 1 ;
Signature · 1 line
2.45.2
PATCH v6 5/11 fsck: add "fsck_refs_options" initialization macros
In reply to shejialuoAdd "FSCK_REFS_OPTIONS_DEFAULT" and "FSCK_REFS_OPTIONS_STRICT" macros to create the refs options easily. Add refs-specific "error_func" callback "fsck_refs_error_function".
"fsck_refs_error_function" will use the "oid" parameter. When the caller passed the oid, it will use "oid_to_hex" to get the corresponding hex value to report to the user.
--- fsck.c | 22 ++++++++++++++++++++++ fsck.h | 14 ++++++++++++++ 2 files changed, 36 insertions(+)
fsck.c +22 -0
@@ -1251,6 +1251,28 @@ int fsck_objects_error_function(struct fsck_options *o,return 1 ; } +int fsck_refs_error_function(struct fsck_options *options UNUSED, +const struct object_id *oid, + enum object_type object_type UNUSED, +const char *checked_ref_name, +enum fsck_msg_type msg_type, + enum fsck_msg_id msg_id UNUSED, +const char *message) +{ +static struct strbuf sb =STRBUF_INIT ; + +strbuf_addstr (&sb, checked_ref_name); +if (oid) +strbuf_addf (&sb," -> (%s)" ,oid_to_hex (oid)); + +if (msg_type ==FSCK_WARN ) { +warning ("%s: %s" , sb.buf, message); +return 0 ; + } +error ("%s: %s" , sb.buf, message); +return 1 ; +} +static int fsck_blobs (struct oidset *blobs_found,struct oidset *blobs_done,enum fsck_msg_id msg_missing,enum fsck_msg_id msg_type,struct fsck_options *options,const char *blob_type)
fsck.h +14 -0
@@ -135,6 +135,13 @@ int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o,enum fsck_msg_type msg_type, enum fsck_msg_id msg_id, const char *message); +int fsck_refs_error_function (struct fsck_options *options, +const struct object_id *oid, +enum object_type object_type, +const char *checked_ref_name, +enum fsck_msg_type msg_type, +enum fsck_msg_id msg_id, +const char *message);struct fsck_refs_options {unsigned verbose:1 ;
@@ -188,6 +195,13 @@ struct fsck_options { .gitattributes_done = OIDSET_INIT, \ } \ } +#define FSCK_REFS_OPTIONS_DEFAULT { \ + .error_func = fsck_refs_error_function, \ +} +#define FSCK_REFS_OPTIONS_STRICT { \ + .strict = 1, \ + .error_func = fsck_refs_error_function, \ +}/* descend in all linked child objects * the return value is:
Signature · 1 line
2.45.2
PATCH v6 6/11 refs: set up ref consistency check infrastructure
In reply to shejialuoThe interfaces defined in the ref_storage_be are carefully structured in semantic. It's organized as the five parts:
1. The name and the initialization interfaces.
2. The ref transaction interfaces.
3. The ref internal interfaces (pack, rename and copy).
4. The ref filesystem interfaces.
5. The reflog related interfaces.
To keep consistent with the git-fsck(1), add a new interface named "fsck_refs_fn" to the end of "ref_storage_be". This semantic cannot be grouped into any above five categories. Explicitly add blank line to make it different from others.
--- refs.c | 5 +++++ refs.h | 8 ++++++++ refs/debug.c | 11 +++++++++++ refs/files-backend.c | 15 ++++++++++++++- refs/packed-backend.c | 8 ++++++++ refs/refs-internal.h | 6 ++++++ refs/reftable-backend.c | 8 ++++++++ 7 files changed, 60 insertions(+), 1 deletion(-)
refs.c +5 -0
@@ -316,6 +316,11 @@ int check_refname_format(const char *refname, int flags)return check_or_sanitize_refname (refname, flags,NULL ); } +int refs_fsck (struct ref_store *refs,struct fsck_options *o) +{ +return refs->be->fsck (refs, o); +} +void sanitize_refname_component (const char *refname,struct strbuf *out) {if (check_or_sanitize_refname (refname,REFNAME_ALLOW_ONELEVEL , out))
refs.h +8 -0
@@ -3,6 +3,7 @@#include "commit.h" +struct fsck_options ;struct object_id ;struct ref_store ;struct repository ;
@@ -547,6 +548,13 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat*/ int check_refname_format (const char *refname,int flags); +/* +* Check the reference database for consistency. Return 0 if refs and +* reflogs are consistent, and non-zero otherwise. The errors will be +* written to stderr. +*/ +int refs_fsck (struct ref_store *refs,struct fsck_options *o); +/* * Apply the rules from check_refname_format, but mutate the result until it * is acceptable, and place the result in "out".
refs/debug.c +11 -0
@@ -419,6 +419,15 @@ static int debug_reflog_expire(struct ref_store *ref_store, const char *refname,return res; } +static int debug_fsck (struct ref_store *ref_store, +struct fsck_options *o) +{ +struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store; +int res = drefs->refs->be->fsck (drefs->refs, o); +trace_printf_key (&trace_refs,"fsck: %d\n" , res); +return res; +} +struct ref_storage_be refs_be_debug = { .name ="debug" , .init =NULL ,
@@ -451,4 +460,6 @@ struct ref_storage_be refs_be_debug = { .create_reflog = debug_create_reflog, .delete_reflog = debug_delete_reflog, .reflog_expire = debug_reflog_expire, + + .fsck = debug_fsck, };
refs/files-backend.c +14 -1
@@ -3406,6 +3406,17 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,return ret; } +static int files_fsck (struct ref_store *ref_store, +struct fsck_options *o) +{ +int ret; +struct files_ref_store *refs = +files_downcast (ref_store,REF_STORE_READ ,"fsck" ); + + ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o); +return ret; +} +struct ref_storage_be refs_be_files = { .name ="files" , .init = files_ref_store_init,
@@ -3432,5 +3443,7 @@ struct ref_storage_be refs_be_files = { .reflog_exists = files_reflog_exists, .create_reflog = files_create_reflog, .delete_reflog = files_delete_reflog, - .reflog_expire = files_reflog_expire + .reflog_expire = files_reflog_expire, + + .fsck = files_fsck, };
refs/packed-backend.c +8 -0
@@ -1733,6 +1733,12 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_sreturn empty_ref_iterator_begin (); } +static int packed_fsck (struct ref_store *ref_store, +struct fsck_options *o) +{ +return 0 ; +} +struct ref_storage_be refs_be_packed = { .name ="packed" , .init = packed_ref_store_init,
@@ -1760,4 +1766,6 @@ struct ref_storage_be refs_be_packed = { .create_reflog = NULL, .delete_reflog = NULL, .reflog_expire = NULL, + + .fsck = packed_fsck, };
refs/refs-internal.h +6 -0
@@ -4,6 +4,7 @@#include "refs.h" #include "iterator.h" +struct fsck_options ;struct ref_transaction ;/*
@@ -650,6 +651,9 @@ typedef int read_raw_ref_fn(struct ref_store *ref_store, const char *refname,typedef int read_symbolic_ref_fn (struct ref_store *ref_store,const char *refname,struct strbuf *referent); +typedef int fsck_fn (struct ref_store *ref_store, +struct fsck_options *o); +struct ref_storage_be {const char *name;ref_store_init_fn *init;
@@ -677,6 +681,8 @@ struct ref_storage_be {create_reflog_fn *create_reflog;delete_reflog_fn *delete_reflog;reflog_expire_fn *reflog_expire; + +fsck_fn *fsck; };extern struct ref_storage_be refs_be_files;
refs/reftable-backend.c +8 -0
@@ -2292,6 +2292,12 @@ static int reftable_be_reflog_expire(struct ref_store *ref_store,return ret; } +static int reftable_be_fsck (struct ref_store *ref_store, +struct fsck_options *o) +{ +return 0 ; +} +struct ref_storage_be refs_be_reftable = { .name ="reftable" , .init = reftable_be_init,
@@ -2319,4 +2325,6 @@ struct ref_storage_be refs_be_reftable = { .create_reflog = reftable_be_create_reflog, .delete_reflog = reftable_be_delete_reflog, .reflog_expire = reftable_be_reflog_expire, + + .fsck = reftable_be_fsck, };
Signature · 1 line
2.45.2
PATCH v6 7/11 builtin/refs: add verify subcommand
In reply to shejialuoIntroduce a new subcommand "verify" in git-refs(1) to allow the user to check the reference database consistency.
--- Documentation/git-refs.txt | 13 +++++++++++ builtin/refs.c | 44 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+)
Documentation/git-refs.txt +13 -0
@@ -10,6 +10,7 @@ SYNOPSIS -------- [verse] 'git refs migrate' --ref-format=<format> [--dry-run] +'git refs verify' [--strict] [--verbose] DESCRIPTION -----------
@@ -22,6 +23,9 @@ COMMANDS migrate:: Migrate ref store between different formats. +verify:: + Verify reference database consistency. + OPTIONS -------
@@ -39,6 +43,15 @@ include::ref-storage-format.txt[] can be used to double check that the migration works as expected before performing the actual migration. +The following options are specific to 'git refs verify': + +--strict:: + Enable more strict checking, every WARN severity for the `Fsck Messages` + be seen as ERROR. See linkgit:git-fsck[1]. + +--verbose:: + When verifying the reference database consistency, be chatty. + KNOWN LIMITATIONS -----------------
builtin/refs.c +44 -0
69 lines not on this page · raw message
Signature · 1 line
2.45.2
PATCH v6 8/11 builtin/fsck: add `git-refs verify` child process
In reply to shejialuoIntroduce a new function "fsck_refs" that initializes and runs a child process to execute the "git-refs verify" command.
--- builtin/fsck.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+)
builtin/fsck.c +17 -0
@@ -897,6 +897,21 @@ static int check_pack_rev_indexes(struct repository *r, int show_progress)return res; } +static void fsck_refs (void ) +{ +struct child_process refs_verify =CHILD_PROCESS_INIT ; +child_process_init (&refs_verify); + refs_verify.git_cmd =1 ; +strvec_pushl (&refs_verify.args,"refs" ,"verify" ,NULL ); +if (verbose) +strvec_push (&refs_verify.args,"--verbose" ); +if (check_strict) +strvec_push (&refs_verify.args,"--strict" ); + +if (run_command (&refs_verify)) + errors_found |=ERROR_REFS ; +} +static char const *const fsck_usage[] = { N_("git fsck [--tags] [--root] [--unreachable] [--cache] [--no-reflogs]\n" " [--[no-]full] [--strict] [--verbose] [--lost-found]\n"
@@ -1066,6 +1081,8 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)check_connectivity (); +fsck_refs (); +if (the_repository->settings.core_commit_graph) {struct child_process commit_graph_verify =CHILD_PROCESS_INIT ;
Signature · 1 line
2.45.2
PATCH v6 9/11 files-backend: add unified interface for refs scanning
In reply to shejialuoFor refs and reflogs, we need to scan its corresponding directories to check every regular file or symbolic link which shares the same pattern. Introduce a unified interface for scanning directories for files-backend.
--- refs/files-backend.c | 77 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-)
refs/files-backend.c +76 -1
@@ -4,6 +4,7 @@#include "../gettext.h" #include "../hash.h" #include "../hex.h" +#include "../fsck.h" #include "../refs.h" #include "refs-internal.h" #include "ref-cache.h"
@@ -3406,6 +3407,78 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,return ret; } +/* +* For refs and reflogs, they share a unified interface when scanning +* the whole directory. This function is used as the callback for each +* regular file or symlink in the directory. +*/ +typedef int (*files_fsck_refs_fn )(struct fsck_options *o, +const char *gitdir, +const char *refs_check_dir, +struct dir_iterator *iter); + +static int files_fsck_refs_dir (struct ref_store *ref_store, +struct fsck_options *o, +const char *refs_check_dir, +files_fsck_refs_fn *fsck_refs_fns) +{ +const char *gitdir = ref_store->gitdir; +struct strbuf sb =STRBUF_INIT ; +struct dir_iterator *iter; +int iter_status; +int ret =0 ; + +strbuf_addf (&sb,"%s/%s" , gitdir, refs_check_dir); + + iter =dir_iterator_begin (sb.buf,0 ); + +if (!iter) { + ret =error_errno ("cannot open directory %s" , sb.buf); + goto out; + } + +while ((iter_status =dir_iterator_advance (iter)) ==ITER_OK ) { +if (S_ISDIR (iter->st.st_mode)) { +continue ; + }else if (S_ISREG (iter->st.st_mode) || +S_ISLNK (iter->st.st_mode)) { +if (o->refs_options.verbose) +fprintf_ln (stderr,"Checking %s/%s" , + refs_check_dir, iter->relative_path); +for (size_t i =0 ; fsck_refs_fns[i]; i++) { +if (fsck_refs_fns[i](o, gitdir, refs_check_dir, iter)) + ret =-1 ; + } + }else { + ret =error (_ ("unexpected file type for '%s'" ), + iter->basename); + } + } + +if (iter_status !=ITER_DONE ) + ret =error (_ ("failed to iterate over '%s'" ), sb.buf); + +out: +strbuf_release (&sb); +return ret; +} + +static int files_fsck_refs (struct ref_store *ref_store, +struct fsck_options *o) +{ +int ret; +files_fsck_refs_fn fsck_refs_fns[]= { +NULL + }; + +if (o->refs_options.verbose) +fprintf_ln (stderr,"Checking references consistency" ); + + ret =files_fsck_refs_dir (ref_store, o,"refs" , fsck_refs_fns); + +return ret; +} +static int files_fsck (struct ref_store *ref_store,struct fsck_options *o) {
@@ -3413,7 +3486,9 @@ static int files_fsck(struct ref_store *ref_store,struct files_ref_store *refs =files_downcast (ref_store,REF_STORE_READ ,"fsck" ); - ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o); + ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o) + |files_fsck_refs (ref_store, o); +return ret; }
Signature · 1 line
2.45.2
PATCH v6 10/11 fsck: add ref name check for files backend
In reply to shejialuoThe git-fsck(1) only implicitly checks the reference, it does not fully check refs with bad format name such as standalone "@" and name ending with ".lock".
In order to provide such checks, add a new fsck message id "badRefName" with default ERROR type. Use existing "check_refname_format" to explicit check the ref name. And add a new unit test to verify the functionality.
--- Documentation/fsck-msgids.txt | 3 + fsck.h | 1 + refs/files-backend.c | 20 +++++++ t/t0602-reffiles-fsck.sh | 101 ++++++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+) create mode 100755 t/t0602-reffiles-fsck.sh
Documentation/fsck-msgids.txt +3 -0
@@ -19,6 +19,9 @@ `badParentSha1`:: (ERROR) A commit object has a bad parent sha1. +`badRefName`:: + (ERROR) A ref has a bad name. + `badTagName`:: (INFO) A tag has an invalid format.
fsck.h +1 -0
@@ -31,6 +31,7 @@ enum fsck_msg_type { FUNC(BAD_NAME, ERROR) \ FUNC(BAD_OBJECT_SHA1, ERROR) \ FUNC(BAD_PARENT_SHA1, ERROR) \ + FUNC(BAD_REF_NAME, ERROR) \ FUNC(BAD_TIMEZONE, ERROR) \ FUNC(BAD_TREE, ERROR) \FUNC (BAD_TREE_SHA1 ,ERROR ) \
refs/files-backend.c +20 -0
@@ -3417,6 +3417,25 @@ typedef int (*files_fsck_refs_fn)(struct fsck_options *o,const char *refs_check_dir, struct dir_iterator *iter); +static int files_fsck_refs_name (struct fsck_options *o, + const char *gitdir UNUSED, +const char *refs_check_dir, +struct dir_iterator *iter) +{ +struct strbuf sb =STRBUF_INIT ; +int ret =0 ; + +if (check_refname_format (iter->basename,REFNAME_ALLOW_ONELEVEL )) { +strbuf_addf (&sb,"%s/%s" , refs_check_dir, iter->relative_path); + ret =fsck_refs_report (o,NULL , sb.buf, +FSCK_MSG_BAD_REF_NAME , +"invalid refname format" ); + } + +strbuf_release (&sb); +return ret; +} + static int files_fsck_refs_dir(struct ref_store *ref_store, struct fsck_options *o, const char *refs_check_dir,
@@ -3468,6 +3487,7 @@ static int files_fsck_refs(struct ref_store *ref_store, {int ret;files_fsck_refs_fn fsck_refs_fns[]= { + files_fsck_refs_name,NULL };
t/t0602-reffiles-fsck.sh new file +101 -0
101 lines not on this page · raw message
Signature · 1 line
2.45.2
PATCH v6 11/11 fsck: add ref content check for files backend
In reply to shejialuoEnhance the git-fsck(1) command by adding a check for reference content in the files backend. The new functionality ensures that symrefs, real symbolic link and regular refs are validated correctly.
In order to check the trailing content of the regular refs, add a new parameter trailing to parse_loose_ref_contents.
For symrefs, parse_loose_ref_contents will set the "referent". However, symbolic link could be either absolute or relative. Use "strbuf_add_real_path" to read the symbolic link and convert the relative path to absolute path. Then use "skip_prefix" to make it align with symref "referent".
Thus, the symrefs and symbolic links could share the same interface. Add a new function "files_fsck_symref_target" which aims at checking the following things:
1. whether the pointee is under the refs/ directory.
2. whether the pointee name is correct.
3. whether the pointee path is a wrong type in filesystem.
Last, add the following FSCK MESSAGEs:
1. "badRefContent(ERROR)": A ref has a bad content
2. "badSymrefPointee(ERROR)": The pointee of a symref is bad.
3. "trailingRefContent(WARN)": A ref content has trailing contents.
--- Documentation/fsck-msgids.txt | 9 +++ fsck.h | 3 + refs.c | 2 +- refs/files-backend.c | 145 +++++++++++++++++++++++++++++++++- refs/refs-internal.h | 5 +- t/t0602-reffiles-fsck.sh | 110 ++++++++++++++++++++++++++ 6 files changed, 269 insertions(+), 5 deletions(-)
Documentation/fsck-msgids.txt +9 -0
@@ -19,9 +19,15 @@ `badParentSha1`:: (ERROR) A commit object has a bad parent sha1. +`badRefContent`:: + (ERROR) A ref has a bad content. + `badRefName`:: (ERROR) A ref has a bad name. +`badSymrefPointee`:: + (ERROR) The pointee of a symref is bad. + `badTagName`:: (INFO) A tag has an invalid format.
@@ -167,6 +173,9 @@ `nullSha1`:: (WARN) Tree contains entries pointing to a null sha1. +`trailingRefContent`:: + (WARN) A ref content has trailing contents. + `treeNotSorted`:: (ERROR) A tree is not properly sorted.
fsck.h +3 -0
@@ -32,6 +32,8 @@ enum fsck_msg_type { FUNC(BAD_OBJECT_SHA1, ERROR) \ FUNC(BAD_PARENT_SHA1, ERROR) \ FUNC(BAD_REF_NAME, ERROR) \ + FUNC(BAD_REF_CONTENT, ERROR) \ + FUNC(BAD_SYMREF_POINTEE, ERROR) \ FUNC(BAD_TIMEZONE, ERROR) \ FUNC(BAD_TREE, ERROR) \FUNC (BAD_TREE_SHA1 ,ERROR ) \
@@ -72,6 +74,7 @@ enum fsck_msg_type { FUNC(HAS_DOTDOT, WARN) \ FUNC(HAS_DOTGIT, WARN) \ FUNC(NULL_SHA1, WARN) \ + FUNC(TRAILING_REF_CONTENT, WARN) \ FUNC(ZERO_PADDED_FILEMODE, WARN) \ FUNC(NUL_IN_COMMIT, WARN) \FUNC (LARGE_PATHNAME ,WARN ) \
refs.c +1 -1
@@ -1758,7 +1758,7 @@ static int refs_read_special_head(struct ref_store *ref_store, } result =parse_loose_ref_contents (content.buf, oid, referent, type, - failure_errno); + failure_errno,NULL ); done:strbuf_release (&full_path);
refs/files-backend.c +143 -2
179 lines not on this page · raw message
refs/refs-internal.h +3 -2
@@ -709,11 +709,12 @@ struct ref_store {/* * Parse contents of a loose ref file. *failure_errno maybe be set to EINVAL for -* invalid contents. +* invalid contents. Also *trailing is set to the first character after the +* refname or NULL if the referent is not empty. */ int parse_loose_ref_contents (const char *buf,struct object_id *oid,struct strbuf *referent,unsigned int *type, -int *failure_errno); +int *failure_errno,const char **trailing);/* * Fill in the generic part of refs and add it to our collection of
t/t0602-reffiles-fsck.sh +110 -0
114 lines not on this page · raw message
Signature · 1 line
2.45.2
Hello,
Quoted 17 lines · shejialuo
Hi All:
This version follows the Junio's advice. Instead of creating the following data structure:
struct fsck_options { enum fsck_type { FSCK_OBJECTS, FSCK_REFS, ... } t; union { struct fsck_objects_options objects; struct fsck_refs_options refs; } u; };
Quoted 5 lines
I simply use the combination idea where "
fsck_options" will incorporate "fsck_objects_options" and "fsck_refs_options". Karthik has told me that I should balance the job I should does and the extensibility for future. So I use the most clear way to do this. Also Junio has said:
If I understood Junio's comments correctly, he was drawing out the point about if we even need the separation of options for refs. Since the only option we're adding is a verbose:
struct fsck_refs_options {
unsigned verbose:1;
};wouldn't it be better if we simply amended fsck_options as so:
diff --git a/fsck.h b/fsck.h
index 6085a384f6..ea97f48acc 100644
--- a/fsck.h
+++ b/fsck.h
@@ -135,6 +135,7 @@ struct fsck_options {
fsck_walk_func walk;
fsck_error error_func;
unsigned strict:1;
+ unsigned verbose_refs:1;
enum fsck_msg_type *msg_type;
struct oidset skiplist;
struct oidset gitmodules_found;Your approach seems to take a different path though, where we create a new route of creating two new structs, one for refs and another for objects and adding both to fsck_objects. If we're doing this, wouldn't it be better to use the enum+union idea, like Junio mentioned? That way we would have clarity around which type it represents.
[snip]
Quoted 50 lines · Karthik Nayak
Hello,
Quoted 17 lines · shejialuo
Hi All:
This version follows the Junio's advice. Instead of creating the following data structure:
struct fsck_options { enum fsck_type { FSCK_OBJECTS, FSCK_REFS, ... } t; union { struct fsck_objects_options objects; struct fsck_refs_options refs; } u; };Quoted 5 lines
I simply use the combination idea where "
fsck_options" will incorporate "fsck_objects_options" and "fsck_refs_options". Karthik has told me that I should balance the job I should does and the extensibility for future. So I use the most clear way to do this. Also Junio has said:If I understood Junio's comments correctly, he was drawing out the point about if we even need the separation of options for refs. Since the only option we're adding is a verbose:
struct fsck_refs_options { unsigned verbose:1; };wouldn't it be better if we simply amended
fsck_optionsas so:diff --git a/fsck.h b/fsck.h index 6085a384f6..ea97f48acc 100644 --- a/fsck.h +++ b/fsck.h @@ -135,6 +135,7 @@ struct fsck_options { fsck_walk_func walk; fsck_error error_func; unsigned strict:1; + unsigned verbose_refs:1; enum fsck_msg_type *msg_type; struct oidset skiplist; struct oidset gitmodules_found;
Quoted 6 lines
Your approach seems to take a different path though, where we create a new route of creating two new structs, one for refs and another for objects and adding both to
fsck_objects. If we're doing this, wouldn't it be better to use the enum+union idea, like Junio mentioned? That way we would have clarity around which type it represents.
I agree. Let's give up breaking the structs. I will send a new version immediately.
Thanks.
Quoted 1 line
[snip]
Quoted 9 lines · Karthik Nayak
If I understood Junio's comments correctly, he was drawing out the point about if we even need the separation of options for refs. Since the only option we're adding is a verbose:
...
Your approach seems to take a different path though, where we create a new route of creating two new structs, one for refs and another for objects and adding both tofsck_objects. If we're doing this, wouldn't it be better to use the enum+union idea, like Junio mentioned? That way we would have clarity around which type it represents.
Yup. If we are going to over-engineer this, enum+union would be a reasonable way to do so, but we should ask if we need to split (and more importantly, if we know the problem space well enough to make the right split) in the first place. Just like premature optimization is bad, premature factoring and over-modularization is bad.
Thanks.
v7 cover ref consistency check infra setup
In reply to shejialuoHi All:
We have come to an agreement that there is no need to add complexity in "fsck_options". So this version just adds a new field into "fsck_option" for refs. The other parts are just the same as the previous version.
CI: https://github.com/shejialuo/git/pull/7
This patch is made upon the latest commit:
06e570c0df (Sync with 'maint', 2024-07-02)
Quoted 2 lines
Just like premature optimization is bad, premature factoring and over-modularization is bad.
I have learned a lot from yesterday's email. I always think we should modularize the code which made a lot of trouble in this series. However, the range-diff below may contain some commits from others. (Because this patch is built upon the latest, sorry for the inconvenience)
Thanks for every reviewer.
shejialuo (9):
fsck: rename "skiplist" to "oid_skiplist"
fsck: add a unified interface for reporting fsck messages
fsck: add refs-related options and error report function
refs: set up ref consistency check infrastructure builtin/refs: add verify subcommand builtin/fsck: add git-refs verify child process
files-backend: add unified interface for refs scanning
fsck: add ref name check for files backend
fsck: add ref content check for files backend
Documentation/fsck-msgids.txt | 12 ++ Documentation/git-refs.txt | 13 ++ builtin/fsck.c | 32 ++++- builtin/mktag.c | 1 + builtin/refs.c | 44 ++++++ fsck.c | 107 +++++++++++--- fsck.h | 63 ++++++--- object-file.c | 11 +- refs.c | 7 +- refs.h | 8 ++ refs/debug.c | 11 ++ refs/files-backend.c | 255 +++++++++++++++++++++++++++++++++- refs/packed-backend.c | 8 ++ refs/refs-internal.h | 11 +- refs/reftable-backend.c | 8 ++ t/t0602-reffiles-fsck.sh | 211 ++++++++++++++++++++++++++++ 16 files changed, 745 insertions(+), 57 deletions(-) create mode 100755 t/t0602-reffiles-fsck.sh
Range-diff against v6:
1: d32ae41a4d < -: ---------- fsck: add "fsck_objects_options" to hold objects-related options 2: 589a7a6fac < -: ---------- fsck: rename "skiplist" to "oid_skiplist" 3: bab97d7c82 < -: ---------- fsck: add "fsck_refs_options" into "fsck_options" -: ---------- > 1: f5fcf36132 fsck: rename "skiplist" to "oid_skiplist" 4: 276da52a6b ! 2: c81b5b6fba fsck: add a unified interface for reporting fsck messages
@@ builtin/fsck.c: static int objerror(struct object *obj, const char *err)
case FSCK_WARN:
@@ builtin/fsck.c: int cmd_fsck(int argc, const char **argv, const char *prefix)
- fsck_walk_options.objects_options.walk = mark_object;
- fsck_obj_options.objects_options.walk = mark_used;
+ fsck_walk_options.walk = mark_object;
+ fsck_obj_options.walk = mark_used;
- fsck_obj_options.error_func = fsck_error_func;
+ fsck_obj_options.error_func = fsck_objects_error_func;
if (check_strict)
@@ builtin/fsck.c: int cmd_fsck(int argc, const char **argv, const char *prefix)
## builtin/mktag.c ##
-@@ builtin/mktag.c: static struct fsck_options fsck_options = FSCK_OBJECTS_OPTIONS_STRICT; +@@ builtin/mktag.c: static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
static int mktag_fsck_error_func(struct fsck_options *o UNUSED,
const struct object_id *oid UNUSED,
enum object_type object_type UNUSED,
@@ fsck.c: static int report(struct fsck_options *options,
+
void fsck_enable_object_names(struct fsck_options *options)
{
- if (!options->objects_options.object_names)
+ if (!options->object_names)
@@ fsck.c: int fsck_buffer(const struct object_id *oid, enum object_type type,
type);
}
@@ fsck.h: int is_valid_msg_type(const char *msg_id, const char *msg_type);
+ enum fsck_msg_id msg_id,
+ const char *message);
- struct fsck_refs_options {
- unsigned verbose:1;
+ struct fsck_options {
+ fsck_walk_func walk;
@@ fsck.h: struct fsck_options {
+ };
- #define FSCK_OBJECTS_OPTIONS_DEFAULT { \
- .oid_skiplist = OIDSET_INIT, \
-- .error_func = fsck_error_function, \
-+ .error_func = fsck_objects_error_function, \
- .objects_options = { \
- .gitmodules_found = OIDSET_INIT, \
- .gitmodules_done = OIDSET_INIT, \
-@@ fsck.h: struct fsck_options {
+ #define FSCK_OPTIONS_DEFAULT { \
+- .skiplist = OIDSET_INIT, \
++ .oid_skiplist = OIDSET_INIT, \
+ .gitmodules_found = OIDSET_INIT, \
+ .gitmodules_done = OIDSET_INIT, \
+ .gitattributes_found = OIDSET_INIT, \
+ .gitattributes_done = OIDSET_INIT, \
+- .error_func = fsck_error_function \
++ .error_func = fsck_objects_error_function \
}
- #define FSCK_OBJECTS_OPTIONS_STRICT { \
+ #define FSCK_OPTIONS_STRICT { \
.strict = 1, \
+@@ fsck.h: struct fsck_options {
+ .gitmodules_done = OIDSET_INIT, \
+ .gitattributes_found = OIDSET_INIT, \
+ .gitattributes_done = OIDSET_INIT, \
- .error_func = fsck_error_function, \
+ .error_func = fsck_objects_error_function, \
- .objects_options = { \
- .gitmodules_found = OIDSET_INIT, \
- .gitmodules_done = OIDSET_INIT, \
-@@ fsck.h: struct fsck_options {
}
- #define FSCK_OBJECTS_OPTIONS_MISSING_GITMODULES { \
+ #define FSCK_OPTIONS_MISSING_GITMODULES { \
.strict = 1, \
+@@ fsck.h: struct fsck_options {
+ .gitmodules_done = OIDSET_INIT, \
+ .gitattributes_found = OIDSET_INIT, \
+ .gitattributes_done = OIDSET_INIT, \
- .error_func = fsck_error_cb_print_missing_gitmodules, \
+ .error_func = fsck_objects_error_cb_print_missing_gitmodules, \
- .objects_options = { \
- .gitmodules_found = OIDSET_INIT, \
- .gitmodules_done = OIDSET_INIT, \
+ }
+
+ /* descend in all linked child objects
@@ fsck.h: int fsck_tag_standalone(const struct object_id *oid, const char *buffer,
*/
int fsck_finish(struct fsck_options *options);
5: e93940c50c ! 3: 53156dc847 fsck: add "fsck_refs_options" initialization macros
@@ Metadata
Author: shejialuo <[email protected]>
## Commit message ##
- fsck: add "fsck_refs_options" initialization macros
+ fsck: add refs-related options and error report function
- Add "FSCK_REFS_OPTIONS_DEFAULT" and "FSCK_REFS_OPTIONS_STRICT" macros to
- create the refs options easily. Add refs-specific "error_func" callback
- "fsck_refs_error_function".
+ Add refs-related options to the "fsck_options", create refs-specific
+ "error_func" callback "fsck_refs_error_function".
"fsck_refs_error_function" will use the "oid" parameter. When the caller
- passed the oid, it will use "oid_to_hex" to get the corresponding hex
- value to report to the user.
+ passes the oid, it will use "oid_to_hex" to get the corresponding hex
+ value to report to the caller.
+
+ Last, add "FSCK_REFS_OPTIONS_DEFAULT" and "FSCK_REFS_OPTIONS_STRICT"
+ macros to create refs options easily.
@@ fsck.h: int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *
+ enum fsck_msg_id msg_id,
+ const char *message);
- struct fsck_refs_options {
- unsigned verbose:1;
+ struct fsck_options {
+ fsck_walk_func walk;
+ fsck_error error_func;
+ unsigned strict:1;
++ unsigned verbose_refs:1;
+ enum fsck_msg_type *msg_type;
+ struct oidset oid_skiplist;
+ struct oidset gitmodules_found;
@@ fsck.h: struct fsck_options {
- .gitattributes_done = OIDSET_INIT, \
- } \
+ .gitattributes_done = OIDSET_INIT, \
+ .error_func = fsck_objects_error_cb_print_missing_gitmodules, \
}
+#define FSCK_REFS_OPTIONS_DEFAULT { \
+ .error_func = fsck_refs_error_function, \
6: 85aa953f6d ! 4: 358f4a1be9 refs: set up ref consistency check infrastructure
@@ refs.c: int check_refname_format(const char *refname, int flags)
## refs.h ##
@@
-
#include "commit.h"
+ #include "repository.h"
+struct fsck_options;
struct object_id;
struct ref_store;
- struct repository;
+ struct strbuf;
@@ refs.h: int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat
*/
int check_refname_format(const char *refname, int flags);
7: ce7adc7372 ! 5: e632859df6 builtin/refs: add verify subcommand
@@ builtin/refs.c: static int cmd_refs_migrate(int argc, const char **argv, const c
+ usage(_("too many arguments"));
+
+ if (verbose)
-+ fsck_refs_options.refs_options.verbose = 1;
++ fsck_refs_options.verbose_refs = 1;
+ if (strict)
+ fsck_refs_options.strict = 1;
+
8: 035eafe10b = 6: 5d7a6261ae builtin/fsck: add `git-refs verify` child process
9: 9398bf3f0d ! 7: 33311af1b1 files-backend: add unified interface for refs scanning
@@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
+ continue;
+ } else if (S_ISREG(iter->st.st_mode) ||
+ S_ISLNK(iter->st.st_mode)) {
-+ if (o->refs_options.verbose)
++ if (o->verbose_refs)
+ fprintf_ln(stderr, "Checking %s/%s",
+ refs_check_dir, iter->relative_path);
+ for (size_t i = 0; fsck_refs_fns[i]; i++) {
@@ refs/files-backend.c: static int files_ref_store_remove_on_disk(struct ref_store
+ NULL
+ };
+
-+ if (o->refs_options.verbose)
++ if (o->verbose_refs)
+ fprintf_ln(stderr, "Checking references consistency");
+
+ ret = files_fsck_refs_dir(ref_store, o, "refs", fsck_refs_fns);
10: 046773e35c = 8: 02bd3ac952 fsck: add ref name check for files backend
11: c9b9599589 ! 9: 94d123471d fsck: add ref content check for files backend
@@ refs.c: static int refs_read_special_head(struct ref_store *ref_store,
## refs/files-backend.c ##
@@
+ #define USE_THE_REPOSITORY_VARIABLE
+
#include "../git-compat-util.h"
+#include "../abspath.h"
#include "../copy.h"Signature · 1 line
2.45.2
PATCH v7 1/9 fsck: rename "skiplist" to "oid_skiplist"
Because we introduce ref consistency check. The original "skiplist" is a common option which is set up during handling user configs. To avoid causing ambiguity, rename "skiplist" to "oid_skiplist".
--- fsck.c | 4 ++-- fsck.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-)
fsck.c +2 -2
@@ -205,7 +205,7 @@ void fsck_set_msg_types(struct fsck_options *options, const char *values)if (!strcmp (buf,"skiplist" )) {if (equal == len)die ("skiplist requires a path" ); -oidset_parse_file (&options->skiplist, buf + equal +1 , +oidset_parse_file (&options->oid_skiplist, buf + equal +1 , the_repository->hash_algo); buf += len +1 ;continue ;
@@ -223,7 +223,7 @@ void fsck_set_msg_types(struct fsck_options *options, const char *values)static int object_on_skiplist (struct fsck_options *opts,const struct object_id *oid) { -return opts && oid &&oidset_contains (&opts->skiplist, oid); +return opts && oid &&oidset_contains (&opts->oid_skiplist, oid); } __attribute__((format (printf, 5, 6)))
fsck.h +1 -1
@@ -136,7 +136,7 @@ struct fsck_options {fsck_error error_func; unsigned strict:1;enum fsck_msg_type *msg_type; -struct oidset skiplist; +struct oidset oid_skiplist;struct oidset gitmodules_found;struct oidset gitmodules_done;struct oidset gitattributes_found;
Signature · 1 line
2.45.2
Re: [GSoC][PATCH v7 1/9] fsck: rename "skiplist" to "oid_skiplist"
Quoted 3 lines · shejialuo
Because we introduce ref consistency check. The original "skiplist" is a common option which is set up during handling user configs. To avoid causing ambiguity, rename "skiplist" to "
oid_skiplist".
I think the commit message could be expanded on to provide additional context and reasoning for the change. From reading this alone, it sounds like we have already introduced the ref consistency check and are now looking to rename a variable. When really this is a preparatory change. Without reading ahead, I'm also left wondering why this name change reduces ambiguity.
-Justin
Re: [GSoC][PATCH v7 1/9] fsck: rename "skiplist" to "oid_skiplist"
Quoted 5 lines · Justin Tobler
Quoted 3 lines · shejialuo
Because we introduce ref consistency check. The original "skiplist" is a common option which is set up during handling user configs. To avoid causing ambiguity, rename "skiplist" to "
oid_skiplist".
Quoted 7 lines
I think the commit message could be expanded on to provide additional context and reasoning for the change. From reading this alone, it sounds like we have already introduced the ref consistency check and are now looking to rename a variable. When really this is a preparatory change. Without reading ahead, I'm also left wondering why this name change reduces ambiguity.
Yes, I will add more information to show the context. "skiplist" is initialized using "git_fsck_config" to parse the user-specific config. In this series, we introduce ref-specific check. "skiplist" is a general name which may make the caller think "skiplist" is related to both the refs and objects.
For later implementation, we may also introduce "skiplist" for refs, but for refs, we concern about the name not the "oidset". I will add more information to explain how this change reduces ambiguity.
Thanks,
Jialuo
PATCH v7 2/9 fsck: add a unified interface for reporting fsck messages
In reply to shejialuoThe static function "report" provided by "fsck.c" aims at reporting the problems related to object database which cannot be reused for refs. In order to provide a unified interface which can report either objects or refs, create a new function "vfsck_report" by adding "checked_ref_name" parameter following the "report" prototype. However, instead of using "...", provide "va_list" to allow more flexibility.
The "vfsck_report" function will use "error_func" registered in "fsck_options" function to report customized messages. Change "error_func" prototype to align with the "vfsck_report".
Change "report" function to make it use "vfsck_report" to report objects-related messages. Add a new function called "fsck_refs_report" to use "vfsck_report" to report refs-related messages.
--- builtin/fsck.c | 15 ++++----- builtin/mktag.c | 1 + fsck.c | 81 ++++++++++++++++++++++++++++++++++++------------- fsck.h | 42 ++++++++++++++++--------- object-file.c | 11 ++++--- 5 files changed, 102 insertions(+), 48 deletions(-)
builtin/fsck.c +8 -7
@@ -89,12 +89,13 @@ static int objerror(struct object *obj, const char *err)return -1 ; } -static int fsck_error_func(struct fsck_options *o UNUSED, -const struct object_id *oid, -enum object_type object_type, -enum fsck_msg_type msg_type, - enum fsck_msg_id msg_id UNUSED, -const char *message) +static int fsck_objects_error_func(struct fsck_options *o UNUSED, +const struct object_id *oid, +enum object_type object_type, + const char *checked_ref_name UNUSED, +enum fsck_msg_type msg_type, + enum fsck_msg_id msg_id UNUSED, +const char *message) {switch (msg_type) {case FSCK_WARN :
@@ -938,7 +939,7 @@ int cmd_fsck(int argc, const char **argv, const char *prefix) fsck_walk_options.walk = mark_object; fsck_obj_options.walk = mark_used; - fsck_obj_options.error_func = fsck_error_func; + fsck_obj_options.error_func = fsck_objects_error_func;if (check_strict) fsck_obj_options.strict =1 ;
builtin/mktag.c +1 -0
@@ -20,6 +20,7 @@ static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT; static int mktag_fsck_error_func(struct fsck_options *o UNUSED, const struct object_id *oid UNUSED, enum object_type object_type UNUSED, + const char *checked_ref_name UNUSED,enum fsck_msg_type msg_type, enum fsck_msg_id msg_id UNUSED,const char *message)
fsck.c +60 -21
115 lines not on this page · raw message
fsck.h +27 -15
81 lines not on this page · raw message
object-file.c +6 -5
@@ -2470,11 +2470,12 @@ int repo_has_object_file(struct repository *r,* give more context. */ static int hash_format_check_report(struct fsck_options *opts UNUSED, - const struct object_id *oid UNUSED, - enum object_type object_type UNUSED, - enum fsck_msg_type msg_type UNUSED, - enum fsck_msg_id msg_id UNUSED, -const char *message) + const struct object_id *oid UNUSED, + enum object_type object_type UNUSED, + const char *ref_checked_name UNUSED, + enum fsck_msg_type msg_type UNUSED, + enum fsck_msg_id msg_id UNUSED, +const char *message) {error (_ ("object fails fsck: %s" ), message);return 1 ;
Signature · 1 line
2.45.2
Re: [GSoC][PATCH v7 2/9] fsck: add a unified interface for reporting fsck messages
Quoted 7 lines · shejialuo
The static function "report" provided by "
fsck.c" aims at reporting the problems related to object database which cannot be reused for refs. In order to provide a unified interface which can report either objects or refs, create a new function "vfsck_report" by adding "checked_ref_name" parameter following the "report" prototype. However, instead of using "...", provide "va_list" to allow more flexibility.
Quoted 7 lines
The "
vfsck_report" function will use "error_func" registered in "fsck_options" function to report customized messages. Change "error_func" prototype to align with the "vfsck_report".Change "report" function to make it use "
vfsck_report" to report objects-related messages. Add a new function called "fsck_refs_report" to use "vfsck_report" to report refs-related messages.
To restate in my own words, the existing "report" function is checked if an fsck message should be returned and also formats the message. It is currently setup to only handle object database related fsck problems. The fsck_error function type is generalized to also accept a reference name enabling fsck problems to be reported for references.
Quoted 107 lines
Mentored-by Patrick Steinhardt <[email protected]>Mentored-by Karthik Nayak <[email protected]>Signed-off-by shejialuo <[email protected]>--- builtin/fsck.c | 15 ++++----- builtin/mktag.c | 1 + fsck.c | 81 ++++++++++++++++++++++++++++++++++++------------- fsck.h | 42 ++++++++++++++++--------- object-file.c | 11 ++++--- 5 files changed, 102 insertions(+), 48 deletions(-)builtin/fsck.c +8
-7@@ -89,12 +89,13 @@ static int objerror(struct object *obj, const char *err)return -1 ; } -static int fsck_error_func(struct fsck_options *o UNUSED, -const struct object_id *oid, -enum object_type object_type, -enum fsck_msg_type msg_type, - enum fsck_msg_id msg_id UNUSED, -const char *message) +static int fsck_objects_error_func(struct fsck_options *o UNUSED, +const struct object_id *oid, +enum object_type object_type, + const char *checked_ref_name UNUSED, +enum fsck_msg_type msg_type, + enum fsck_msg_id msg_id UNUSED, +const char *message) {switch (msg_type) {case FSCK_WARN :@@ -938,7 +939,7 @@ int cmd_fsck(int argc, const char **argv, const char *prefix) fsck_walk_options.walk = mark_object; fsck_obj_options.walk = mark_used; - fsck_obj_options.error_func = fsck_error_func; + fsck_obj_options.error_func = fsck_objects_error_func;if (check_strict) fsck_obj_options.strict =1 ;builtin/mktag.c +1
-0@@ -20,6 +20,7 @@ static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT; static int mktag_fsck_error_func(struct fsck_options *o UNUSED, const struct object_id *oid UNUSED, enum object_type object_type UNUSED, + const char *checked_ref_name UNUSED,enum fsck_msg_type msg_type, enum fsck_msg_id msg_id UNUSED,const char *message)fsck.c +19
-8@@ -226,12 +226,18 @@ static int object_on_skiplist(struct fsck_options *opts,return opts && oid &&oidset_contains (&opts->oid_skiplist, oid); } -__attribute__((format (printf,5 ,6 ))) -static int report (struct fsck_options *options, -const struct object_id *oid,enum object_type object_type, -enum fsck_msg_id msg_id,const char *fmt, ...) +/* +* Provide a unified interface for either fscking refs or objects. +* It will get the current msg error type and call the error_func callback +* which is registered in the "fsck_options" struct. +*/ +static int vfsck_report (struct fsck_options *options, +const struct object_id *oid, +enum object_type object_type, +const char *checked_ref_name, +enum fsck_msg_id msg_id,const char *fmt,va_list ap) { -va_list ap; +va_list ap_copy;struct strbuf sb =STRBUF_INIT ;enum fsck_msg_type msg_type =fsck_msg_type (msg_id, options);int result;@@ -250,9 +256,9 @@ static int report(struct fsck_options *options,prepare_msg_ids ();strbuf_addf (&sb,"%s: " , msg_id_info[msg_id].camelcased); -va_start (ap, fmt); -strbuf_vaddf (&sb, fmt, ap); - result = options->error_func (options, oid, object_type, +va_copy (ap_copy, ap); +strbuf_vaddf (&sb, fmt, ap_copy); + result = options->error_func (options, oid, object_type, checked_ref_name, msg_type, msg_id, sb.buf);strbuf_release (&sb);va_end (ap);@@ -260,6 +266,36 @@ static int report(struct fsck_options *options,return result; } +__attribute__((format (printf,5 ,6 ))) +static int report (struct fsck_options *options, +const struct object_id *oid,enum object_type object_type, +enum fsck_msg_id msg_id,const char *fmt, ...) +{
fsck.c static int report(struct fsck_options *options, · line 283
Quoted 10 lines
fsck.c +10
-0+va_list ap; +int result; +va_start (ap, fmt); + result =vfsck_report (options, oid, object_type,NULL , + msg_id, fmt, ap); +va_end (ap); +return result; +} + +
Looks like there are some extra new lines here.
fsck.c static int report(struct fsck_options *options, · line 294
Quoted 11 lines
fsck.c +11
-0+ +int fsck_refs_report (struct fsck_options *options, +const struct object_id *oid, +const char *checked_ref_name, +enum fsck_msg_id msg_id,const char *fmt, ...) +{ +va_list ap; +int result; +va_start (ap, fmt); + result =vfsck_report (options, oid,OBJ_NONE , + checked_ref_name, msg_id, fmt, ap);
Do we expect that reference related fsck problems may also be skipped via the oid_skiplist?
Quoted 97 lines
fsck.c +20
-13+va_end (ap); +return result; +} +void fsck_enable_object_names (struct fsck_options *options) {if (!options->object_names)@@ -1200,12 +1236,13 @@ int fsck_buffer(const struct object_id *oid, enum object_type type, type); } -int fsck_error_function (struct fsck_options *o, -const struct object_id *oid, - enum object_type object_type UNUSED, -enum fsck_msg_type msg_type, - enum fsck_msg_id msg_id UNUSED, -const char *message) +int fsck_objects_error_function (struct fsck_options *o, +const struct object_id *oid, + enum object_type object_type UNUSED, + const char *checked_ref_name UNUSED, +enum fsck_msg_type msg_type, + enum fsck_msg_id msg_id UNUSED, +const char *message) {if (msg_type ==FSCK_WARN ) {warning ("object %s: %s" ,fsck_describe_object (o, oid), message);@@ -1303,16 +1340,18 @@ int git_fsck_config(const char *var, const char *value,* Custom error callbacks that are used in more than one place. */ -int fsck_error_cb_print_missing_gitmodules (struct fsck_options *o, -const struct object_id *oid, -enum object_type object_type, -enum fsck_msg_type msg_type, -enum fsck_msg_id msg_id, -const char *message) +int fsck_objects_error_cb_print_missing_gitmodules (struct fsck_options *o, +const struct object_id *oid, +enum object_type object_type, +const char *checked_ref_name, +enum fsck_msg_type msg_type, +enum fsck_msg_id msg_id, +const char *message) {if (msg_id ==FSCK_MSG_GITMODULES_MISSING ) {puts (oid_to_hex (oid));return 0 ; } -return fsck_error_function (o, oid, object_type, msg_type, msg_id, message); +return fsck_objects_error_function (o, oid, object_type, checked_ref_name, + msg_type, msg_id, message); }fsck.h +16
-11@@ -114,22 +114,27 @@ int is_valid_msg_type(const char *msg_id, const char *msg_type);typedef int (*fsck_walk_func )(struct object *obj,enum object_type object_type,void *data,struct fsck_options *options); -/* callback for fsck_object, type is FSCK_ERROR or FSCK_WARN */ +/* +* callback function for reporting errors when checking either objects or refs +*/ typedef int (*fsck_error )(struct fsck_options *o,const struct object_id *oid,enum object_type object_type, +const char *checked_ref_name,enum fsck_msg_type msg_type,enum fsck_msg_id msg_id,const char *message); -int fsck_error_function (struct fsck_options *o, -const struct object_id *oid,enum object_type object_type, -enum fsck_msg_type msg_type,enum fsck_msg_id msg_id, -const char *message); -int fsck_error_cb_print_missing_gitmodules (struct fsck_options *o, -const struct object_id *oid, -enum object_type object_type, -enum fsck_msg_type msg_type, -enum fsck_msg_id msg_id, -const char *message); +int fsck_objects_error_function (struct fsck_options *o, +const struct object_id *oid,enum object_type object_type, +const char *checked_ref_name, +enum fsck_msg_type msg_type,enum fsck_msg_id msg_id, +const char *message); +int fsck_objects_error_cb_print_missing_gitmodules (struct fsck_options *o, +const struct object_id *oid, +enum object_type object_type, +const char *checked_ref_name, +enum fsck_msg_type msg_type, +enum fsck_msg_id msg_id, +const char *message);struct fsck_options {fsck_walk_func walk;
fsck.h struct fsck_options { · line 153
Quoted 5 lines
fsck.h +1
-1@@ -145,12 +150,12 @@ struct fsck_options { };#define FSCK_OPTIONS_DEFAULT { \ - .skiplist = OIDSET_INIT, \ + .oid_skiplist = OIDSET_INIT, \
Since we renamed skiplist -> oid_skiplist in the previous patch, shouldn't we update this there?
Quoted 28 lines
fsck.h +3
-3.gitmodules_found = OIDSET_INIT, \ .gitmodules_done = OIDSET_INIT, \ .gitattributes_found = OIDSET_INIT, \ .gitattributes_done = OIDSET_INIT, \ - .error_func = fsck_error_function \ + .error_func = fsck_objects_error_function \ }#define FSCK_OPTIONS_STRICT { \ .strict = 1, \@@ -158,7 +163,7 @@ struct fsck_options { .gitmodules_done = OIDSET_INIT, \ .gitattributes_found = OIDSET_INIT, \ .gitattributes_done = OIDSET_INIT, \ - .error_func = fsck_error_function, \ + .error_func = fsck_objects_error_function, \ }#define FSCK_OPTIONS_MISSING_GITMODULES { \ .strict = 1, \@@ -166,7 +171,7 @@ struct fsck_options { .gitmodules_done = OIDSET_INIT, \ .gitattributes_found = OIDSET_INIT, \ .gitattributes_done = OIDSET_INIT, \ - .error_func = fsck_error_cb_print_missing_gitmodules, \ + .error_func = fsck_objects_error_cb_print_missing_gitmodules, \ }/* descend in all linked child objects
fsck.h int fsck_tag_standalone(const struct object_id *oid, const char *buffer, · line 216
Quoted 3 lines
fsck.h +0
-0@@ -209,6 +214,13 @@ int fsck_tag_standalone(const struct object_id *oid, const char *buffer,*/ int fsck_finish (struct fsck_options *options);
In my opinion it would be nice to add a comment to document fsck_refs_report() here.
Quoted 36 lines
fsck.h +7
-0+__attribute__((format (printf,5 ,6 ))) +int fsck_refs_report (struct fsck_options *options, +const struct object_id *oid, +const char *checked_ref_name, +enum fsck_msg_id msg_id, +const char *fmt, ...); +/* * Subsystem for storing human-readable names for each object. * object-file.c +6
-5@@ -2470,11 +2470,12 @@ int repo_has_object_file(struct repository *r,* give more context. */ static int hash_format_check_report(struct fsck_options *opts UNUSED, - const struct object_id *oid UNUSED, - enum object_type object_type UNUSED, - enum fsck_msg_type msg_type UNUSED, - enum fsck_msg_id msg_id UNUSED, -const char *message) + const struct object_id *oid UNUSED, + enum object_type object_type UNUSED, + const char *ref_checked_name UNUSED, + enum fsck_msg_type msg_type UNUSED, + enum fsck_msg_id msg_id UNUSED, +const char *message) {error (_ ("object fails fsck: %s" ), message);return 1 ;Signature · 1 line
2.45.2
Re: [GSoC][PATCH v7 2/9] fsck: add a unified interface for reporting fsck messages
Quoted 16 lines · Justin Tobler
Quoted 7 lines · shejialuo
The static function "report" provided by "
fsck.c" aims at reporting the problems related to object database which cannot be reused for refs. In order to provide a unified interface which can report either objects or refs, create a new function "vfsck_report" by adding "checked_ref_name" parameter following the "report" prototype. However, instead of using "...", provide "va_list" to allow more flexibility.Quoted 7 lines
The "
vfsck_report" function will use "error_func" registered in "fsck_options" function to report customized messages. Change "error_func" prototype to align with the "vfsck_report".Change "report" function to make it use "
vfsck_report" to report objects-related messages. Add a new function called "fsck_refs_report" to use "vfsck_report" to report refs-related messages.
Quoted 6 lines
To restate in my own words, the existing "report" function is checked if an fsck message should be returned and also formats the message. It is currently setup to only handle object database related fsck problems. The
fsck_errorfunction type is generalized to also accept a reference name enabling fsck problems to be reported for references.
Exactly, I will add more information in the commit message.
Thanks,
Jialuo
PATCH v7 3/9 fsck: add refs-related options and error report function
In reply to shejialuoAdd refs-related options to the "fsck_options", create refs-specific "error_func" callback "fsck_refs_error_function".
"fsck_refs_error_function" will use the "oid" parameter. When the caller passes the oid, it will use "oid_to_hex" to get the corresponding hex value to report to the caller.
Last, add "FSCK_REFS_OPTIONS_DEFAULT" and "FSCK_REFS_OPTIONS_STRICT" macros to create refs options easily.
--- fsck.c | 22 ++++++++++++++++++++++ fsck.h | 15 +++++++++++++++ 2 files changed, 37 insertions(+)
fsck.c +22 -0
@@ -1252,6 +1252,28 @@ int fsck_objects_error_function(struct fsck_options *o,return 1 ; } +int fsck_refs_error_function(struct fsck_options *options UNUSED, +const struct object_id *oid, + enum object_type object_type UNUSED, +const char *checked_ref_name, +enum fsck_msg_type msg_type, + enum fsck_msg_id msg_id UNUSED, +const char *message) +{ +static struct strbuf sb =STRBUF_INIT ; + +strbuf_addstr (&sb, checked_ref_name); +if (oid) +strbuf_addf (&sb," -> (%s)" ,oid_to_hex (oid)); + +if (msg_type ==FSCK_WARN ) { +warning ("%s: %s" , sb.buf, message); +return 0 ; + } +error ("%s: %s" , sb.buf, message); +return 1 ; +} +static int fsck_blobs (struct oidset *blobs_found,struct oidset *blobs_done,enum fsck_msg_id msg_missing,enum fsck_msg_id msg_type,struct fsck_options *options,const char *blob_type)
fsck.h +15 -0
@@ -135,11 +135,19 @@ int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o,enum fsck_msg_type msg_type, enum fsck_msg_id msg_id, const char *message); +int fsck_refs_error_function (struct fsck_options *options, +const struct object_id *oid, +enum object_type object_type, +const char *checked_ref_name, +enum fsck_msg_type msg_type, +enum fsck_msg_id msg_id, +const char *message);struct fsck_options {fsck_walk_func walk;fsck_error error_func;unsigned strict:1 ; +unsigned verbose_refs:1 ;enum fsck_msg_type *msg_type;struct oidset oid_skiplist;struct oidset gitmodules_found;
@@ -173,6 +181,13 @@ struct fsck_options { .gitattributes_done = OIDSET_INIT, \ .error_func = fsck_objects_error_cb_print_missing_gitmodules, \ } +#define FSCK_REFS_OPTIONS_DEFAULT { \ + .error_func = fsck_refs_error_function, \ +} +#define FSCK_REFS_OPTIONS_STRICT { \ + .strict = 1, \ + .error_func = fsck_refs_error_function, \ +}/* descend in all linked child objects * the return value is:
Signature · 1 line
2.45.2
PATCH v7 4/9 refs: set up ref consistency check infrastructure
In reply to shejialuoThe interfaces defined in the ref_storage_be are carefully structured in semantic. It's organized as the five parts:
1. The name and the initialization interfaces.
2. The ref transaction interfaces.
3. The ref internal interfaces (pack, rename and copy).
4. The ref filesystem interfaces.
5. The reflog related interfaces.
To keep consistent with the git-fsck(1), add a new interface named "fsck_refs_fn" to the end of "ref_storage_be". This semantic cannot be grouped into any above five categories. Explicitly add blank line to make it different from others.
--- refs.c | 5 +++++ refs.h | 8 ++++++++ refs/debug.c | 11 +++++++++++ refs/files-backend.c | 15 ++++++++++++++- refs/packed-backend.c | 8 ++++++++ refs/refs-internal.h | 6 ++++++ refs/reftable-backend.c | 8 ++++++++ 7 files changed, 60 insertions(+), 1 deletion(-)
refs.c +5 -0
@@ -318,6 +318,11 @@ int check_refname_format(const char *refname, int flags)return check_or_sanitize_refname (refname, flags,NULL ); } +int refs_fsck (struct ref_store *refs,struct fsck_options *o) +{ +return refs->be->fsck (refs, o); +} +void sanitize_refname_component (const char *refname,struct strbuf *out) {if (check_or_sanitize_refname (refname,REFNAME_ALLOW_ONELEVEL , out))
refs.h +8 -0
@@ -4,6 +4,7 @@#include "commit.h" #include "repository.h" +struct fsck_options ;struct object_id ;struct ref_store ;struct strbuf ;
@@ -541,6 +542,13 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat*/ int check_refname_format (const char *refname,int flags); +/* +* Check the reference database for consistency. Return 0 if refs and +* reflogs are consistent, and non-zero otherwise. The errors will be +* written to stderr. +*/ +int refs_fsck (struct ref_store *refs,struct fsck_options *o); +/* * Apply the rules from check_refname_format, but mutate the result until it * is acceptable, and place the result in "out".
refs/debug.c +11 -0
@@ -419,6 +419,15 @@ static int debug_reflog_expire(struct ref_store *ref_store, const char *refname,return res; } +static int debug_fsck (struct ref_store *ref_store, +struct fsck_options *o) +{ +struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store; +int res = drefs->refs->be->fsck (drefs->refs, o); +trace_printf_key (&trace_refs,"fsck: %d\n" , res); +return res; +} +struct ref_storage_be refs_be_debug = { .name ="debug" , .init =NULL ,
@@ -451,4 +460,6 @@ struct ref_storage_be refs_be_debug = { .create_reflog = debug_create_reflog, .delete_reflog = debug_delete_reflog, .reflog_expire = debug_reflog_expire, + + .fsck = debug_fsck, };
refs/files-backend.c +14 -1
@@ -3408,6 +3408,17 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,return ret; } +static int files_fsck (struct ref_store *ref_store, +struct fsck_options *o) +{ +int ret; +struct files_ref_store *refs = +files_downcast (ref_store,REF_STORE_READ ,"fsck" ); + + ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o); +return ret; +} +struct ref_storage_be refs_be_files = { .name ="files" , .init = files_ref_store_init,
@@ -3434,5 +3445,7 @@ struct ref_storage_be refs_be_files = { .reflog_exists = files_reflog_exists, .create_reflog = files_create_reflog, .delete_reflog = files_delete_reflog, - .reflog_expire = files_reflog_expire + .reflog_expire = files_reflog_expire, + + .fsck = files_fsck, };
refs/packed-backend.c +8 -0
@@ -1735,6 +1735,12 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_sreturn empty_ref_iterator_begin (); } +static int packed_fsck (struct ref_store *ref_store, +struct fsck_options *o) +{ +return 0 ; +} +struct ref_storage_be refs_be_packed = { .name ="packed" , .init = packed_ref_store_init,
@@ -1762,4 +1768,6 @@ struct ref_storage_be refs_be_packed = { .create_reflog = NULL, .delete_reflog = NULL, .reflog_expire = NULL, + + .fsck = packed_fsck, };
refs/refs-internal.h +6 -0
@@ -4,6 +4,7 @@#include "refs.h" #include "iterator.h" +struct fsck_options ;struct ref_transaction ;/*
@@ -650,6 +651,9 @@ typedef int read_raw_ref_fn(struct ref_store *ref_store, const char *refname,typedef int read_symbolic_ref_fn (struct ref_store *ref_store,const char *refname,struct strbuf *referent); +typedef int fsck_fn (struct ref_store *ref_store, +struct fsck_options *o); +struct ref_storage_be {const char *name;ref_store_init_fn *init;
@@ -677,6 +681,8 @@ struct ref_storage_be {create_reflog_fn *create_reflog;delete_reflog_fn *delete_reflog;reflog_expire_fn *reflog_expire; + +fsck_fn *fsck; };extern struct ref_storage_be refs_be_files;
refs/reftable-backend.c +8 -0
@@ -2303,6 +2303,12 @@ static int reftable_be_reflog_expire(struct ref_store *ref_store,return ret; } +static int reftable_be_fsck (struct ref_store *ref_store, +struct fsck_options *o) +{ +return 0 ; +} +struct ref_storage_be refs_be_reftable = { .name ="reftable" , .init = reftable_be_init,
@@ -2330,4 +2336,6 @@ struct ref_storage_be refs_be_reftable = { .create_reflog = reftable_be_create_reflog, .delete_reflog = reftable_be_delete_reflog, .reflog_expire = reftable_be_reflog_expire, + + .fsck = reftable_be_fsck, };
Signature · 1 line
2.45.2
PATCH v7 5/9 builtin/refs: add verify subcommand
In reply to shejialuoIntroduce a new subcommand "verify" in git-refs(1) to allow the user to check the reference database consistency.
--- Documentation/git-refs.txt | 13 +++++++++++ builtin/refs.c | 44 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+)
Documentation/git-refs.txt +13 -0
@@ -10,6 +10,7 @@ SYNOPSIS -------- [verse] 'git refs migrate' --ref-format=<format> [--dry-run] +'git refs verify' [--strict] [--verbose] DESCRIPTION -----------
@@ -22,6 +23,9 @@ COMMANDS migrate:: Migrate ref store between different formats. +verify:: + Verify reference database consistency. + OPTIONS -------
@@ -39,6 +43,15 @@ include::ref-storage-format.txt[] can be used to double check that the migration works as expected before performing the actual migration. +The following options are specific to 'git refs verify': + +--strict:: + Enable more strict checking, every WARN severity for the `Fsck Messages` + be seen as ERROR. See linkgit:git-fsck[1]. + +--verbose:: + When verifying the reference database consistency, be chatty. + KNOWN LIMITATIONS -----------------
builtin/refs.c +44 -0
69 lines not on this page · raw message
Signature · 1 line
2.45.2
PATCH v7 6/9 builtin/fsck: add `git-refs verify` child process
In reply to shejialuoIntroduce a new function "fsck_refs" that initializes and runs a child process to execute the "git-refs verify" command.
--- builtin/fsck.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+)
builtin/fsck.c +17 -0
@@ -897,6 +897,21 @@ static int check_pack_rev_indexes(struct repository *r, int show_progress)return res; } +static void fsck_refs (void ) +{ +struct child_process refs_verify =CHILD_PROCESS_INIT ; +child_process_init (&refs_verify); + refs_verify.git_cmd =1 ; +strvec_pushl (&refs_verify.args,"refs" ,"verify" ,NULL ); +if (verbose) +strvec_push (&refs_verify.args,"--verbose" ); +if (check_strict) +strvec_push (&refs_verify.args,"--strict" ); + +if (run_command (&refs_verify)) + errors_found |=ERROR_REFS ; +} +static char const *const fsck_usage[] = { N_("git fsck [--tags] [--root] [--unreachable] [--cache] [--no-reflogs]\n" " [--[no-]full] [--strict] [--verbose] [--lost-found]\n"
@@ -1066,6 +1081,8 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)check_connectivity (); +fsck_refs (); +if (the_repository->settings.core_commit_graph) {struct child_process commit_graph_verify =CHILD_PROCESS_INIT ;
Signature · 1 line
2.45.2
PATCH v7 7/9 files-backend: add unified interface for refs scanning
In reply to shejialuoFor refs and reflogs, we need to scan its corresponding directories to check every regular file or symbolic link which shares the same pattern. Introduce a unified interface for scanning directories for files-backend.
--- refs/files-backend.c | 77 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-)
refs/files-backend.c +76 -1
@@ -6,6 +6,7 @@#include "../gettext.h" #include "../hash.h" #include "../hex.h" +#include "../fsck.h" #include "../refs.h" #include "refs-internal.h" #include "ref-cache.h"
@@ -3408,6 +3409,78 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,return ret; } +/* +* For refs and reflogs, they share a unified interface when scanning +* the whole directory. This function is used as the callback for each +* regular file or symlink in the directory. +*/ +typedef int (*files_fsck_refs_fn )(struct fsck_options *o, +const char *gitdir, +const char *refs_check_dir, +struct dir_iterator *iter); + +static int files_fsck_refs_dir (struct ref_store *ref_store, +struct fsck_options *o, +const char *refs_check_dir, +files_fsck_refs_fn *fsck_refs_fns) +{ +const char *gitdir = ref_store->gitdir; +struct strbuf sb =STRBUF_INIT ; +struct dir_iterator *iter; +int iter_status; +int ret =0 ; + +strbuf_addf (&sb,"%s/%s" , gitdir, refs_check_dir); + + iter =dir_iterator_begin (sb.buf,0 ); + +if (!iter) { + ret =error_errno ("cannot open directory %s" , sb.buf); + goto out; + } + +while ((iter_status =dir_iterator_advance (iter)) ==ITER_OK ) { +if (S_ISDIR (iter->st.st_mode)) { +continue ; + }else if (S_ISREG (iter->st.st_mode) || +S_ISLNK (iter->st.st_mode)) { +if (o->verbose_refs) +fprintf_ln (stderr,"Checking %s/%s" , + refs_check_dir, iter->relative_path); +for (size_t i =0 ; fsck_refs_fns[i]; i++) { +if (fsck_refs_fns[i](o, gitdir, refs_check_dir, iter)) + ret =-1 ; + } + }else { + ret =error (_ ("unexpected file type for '%s'" ), + iter->basename); + } + } + +if (iter_status !=ITER_DONE ) + ret =error (_ ("failed to iterate over '%s'" ), sb.buf); + +out: +strbuf_release (&sb); +return ret; +} + +static int files_fsck_refs (struct ref_store *ref_store, +struct fsck_options *o) +{ +int ret; +files_fsck_refs_fn fsck_refs_fns[]= { +NULL + }; + +if (o->verbose_refs) +fprintf_ln (stderr,"Checking references consistency" ); + + ret =files_fsck_refs_dir (ref_store, o,"refs" , fsck_refs_fns); + +return ret; +} +static int files_fsck (struct ref_store *ref_store,struct fsck_options *o) {
@@ -3415,7 +3488,9 @@ static int files_fsck(struct ref_store *ref_store,struct files_ref_store *refs =files_downcast (ref_store,REF_STORE_READ ,"fsck" ); - ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o); + ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o) + |files_fsck_refs (ref_store, o); +return ret; }
Signature · 1 line
2.45.2
PATCH v7 8/9 fsck: add ref name check for files backend
In reply to shejialuoThe git-fsck(1) only implicitly checks the reference, it does not fully check refs with bad format name such as standalone "@" and name ending with ".lock".
In order to provide such checks, add a new fsck message id "badRefName" with default ERROR type. Use existing "check_refname_format" to explicit check the ref name. And add a new unit test to verify the functionality.
--- Documentation/fsck-msgids.txt | 3 + fsck.h | 1 + refs/files-backend.c | 20 +++++++ t/t0602-reffiles-fsck.sh | 101 ++++++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+) create mode 100755 t/t0602-reffiles-fsck.sh
Documentation/fsck-msgids.txt +3 -0
@@ -19,6 +19,9 @@ `badParentSha1`:: (ERROR) A commit object has a bad parent sha1. +`badRefName`:: + (ERROR) A ref has a bad name. + `badTagName`:: (INFO) A tag has an invalid format.
fsck.h +1 -0
@@ -31,6 +31,7 @@ enum fsck_msg_type { FUNC(BAD_NAME, ERROR) \ FUNC(BAD_OBJECT_SHA1, ERROR) \ FUNC(BAD_PARENT_SHA1, ERROR) \ + FUNC(BAD_REF_NAME, ERROR) \ FUNC(BAD_TIMEZONE, ERROR) \ FUNC(BAD_TREE, ERROR) \FUNC (BAD_TREE_SHA1 ,ERROR ) \
refs/files-backend.c +20 -0
@@ -3419,6 +3419,25 @@ typedef int (*files_fsck_refs_fn)(struct fsck_options *o,const char *refs_check_dir, struct dir_iterator *iter); +static int files_fsck_refs_name (struct fsck_options *o, + const char *gitdir UNUSED, +const char *refs_check_dir, +struct dir_iterator *iter) +{ +struct strbuf sb =STRBUF_INIT ; +int ret =0 ; + +if (check_refname_format (iter->basename,REFNAME_ALLOW_ONELEVEL )) { +strbuf_addf (&sb,"%s/%s" , refs_check_dir, iter->relative_path); + ret =fsck_refs_report (o,NULL , sb.buf, +FSCK_MSG_BAD_REF_NAME , +"invalid refname format" ); + } + +strbuf_release (&sb); +return ret; +} + static int files_fsck_refs_dir(struct ref_store *ref_store, struct fsck_options *o, const char *refs_check_dir,
@@ -3470,6 +3489,7 @@ static int files_fsck_refs(struct ref_store *ref_store, {int ret;files_fsck_refs_fn fsck_refs_fns[]= { + files_fsck_refs_name,NULL };
t/t0602-reffiles-fsck.sh new file +101 -0
101 lines not on this page · raw message
Signature · 1 line
2.45.2
PATCH v7 9/9 fsck: add ref content check for files backend
In reply to shejialuoEnhance the git-fsck(1) command by adding a check for reference content in the files backend. The new functionality ensures that symrefs, real symbolic link and regular refs are validated correctly.
In order to check the trailing content of the regular refs, add a new parameter trailing to parse_loose_ref_contents.
For symrefs, parse_loose_ref_contents will set the "referent". However, symbolic link could be either absolute or relative. Use "strbuf_add_real_path" to read the symbolic link and convert the relative path to absolute path. Then use "skip_prefix" to make it align with symref "referent".
Thus, the symrefs and symbolic links could share the same interface. Add a new function "files_fsck_symref_target" which aims at checking the following things:
1. whether the pointee is under the refs/ directory.
2. whether the pointee name is correct.
3. whether the pointee path is a wrong type in filesystem.
Last, add the following FSCK MESSAGEs:
1. "badRefContent(ERROR)": A ref has a bad content
2. "badSymrefPointee(ERROR)": The pointee of a symref is bad.
3. "trailingRefContent(WARN)": A ref content has trailing contents.
--- Documentation/fsck-msgids.txt | 9 +++ fsck.h | 3 + refs.c | 2 +- refs/files-backend.c | 145 +++++++++++++++++++++++++++++++++- refs/refs-internal.h | 5 +- t/t0602-reffiles-fsck.sh | 110 ++++++++++++++++++++++++++ 6 files changed, 269 insertions(+), 5 deletions(-)
Documentation/fsck-msgids.txt +9 -0
@@ -19,9 +19,15 @@ `badParentSha1`:: (ERROR) A commit object has a bad parent sha1. +`badRefContent`:: + (ERROR) A ref has a bad content. + `badRefName`:: (ERROR) A ref has a bad name. +`badSymrefPointee`:: + (ERROR) The pointee of a symref is bad. + `badTagName`:: (INFO) A tag has an invalid format.
@@ -167,6 +173,9 @@ `nullSha1`:: (WARN) Tree contains entries pointing to a null sha1. +`trailingRefContent`:: + (WARN) A ref content has trailing contents. + `treeNotSorted`:: (ERROR) A tree is not properly sorted.
fsck.h +3 -0
@@ -32,6 +32,8 @@ enum fsck_msg_type { FUNC(BAD_OBJECT_SHA1, ERROR) \ FUNC(BAD_PARENT_SHA1, ERROR) \ FUNC(BAD_REF_NAME, ERROR) \ + FUNC(BAD_REF_CONTENT, ERROR) \ + FUNC(BAD_SYMREF_POINTEE, ERROR) \ FUNC(BAD_TIMEZONE, ERROR) \ FUNC(BAD_TREE, ERROR) \FUNC (BAD_TREE_SHA1 ,ERROR ) \
@@ -72,6 +74,7 @@ enum fsck_msg_type { FUNC(HAS_DOTDOT, WARN) \ FUNC(HAS_DOTGIT, WARN) \ FUNC(NULL_SHA1, WARN) \ + FUNC(TRAILING_REF_CONTENT, WARN) \ FUNC(ZERO_PADDED_FILEMODE, WARN) \ FUNC(NUL_IN_COMMIT, WARN) \FUNC (LARGE_PATHNAME ,WARN ) \
refs.c +1 -1
@@ -1760,7 +1760,7 @@ static int refs_read_special_head(struct ref_store *ref_store, } result =parse_loose_ref_contents (content.buf, oid, referent, type, - failure_errno); + failure_errno,NULL ); done:strbuf_release (&full_path);
refs/files-backend.c +143 -2
181 lines not on this page · raw message
refs/refs-internal.h +3 -2
@@ -709,11 +709,12 @@ struct ref_store {/* * Parse contents of a loose ref file. *failure_errno maybe be set to EINVAL for -* invalid contents. +* invalid contents. Also *trailing is set to the first character after the +* refname or NULL if the referent is not empty. */ int parse_loose_ref_contents (const char *buf,struct object_id *oid,struct strbuf *referent,unsigned int *type, -int *failure_errno); +int *failure_errno,const char **trailing);/* * Fill in the generic part of refs and add it to our collection of
t/t0602-reffiles-fsck.sh +110 -0
114 lines not on this page · raw message
Signature · 1 line
2.45.2
v8 cover ref consistency check infra setup
In reply to shejialuoHi All:
This version enhances the patch 1 and patch 2 commit messages according to the review of Justin.
Thanks,
Jialuo
shejialuo (9):
fsck: rename "skiplist" to "oid_skiplist"
fsck: add a unified interface for reporting fsck messages
fsck: add refs-related options and error report function
refs: set up ref consistency check infrastructure builtin/refs: add verify subcommand builtin/fsck: add git-refs verify child process
files-backend: add unified interface for refs scanning
fsck: add ref name check for files backend
fsck: add ref content check for files backend
Documentation/fsck-msgids.txt | 12 ++ Documentation/git-refs.txt | 13 ++ builtin/fsck.c | 32 ++++- builtin/mktag.c | 1 + builtin/refs.c | 44 ++++++ fsck.c | 107 +++++++++++--- fsck.h | 63 ++++++--- object-file.c | 11 +- refs.c | 7 +- refs.h | 8 ++ refs/debug.c | 11 ++ refs/files-backend.c | 255 +++++++++++++++++++++++++++++++++- refs/packed-backend.c | 8 ++ refs/refs-internal.h | 11 +- refs/reftable-backend.c | 8 ++ t/t0602-reffiles-fsck.sh | 211 ++++++++++++++++++++++++++++ 16 files changed, 745 insertions(+), 57 deletions(-) create mode 100755 t/t0602-reffiles-fsck.sh
Range-diff against v7:
1: f5fcf36132 ! 1: 61e475840f fsck: rename "skiplist" to "oid_skiplist"
@@ Metadata
## Commit message ##
fsck: rename "skiplist" to "oid_skiplist"
- Because we introduce ref consistency check. The original "skiplist" is a
- common option which is set up during handling user configs. To avoid
- causing ambiguity, rename "skiplist" to "oid_skiplist".
+ The "skiplist" field in "fsck_options" is related to objects. Because we
+ are going to introduce ref consistency check, the "skiplist" name is too
+ general which will make the caller think "skiplist" is related to both
+ the refs and objects.
+
+ It may seem that for both refs and objects, we should provide a general
+ "skiplist" here. However, the type for "skiplist" is struct oidset
+ which is totally unsuitable for refs.
+
+ To avoid above ambiguity, rename "skiplist" to "oid_skiplist".
2: c81b5b6fba ! 2: f2576d88a9 fsck: add a unified interface for reporting fsck messages
@@ Metadata
## Commit message ##
fsck: add a unified interface for reporting fsck messages
- The static function "report" provided by "fsck.c" aims at reporting the
- problems related to object database which cannot be reused for refs.
- In order to provide a unified interface which can report either objects
- or refs, create a new function "vfsck_report" by adding
- "checked_ref_name" parameter following the "report" prototype. However,
- instead of using "...", provide "va_list" to allow more flexibility.
+ The static function "report" provided by "fsck.c" aims at checking fsck
+ error type and calling the callback "error_func" to report the message.
+ However, "report" function is only related to object database which
+ cannot be reused for refs. In order to provide a unified interface which
+ can report either objects or refs, create a new function "vfsck_report"
+ by adding "checked_ref_name" parameter following the "report" prototype.
+ Instead of using "...", provide "va_list" to allow more flexibility.
- The "vfsck_report" function will use "error_func" registered in
- "fsck_options" function to report customized messages. Change
- "error_func" prototype to align with the "vfsck_report".
+ Like "report", the "vfsck_report" function will use "error_func"
+ registered in "fsck_options" to report customized messages. Change
+ "error_func" prototype to align with the new "vfsck_report".
- Change "report" function to make it use "vfsck_report" to report
- objects-related messages. Add a new function called "fsck_refs_report"
- to use "vfsck_report" to report refs-related messages.
+ Then, change "report" function to use "vfsck_report" to report objects
+ related messages. Add a new function called "fsck_refs_report" to use
+ "vfsck_report" to report refs related messages.
3: 53156dc847 = 3: c3c2dda50c fsck: add refs-related options and error report function 4: 358f4a1be9 = 4: e826dc17ec refs: set up ref consistency check infrastructure 5: e632859df6 = 5: 33cac4882b builtin/refs: add verify subcommand 6: 5d7a6261ae = 6: 32668e3543 builtin/fsck: add `git-refs verify` child process 7: 33311af1b1 = 7: df83b2a990 files-backend: add unified interface for refs scanning 8: 02bd3ac952 = 8: c696c15651 fsck: add ref name check for files backend 9: 94d123471d = 9: 8b0f3aeb9c fsck: add ref content check for files backend
Signature · 1 line
2.45.2
PATCH v8 1/9 fsck: rename "skiplist" to "oid_skiplist"
In reply to shejialuoThe "skiplist" field in "fsck_options" is related to objects. Because we are going to introduce ref consistency check, the "skiplist" name is too general which will make the caller think "skiplist" is related to both the refs and objects.
It may seem that for both refs and objects, we should provide a general "skiplist" here. However, the type for "skiplist" is struct oidset which is totally unsuitable for refs.
To avoid above ambiguity, rename "skiplist" to "oid_skiplist".
--- fsck.c | 4 ++-- fsck.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-)
fsck.c +2 -2
@@ -205,7 +205,7 @@ void fsck_set_msg_types(struct fsck_options *options, const char *values)if (!strcmp (buf,"skiplist" )) {if (equal == len)die ("skiplist requires a path" ); -oidset_parse_file (&options->skiplist, buf + equal +1 , +oidset_parse_file (&options->oid_skiplist, buf + equal +1 , the_repository->hash_algo); buf += len +1 ;continue ;
@@ -223,7 +223,7 @@ void fsck_set_msg_types(struct fsck_options *options, const char *values)static int object_on_skiplist (struct fsck_options *opts,const struct object_id *oid) { -return opts && oid &&oidset_contains (&opts->skiplist, oid); +return opts && oid &&oidset_contains (&opts->oid_skiplist, oid); } __attribute__((format (printf, 5, 6)))
fsck.h +1 -1
@@ -136,7 +136,7 @@ struct fsck_options {fsck_error error_func; unsigned strict:1;enum fsck_msg_type *msg_type; -struct oidset skiplist; +struct oidset oid_skiplist;struct oidset gitmodules_found;struct oidset gitmodules_done;struct oidset gitattributes_found;
Signature · 1 line
2.45.2
PATCH v8 2/9 fsck: add a unified interface for reporting fsck messages
In reply to shejialuoThe static function "report" provided by "fsck.c" aims at checking fsck error type and calling the callback "error_func" to report the message. However, "report" function is only related to object database which cannot be reused for refs. In order to provide a unified interface which can report either objects or refs, create a new function "vfsck_report" by adding "checked_ref_name" parameter following the "report" prototype. Instead of using "...", provide "va_list" to allow more flexibility.
Like "report", the "vfsck_report" function will use "error_func" registered in "fsck_options" to report customized messages. Change "error_func" prototype to align with the new "vfsck_report".
Then, change "report" function to use "vfsck_report" to report objects related messages. Add a new function called "fsck_refs_report" to use "vfsck_report" to report refs related messages.
--- builtin/fsck.c | 15 ++++----- builtin/mktag.c | 1 + fsck.c | 81 ++++++++++++++++++++++++++++++++++++------------- fsck.h | 42 ++++++++++++++++--------- object-file.c | 11 ++++--- 5 files changed, 102 insertions(+), 48 deletions(-)
builtin/fsck.c +8 -7
@@ -89,12 +89,13 @@ static int objerror(struct object *obj, const char *err)return -1 ; } -static int fsck_error_func(struct fsck_options *o UNUSED, -const struct object_id *oid, -enum object_type object_type, -enum fsck_msg_type msg_type, - enum fsck_msg_id msg_id UNUSED, -const char *message) +static int fsck_objects_error_func(struct fsck_options *o UNUSED, +const struct object_id *oid, +enum object_type object_type, + const char *checked_ref_name UNUSED, +enum fsck_msg_type msg_type, + enum fsck_msg_id msg_id UNUSED, +const char *message) {switch (msg_type) {case FSCK_WARN :
@@ -938,7 +939,7 @@ int cmd_fsck(int argc, const char **argv, const char *prefix) fsck_walk_options.walk = mark_object; fsck_obj_options.walk = mark_used; - fsck_obj_options.error_func = fsck_error_func; + fsck_obj_options.error_func = fsck_objects_error_func;if (check_strict) fsck_obj_options.strict =1 ;
builtin/mktag.c +1 -0
@@ -20,6 +20,7 @@ static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT; static int mktag_fsck_error_func(struct fsck_options *o UNUSED, const struct object_id *oid UNUSED, enum object_type object_type UNUSED, + const char *checked_ref_name UNUSED,enum fsck_msg_type msg_type, enum fsck_msg_id msg_id UNUSED,const char *message)
fsck.c +60 -21
115 lines not on this page · raw message
fsck.h +27 -15
81 lines not on this page · raw message
object-file.c +6 -5
@@ -2470,11 +2470,12 @@ int repo_has_object_file(struct repository *r,* give more context. */ static int hash_format_check_report(struct fsck_options *opts UNUSED, - const struct object_id *oid UNUSED, - enum object_type object_type UNUSED, - enum fsck_msg_type msg_type UNUSED, - enum fsck_msg_id msg_id UNUSED, -const char *message) + const struct object_id *oid UNUSED, + enum object_type object_type UNUSED, + const char *ref_checked_name UNUSED, + enum fsck_msg_type msg_type UNUSED, + enum fsck_msg_id msg_id UNUSED, +const char *message) {error (_ ("object fails fsck: %s" ), message);return 1 ;
Signature · 1 line
2.45.2
Re: [GSoC][PATCH v8 2/9] fsck: add a unified interface for reporting fsck messages
Quoted 8 lines · shejialuo
The static function "report" provided by "
fsck.c" aims at checking fsck error type and calling the callback "error_func" to report the message. However, "report" function is only related to object database which cannot be reused for refs. In order to provide a unified interface which can report either objects or refs, create a new function "vfsck_report" by adding "checked_ref_name" parameter following the "report" prototype. Instead of using "...", provide "va_list" to allow more flexibility.
Quoted 8 lines
Like "report", the "
vfsck_report" function will use "error_func" registered in "fsck_options" to report customized messages. Change "error_func" prototype to align with the new "vfsck_report".Then, change "report" function to use "
vfsck_report" to report objects related messages. Add a new function called "fsck_refs_report" to use "vfsck_report" to report refs related messages.
Not sure I really understand why we need to do this. Why can't we simply add const char *checked_ref_name to the existing 'report' and propagate this also to 'error_func'. Why do we need all this parallel flows?
Apart from that, what does 'v' in 'vfsck_report' signify?
Perhaps it is also because this commit is doing a lot of things and we could have simplified it into smaller commits?
Quoted 98 lines
Mentored-by Patrick Steinhardt <[email protected]>Mentored-by Karthik Nayak <[email protected]>Signed-off-by shejialuo <[email protected]>--- builtin/fsck.c | 15 ++++----- builtin/mktag.c | 1 + fsck.c | 81 ++++++++++++++++++++++++++++++++++++------------- fsck.h | 42 ++++++++++++++++--------- object-file.c | 11 ++++--- 5 files changed, 102 insertions(+), 48 deletions(-)builtin/fsck.c +8
-7@@ -89,12 +89,13 @@ static int objerror(struct object *obj, const char *err)return -1 ; } -static int fsck_error_func(struct fsck_options *o UNUSED, -const struct object_id *oid, -enum object_type object_type, -enum fsck_msg_type msg_type, - enum fsck_msg_id msg_id UNUSED, -const char *message) +static int fsck_objects_error_func(struct fsck_options *o UNUSED, +const struct object_id *oid, +enum object_type object_type, + const char *checked_ref_name UNUSED, +enum fsck_msg_type msg_type, + enum fsck_msg_id msg_id UNUSED, +const char *message) {switch (msg_type) {case FSCK_WARN :@@ -938,7 +939,7 @@ int cmd_fsck(int argc, const char **argv, const char *prefix) fsck_walk_options.walk = mark_object; fsck_obj_options.walk = mark_used; - fsck_obj_options.error_func = fsck_error_func; + fsck_obj_options.error_func = fsck_objects_error_func;if (check_strict) fsck_obj_options.strict =1 ;builtin/mktag.c +1
-0@@ -20,6 +20,7 @@ static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT; static int mktag_fsck_error_func(struct fsck_options *o UNUSED, const struct object_id *oid UNUSED, enum object_type object_type UNUSED, + const char *checked_ref_name UNUSED,enum fsck_msg_type msg_type, enum fsck_msg_id msg_id UNUSED,const char *message)fsck.c +14
-8@@ -226,12 +226,18 @@ static int object_on_skiplist(struct fsck_options *opts,return opts && oid &&oidset_contains (&opts->oid_skiplist, oid); } -__attribute__((format (printf,5 ,6 ))) -static int report (struct fsck_options *options, -const struct object_id *oid,enum object_type object_type, -enum fsck_msg_id msg_id,const char *fmt, ...) +/* +* Provide a unified interface for either fscking refs or objects. +* It will get the current msg error type and call the error_func callback +* which is registered in the "fsck_options" struct. +*/ +static int vfsck_report (struct fsck_options *options, +const struct object_id *oid, +enum object_type object_type, +const char *checked_ref_name, +enum fsck_msg_id msg_id,const char *fmt,va_list ap) { -va_list ap; +va_list ap_copy;struct strbuf sb =STRBUF_INIT ;enum fsck_msg_type msg_type =fsck_msg_type (msg_id, options);int result;@@ -250,9 +256,9 @@ static int report(struct fsck_options *options,prepare_msg_ids ();strbuf_addf (&sb,"%s: " , msg_id_info[msg_id].camelcased); -va_start (ap, fmt); -strbuf_vaddf (&sb, fmt, ap); - result = options->error_func (options, oid, object_type, +va_copy (ap_copy, ap); +strbuf_vaddf (&sb, fmt, ap_copy); + result = options->error_func (options, oid, object_type, checked_ref_name, msg_type, msg_id, sb.buf);strbuf_release (&sb);va_end (ap);
fsck.c static int report(struct fsck_options *options, · line 270
Quoted 5 lines
fsck.c +1
-0@@ -260,6 +266,36 @@ static int report(struct fsck_options *options,return result; } +__attribute__((format (printf, 5, 6)))
Shouldn't this be moved to the header file too?
Quoted 5 lines
fsck.c +5
-0+static int report (struct fsck_options *options, +const struct object_id *oid,enum object_type object_type, +enum fsck_msg_id msg_id,const char *fmt, ...) +{ +va_list ap;
fsck.c static int report(struct fsck_options *options, · line 284
Quoted 10 lines
fsck.c +10
-0+int result; +va_start (ap, fmt); + result =vfsck_report (options, oid, object_type,NULL , + msg_id, fmt, ap); +va_end (ap); +return result; +} + + +
There is an extra newline here.
Quoted 177 lines
fsck.c +14
-0+int fsck_refs_report (struct fsck_options *options, +const struct object_id *oid, +const char *checked_ref_name, +enum fsck_msg_id msg_id,const char *fmt, ...) +{ +va_list ap; +int result; +va_start (ap, fmt); + result =vfsck_report (options, oid,OBJ_NONE , + checked_ref_name, msg_id, fmt, ap); +va_end (ap); +return result; +} +void fsck_enable_object_names (struct fsck_options *options) {fsck.c +16
-13if (!options->object_names)@@ -1200,12 +1236,13 @@ int fsck_buffer(const struct object_id *oid, enum object_type type, type); } -int fsck_error_function (struct fsck_options *o, -const struct object_id *oid, - enum object_type object_type UNUSED, -enum fsck_msg_type msg_type, - enum fsck_msg_id msg_id UNUSED, -const char *message) +int fsck_objects_error_function (struct fsck_options *o, +const struct object_id *oid, + enum object_type object_type UNUSED, + const char *checked_ref_name UNUSED, +enum fsck_msg_type msg_type, + enum fsck_msg_id msg_id UNUSED, +const char *message) {if (msg_type ==FSCK_WARN ) {warning ("object %s: %s" ,fsck_describe_object (o, oid), message);@@ -1303,16 +1340,18 @@ int git_fsck_config(const char *var, const char *value,* Custom error callbacks that are used in more than one place. */ -int fsck_error_cb_print_missing_gitmodules (struct fsck_options *o, -const struct object_id *oid, -enum object_type object_type, -enum fsck_msg_type msg_type, -enum fsck_msg_id msg_id, -const char *message) +int fsck_objects_error_cb_print_missing_gitmodules (struct fsck_options *o, +const struct object_id *oid, +enum object_type object_type, +const char *checked_ref_name, +enum fsck_msg_type msg_type, +enum fsck_msg_id msg_id, +const char *message) {if (msg_id ==FSCK_MSG_GITMODULES_MISSING ) {puts (oid_to_hex (oid));return 0 ; } -return fsck_error_function (o, oid, object_type, msg_type, msg_id, message); +return fsck_objects_error_function (o, oid, object_type, checked_ref_name, + msg_type, msg_id, message); }fsck.h +27
-15@@ -114,22 +114,27 @@ int is_valid_msg_type(const char *msg_id, const char *msg_type);typedef int (*fsck_walk_func )(struct object *obj,enum object_type object_type,void *data,struct fsck_options *options); -/* callback for fsck_object, type is FSCK_ERROR or FSCK_WARN */ +/* +* callback function for reporting errors when checking either objects or refs +*/ typedef int (*fsck_error )(struct fsck_options *o,const struct object_id *oid,enum object_type object_type, +const char *checked_ref_name,enum fsck_msg_type msg_type,enum fsck_msg_id msg_id,const char *message); -int fsck_error_function (struct fsck_options *o, -const struct object_id *oid,enum object_type object_type, -enum fsck_msg_type msg_type,enum fsck_msg_id msg_id, -const char *message); -int fsck_error_cb_print_missing_gitmodules (struct fsck_options *o, -const struct object_id *oid, -enum object_type object_type, -enum fsck_msg_type msg_type, -enum fsck_msg_id msg_id, -const char *message); +int fsck_objects_error_function (struct fsck_options *o, +const struct object_id *oid,enum object_type object_type, +const char *checked_ref_name, +enum fsck_msg_type msg_type,enum fsck_msg_id msg_id, +const char *message); +int fsck_objects_error_cb_print_missing_gitmodules (struct fsck_options *o, +const struct object_id *oid, +enum object_type object_type, +const char *checked_ref_name, +enum fsck_msg_type msg_type, +enum fsck_msg_id msg_id, +const char *message);struct fsck_options {fsck_walk_func walk;@@ -145,12 +150,12 @@ struct fsck_options { };#define FSCK_OPTIONS_DEFAULT { \ - .skiplist = OIDSET_INIT, \ + .oid_skiplist = OIDSET_INIT, \ .gitmodules_found = OIDSET_INIT, \ .gitmodules_done = OIDSET_INIT, \ .gitattributes_found = OIDSET_INIT, \ .gitattributes_done = OIDSET_INIT, \ - .error_func = fsck_error_function \ + .error_func = fsck_objects_error_function \ }#define FSCK_OPTIONS_STRICT { \ .strict = 1, \@@ -158,7 +163,7 @@ struct fsck_options { .gitmodules_done = OIDSET_INIT, \ .gitattributes_found = OIDSET_INIT, \ .gitattributes_done = OIDSET_INIT, \ - .error_func = fsck_error_function, \ + .error_func = fsck_objects_error_function, \ }#define FSCK_OPTIONS_MISSING_GITMODULES { \ .strict = 1, \@@ -166,7 +171,7 @@ struct fsck_options { .gitmodules_done = OIDSET_INIT, \ .gitattributes_found = OIDSET_INIT, \ .gitattributes_done = OIDSET_INIT, \ - .error_func = fsck_error_cb_print_missing_gitmodules, \ + .error_func = fsck_objects_error_cb_print_missing_gitmodules, \ }/* descend in all linked child objects @@ -209,6 +214,13 @@ int fsck_tag_standalone(const struct object_id *oid, const char *buffer,*/ int fsck_finish (struct fsck_options *options); +__attribute__((format (printf,5 ,6 ))) +int fsck_refs_report (struct fsck_options *options, +const struct object_id *oid, +const char *checked_ref_name, +enum fsck_msg_id msg_id, +const char *fmt, ...); +/* * Subsystem for storing human-readable names for each object. * object-file.c +6
-6@@ -2470,11 +2470,12 @@ int repo_has_object_file(struct repository *r,* give more context. */ static int hash_format_check_report(struct fsck_options *opts UNUSED, - const struct object_id *oid UNUSED, - enum object_type object_type UNUSED, - enum fsck_msg_type msg_type UNUSED, - enum fsck_msg_id msg_id UNUSED, -const char *message) + const struct object_id *oid UNUSED, + enum object_type object_type UNUSED, + const char *ref_checked_name UNUSED, + enum fsck_msg_type msg_type UNUSED, + enum fsck_msg_id msg_id UNUSED, +const char *message) {error (_ ("object fails fsck: %s" ), message);return 1 ; --2.45.2
Re: [GSoC][PATCH v8 2/9] fsck: add a unified interface for reporting fsck messages
Quoted 19 lines · Karthik Nayak
Quoted 8 lines · shejialuo
The static function "report" provided by "
fsck.c" aims at checking fsck error type and calling the callback "error_func" to report the message. However, "report" function is only related to object database which cannot be reused for refs. In order to provide a unified interface which can report either objects or refs, create a new function "vfsck_report" by adding "checked_ref_name" parameter following the "report" prototype. Instead of using "...", provide "va_list" to allow more flexibility.Quoted 8 lines
Like "report", the "
vfsck_report" function will use "error_func" registered in "fsck_options" to report customized messages. Change "error_func" prototype to align with the new "vfsck_report".Then, change "report" function to use "
vfsck_report" to report objects related messages. Add a new function called "fsck_refs_report" to use "vfsck_report" to report refs related messages.
Quoted 5 lines
Not sure I really understand why we need to do this. Why can't we simply add
const char *checked_ref_nameto the existing 'report' and propagate this also to 'error_func'. Why do we need all this parallel flows?
Yes, we could just add a parameter "const char *checked_ref_name" to the existing "report". This may seem the simplest way to do. However, it will also introduce some trouble below:
1. "report" function should be exported to the outside, we need to rename it to "fsck_report". Well, we need to change a lot of code here. And we MUST do this, because "report" is a general name. When exporting to the outside, it's not proper.
2. When we add a new parameter in "report", for all the "report" calls, we need to pass this new parameter with NULL.
Use this way, we could do not change "report" function prototype and the corresponding calls. Most importantly, we could let the caller feel transparent. Using "report", caller can just ignore "checked_ref_name". Also for "fsck_refs_report", we could ignore some UNUSED parameters.
So I think this design is more elegant than just adding a new parameter in the existing "report" function.
Quoted 2 lines
Apart from that, what does 'v' in '
vfsck_report' signify?
Because I use "va_list" parameter, I want to emphasis on this. And this provides flexibility that we could add a "fsck_report" function later. There are many codes in git code base using this way. I just followed this.
Quoted 3 lines
Perhaps it is also because this commit is doing a lot of things and we could have simplified it into smaller commits?
Actually, this commit is very clear. I just want to provide a unified function "vfsck_report" here. And let the "report" use this function and "fsck_refs_report" function use this.
So I don't know whether we should split this commit into multiple commits. They are just tied together.
Re: [GSoC][PATCH v8 2/9] fsck: add a unified interface for reporting fsck messages
Quoted 26 lines · shejialuo
Quoted 19 lines · Karthik Nayak
Quoted 8 lines · shejialuo
The static function "report" provided by "
fsck.c" aims at checking fsck error type and calling the callback "error_func" to report the message. However, "report" function is only related to object database which cannot be reused for refs. In order to provide a unified interface which can report either objects or refs, create a new function "vfsck_report" by adding "checked_ref_name" parameter following the "report" prototype. Instead of using "...", provide "va_list" to allow more flexibility.Quoted 8 lines
Like "report", the "
vfsck_report" function will use "error_func" registered in "fsck_options" to report customized messages. Change "error_func" prototype to align with the new "vfsck_report".Then, change "report" function to use "
vfsck_report" to report objects related messages. Add a new function called "fsck_refs_report" to use "vfsck_report" to report refs related messages.Quoted 5 lines
Not sure I really understand why we need to do this. Why can't we simply add
const char *checked_ref_nameto the existing 'report' and propagate this also to 'error_func'. Why do we need all this parallel flows?
Quoted 9 lines
Yes, we could just add a parameter "const char
*checked_ref_name" to the existing "report". This may seem the simplest way to do. However, it will also introduce some trouble below:1. "report" function should be exported to the outside, we need to rename it to "
fsck_report". Well, we need to change a lot of code here. And we MUST do this, because "report" is a general name. When exporting to the outside, it's not proper.
agreed.
Quoted 3 lines
2. When we add a new parameter in "report", for all the "report" calls, we need to pass this new parameter with NULL.
agreed too.
Quoted 8 lines
Use this way, we could do not change "report" function prototype and the corresponding calls. Most importantly, we could let the caller feel transparent. Using "report", caller can just ignore "
checked_ref_name". Also for "fsck_refs_report", we could ignore some UNUSED parameters.So I think this design is more elegant than just adding a new parameter in the existing "report" function.
I understand what you're saying, I also checked and can see that there are 60 references to the report() function. So perhaps there is some merit in keeping it as is and adding a new 'report_refs()'.
Quoted 8 lines
Quoted 2 lines
Apart from that, what does 'v' in '
vfsck_report' signify?Because I use "
va_list" parameter, I want to emphasis on this. And this provides flexibility that we could add a "fsck_report" function later. There are many codes in git code base using this way. I just followed this.
I see. Makes sense.
Quoted 10 lines
Quoted 3 lines
Perhaps it is also because this commit is doing a lot of things and we could have simplified it into smaller commits?
Actually, this commit is very clear. I just want to provide a unified function "
vfsck_report" here. And let the "report" use this function and "fsck_refs_report" function use this.So I don't know whether we should split this commit into multiple commits. They are just tied together.
PATCH v8 3/9 fsck: add refs-related options and error report function
In reply to shejialuoAdd refs-related options to the "fsck_options", create refs-specific "error_func" callback "fsck_refs_error_function".
"fsck_refs_error_function" will use the "oid" parameter. When the caller passes the oid, it will use "oid_to_hex" to get the corresponding hex value to report to the caller.
Last, add "FSCK_REFS_OPTIONS_DEFAULT" and "FSCK_REFS_OPTIONS_STRICT" macros to create refs options easily.
--- fsck.c | 22 ++++++++++++++++++++++ fsck.h | 15 +++++++++++++++ 2 files changed, 37 insertions(+)
fsck.c +22 -0
@@ -1252,6 +1252,28 @@ int fsck_objects_error_function(struct fsck_options *o,return 1 ; } +int fsck_refs_error_function(struct fsck_options *options UNUSED, +const struct object_id *oid, + enum object_type object_type UNUSED, +const char *checked_ref_name, +enum fsck_msg_type msg_type, + enum fsck_msg_id msg_id UNUSED, +const char *message) +{ +static struct strbuf sb =STRBUF_INIT ; + +strbuf_addstr (&sb, checked_ref_name); +if (oid) +strbuf_addf (&sb," -> (%s)" ,oid_to_hex (oid)); + +if (msg_type ==FSCK_WARN ) { +warning ("%s: %s" , sb.buf, message); +return 0 ; + } +error ("%s: %s" , sb.buf, message); +return 1 ; +} +static int fsck_blobs (struct oidset *blobs_found,struct oidset *blobs_done,enum fsck_msg_id msg_missing,enum fsck_msg_id msg_type,struct fsck_options *options,const char *blob_type)
fsck.h +15 -0
@@ -135,11 +135,19 @@ int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o,enum fsck_msg_type msg_type, enum fsck_msg_id msg_id, const char *message); +int fsck_refs_error_function (struct fsck_options *options, +const struct object_id *oid, +enum object_type object_type, +const char *checked_ref_name, +enum fsck_msg_type msg_type, +enum fsck_msg_id msg_id, +const char *message);struct fsck_options {fsck_walk_func walk;fsck_error error_func;unsigned strict:1 ; +unsigned verbose_refs:1 ;enum fsck_msg_type *msg_type;struct oidset oid_skiplist;struct oidset gitmodules_found;
@@ -173,6 +181,13 @@ struct fsck_options { .gitattributes_done = OIDSET_INIT, \ .error_func = fsck_objects_error_cb_print_missing_gitmodules, \ } +#define FSCK_REFS_OPTIONS_DEFAULT { \ + .error_func = fsck_refs_error_function, \ +} +#define FSCK_REFS_OPTIONS_STRICT { \ + .strict = 1, \ + .error_func = fsck_refs_error_function, \ +}/* descend in all linked child objects * the return value is:
Signature · 1 line
2.45.2
Re: [GSoC][PATCH v8 3/9] fsck: add refs-related options and error report function
Quoted 10 lines · shejialuo
Add refs-related options to the "
fsck_options", create refs-specific "error_func" callback "fsck_refs_error_function"."
fsck_refs_error_function" will use the "oid" parameter. When the caller passes the oid, it will use "oid_to_hex" to get the corresponding hex value to report to the caller.Last, add "
FSCK_REFS_OPTIONS_DEFAULT" and "FSCK_REFS_OPTIONS_STRICT" macros to create refs options easily.
Carrying over from the previous commit, couldn't we simply do something like:
diff --git a/fsck.c b/fsck.c
index eea7145470..32ae36a4fc 100644
--- a/fsck.c
+++ b/fsck.c
@@ -1202,17 +1203,33 @@ int fsck_buffer(const struct object_id*oid, enum object_type type,
int fsck_error_function(struct fsck_options *o,
const struct object_id *oid,
- enum object_type object_type UNUSED,
+ enum object_type object_type,
+ const char *checked_ref_name,
enum fsck_msg_type msg_type,
enum fsck_msg_id msg_id UNUSED,
const char *message)
{
+ static struct strbuf sb = STRBUF_INIT;
+ int ret = 0;
+
+ if (checked_ref_name) {
+ strbuf_addstr("ref %s", checked_ref_name);
+ if (oid)
+ strbuf_addf(&sb, " -> (%s)", oid_to_hex(oid));
+ } else {
+ strbuf_addstr("object %s", fsck_describe_object(o, oid));
+ }
+
if (msg_type == FSCK_WARN) {
- warning("object %s: %s", fsck_describe_object(o, oid), message);
- return 0;
+ warning("%s: %s", sb.buf, message);
+ ret = 0;
+ goto cleanup;
}
- error("object %s: %s", fsck_describe_object(o, oid), message);
- return 1;
+ error("%s: %s", sb.buf, message);
+
+cleanup:
+ strbuf_release(&sb);
+ return ret;
}Quoted 28 lines
Mentored-by Patrick Steinhardt <[email protected]>Mentored-by Karthik Nayak <[email protected]>Signed-off-by shejialuo <[email protected]>--- fsck.c | 22 ++++++++++++++++++++++ fsck.h | 15 +++++++++++++++ 2 files changed, 37 insertions(+)fsck.c +12
-0@@ -1252,6 +1252,28 @@ int fsck_objects_error_function(struct fsck_options *o,return 1 ; } +int fsck_refs_error_function(struct fsck_options *options UNUSED, +const struct object_id *oid, + enum object_type object_type UNUSED, +const char *checked_ref_name, +enum fsck_msg_type msg_type, + enum fsck_msg_id msg_id UNUSED, +const char *message) +{ +static struct strbuf sb =STRBUF_INIT ; + +strbuf_addstr (&sb, checked_ref_name); +if (oid)
fsck.c int fsck_objects_error_function(struct fsck_options *o, · line 1276
Quoted 10 lines
fsck.c +10
-0+strbuf_addf (&sb," -> (%s)" ,oid_to_hex (oid)); + +if (msg_type ==FSCK_WARN ) { +warning ("%s: %s" , sb.buf, message); +return 0 ; + } +error ("%s: %s" , sb.buf, message); +return 1 ; +} +
We don't free strbuf here.
Quoted 14 lines
fsck.c +0
-0static int fsck_blobs (struct oidset *blobs_found,struct oidset *blobs_done,enum fsck_msg_id msg_missing,enum fsck_msg_id msg_type,struct fsck_options *options,const char *blob_type)fsck.h +3
-0@@ -135,11 +135,19 @@ int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o,enum fsck_msg_type msg_type, enum fsck_msg_id msg_id, const char *message); +int fsck_refs_error_function(struct fsck_options *options, + const struct object_id *oid, + enum object_type object_type,
fsck.h int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o, · line 150
Quoted 10 lines
fsck.h +5
-0+const char *checked_ref_name, + enum fsck_msg_type msg_type, + enum fsck_msg_id msg_id, + const char *message);struct fsck_options {fsck_walk_func walk;fsck_error error_func;unsigned strict:1 ; +unsigned verbose_refs:1 ;
Nit: Here we have the subject 'refs' towards the end of the name.
fsck.h int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o, · line 152
Quoted 2 lines
fsck.h +0
-0enum fsck_msg_type *msg_type;struct oidset oid_skiplist;
but here we have the subject 'oid' at the start of the name. Perhaps we can rename this to 'skip_oids'?
Quoted 17 lines
fsck.h +7
-1struct oidset gitmodules_found;@@ -173,6 +181,13 @@ struct fsck_options { .gitattributes_done = OIDSET_INIT, \ .error_func = fsck_objects_error_cb_print_missing_gitmodules, \ } +#define FSCK_REFS_OPTIONS_DEFAULT { \ + .error_func = fsck_refs_error_function, \ +} +#define FSCK_REFS_OPTIONS_STRICT { \ + .strict = 1, \ + .error_func = fsck_refs_error_function, \ +}/* descend in all linked child objects * the return value is: -- 2.45.2
Re: [GSoC][PATCH v8 3/9] fsck: add refs-related options and error report function
Quoted 50 lines · Karthik Nayak
Quoted 10 lines · shejialuo
Add refs-related options to the "
fsck_options", create refs-specific "error_func" callback "fsck_refs_error_function"."
fsck_refs_error_function" will use the "oid" parameter. When the caller passes the oid, it will use "oid_to_hex" to get the corresponding hex value to report to the caller.Last, add "
FSCK_REFS_OPTIONS_DEFAULT" and "FSCK_REFS_OPTIONS_STRICT" macros to create refs options easily.Carrying over from the previous commit, couldn't we simply do something like:
diff --git a/fsck.c b/fsck.c index eea7145470..32ae36a4fc 100644 --- a/fsck.c +++ b/fsck.c @@ -1202,17 +1203,33 @@ int fsck_buffer(const struct object_id*oid, enum
object_typetype,int fsck_error_function(struct fsck_options *o, const struct object_id *oid, - enum object_type object_type UNUSED, + enum object_type object_type, + const char *checked_ref_name, enum fsck_msg_type msg_type, enum fsck_msg_id msg_id UNUSED, const char *message) { + static struct strbuf sb = STRBUF_INIT; + int ret = 0; + + if (checked_ref_name) { + strbuf_addstr("ref %s", checked_ref_name); + if (oid) + strbuf_addf(&sb, " -> (%s)", oid_to_hex(oid)); + } else { + strbuf_addstr("object %s", fsck_describe_object(o, oid)); + } + if (msg_type == FSCK_WARN) { - warning("object %s: %s", fsck_describe_object(o, oid), message); - return 0; + warning("%s: %s", sb.buf, message); + ret = 0; + goto cleanup; }
Quoted 10 lines
- error("object %s: %s", fsck_describe_object(o, oid), message);
- return 1;
+ error("%s: %s", sb.buf, message);
+
+cleanup:
+ strbuf_release(&sb);
+ return ret;
}However, "fsck_error_function" will be used as the callback for object-related checks. I don't think we should use one function to incorporate all the situations.
For example, if we pass both "checked_ref_name" and "oid" here, how does this one function knows whether we handle refs or objects. Although we may use some flags here to choose the different situations, it's bad idea from my perspective. Adding a new callback here will be cleaner.
Quoted 32 lines
Quoted 32 lines
Mentored-by Patrick Steinhardt <[email protected]>Mentored-by Karthik Nayak <[email protected]>Signed-off-by shejialuo <[email protected]>--- fsck.c | 22 ++++++++++++++++++++++ fsck.h | 15 +++++++++++++++ 2 files changed, 37 insertions(+)fsck.c +16
-0@@ -1252,6 +1252,28 @@ int fsck_objects_error_function(struct fsck_options *o,return 1 ; } +int fsck_refs_error_function(struct fsck_options *options UNUSED, +const struct object_id *oid, + enum object_type object_type UNUSED, +const char *checked_ref_name, +enum fsck_msg_type msg_type, + enum fsck_msg_id msg_id UNUSED, +const char *message) +{ +static struct strbuf sb =STRBUF_INIT ; + +strbuf_addstr (&sb, checked_ref_name); +if (oid) +strbuf_addf (&sb," -> (%s)" ,oid_to_hex (oid)); + +if (msg_type ==FSCK_WARN ) { +warning ("%s: %s" , sb.buf, message);
Quoted 9 lines
fsck.c int fsck_objects_error_function(struct fsck_options *o, · line 1276
Quoted 6 lines
fsck.c +6
-0+return 0 ; + } +error ("%s: %s" , sb.buf, message); +return 1 ; +} +We don't free strbuf here.
Yes, I made a mistake here. I should not define the "sb" static, will fix it in the next version.
Quoted 27 lines
Quoted 14 lines
fsck.c +0
-0static int fsck_blobs (struct oidset *blobs_found,struct oidset *blobs_done,enum fsck_msg_id msg_missing,enum fsck_msg_id msg_type,struct fsck_options *options,const char *blob_type)fsck.h +3
-0@@ -135,11 +135,19 @@ int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o,enum fsck_msg_type msg_type, enum fsck_msg_id msg_id, const char *message); +int fsck_refs_error_function(struct fsck_options *options, + const struct object_id *oid, + enum object_type object_type,fsck.h int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o, · line 150
Quoted 10 lines
fsck.h +5
-0+const char *checked_ref_name, + enum fsck_msg_type msg_type, + enum fsck_msg_id msg_id, + const char *message);struct fsck_options {fsck_walk_func walk;fsck_error error_func;unsigned strict:1 ; +unsigned verbose_refs:1 ;Nit: Here we have the subject 'refs' towards the end of the name.
Quoted 6 lines
fsck.h int fsck_objects_error_cb_print_missing_gitmodules(struct fsck_options *o, · line 152
Quoted 2 lines
fsck.h +0
-0enum fsck_msg_type *msg_type;struct oidset oid_skiplist;but here we have the subject 'oid' at the start of the name. Perhaps we can rename this to '
skip_oids'?
I agree. "skip_oids" will be more meaningful. I will fix it in the next version.
Quoted 17 lines
Quoted 17 lines
fsck.h +7
-1struct oidset gitmodules_found;@@ -173,6 +181,13 @@ struct fsck_options { .gitattributes_done = OIDSET_INIT, \ .error_func = fsck_objects_error_cb_print_missing_gitmodules, \ } +#define FSCK_REFS_OPTIONS_DEFAULT { \ + .error_func = fsck_refs_error_function, \ +} +#define FSCK_REFS_OPTIONS_STRICT { \ + .strict = 1, \ + .error_func = fsck_refs_error_function, \ +}/* descend in all linked child objects * the return value is: -- 2.45.2
PATCH v8 4/9 refs: set up ref consistency check infrastructure
In reply to shejialuoThe interfaces defined in the ref_storage_be are carefully structured in semantic. It's organized as the five parts:
1. The name and the initialization interfaces.
2. The ref transaction interfaces.
3. The ref internal interfaces (pack, rename and copy).
4. The ref filesystem interfaces.
5. The reflog related interfaces.
To keep consistent with the git-fsck(1), add a new interface named "fsck_refs_fn" to the end of "ref_storage_be". This semantic cannot be grouped into any above five categories. Explicitly add blank line to make it different from others.
--- refs.c | 5 +++++ refs.h | 8 ++++++++ refs/debug.c | 11 +++++++++++ refs/files-backend.c | 15 ++++++++++++++- refs/packed-backend.c | 8 ++++++++ refs/refs-internal.h | 6 ++++++ refs/reftable-backend.c | 8 ++++++++ 7 files changed, 60 insertions(+), 1 deletion(-)
refs.c +5 -0
@@ -318,6 +318,11 @@ int check_refname_format(const char *refname, int flags)return check_or_sanitize_refname (refname, flags,NULL ); } +int refs_fsck (struct ref_store *refs,struct fsck_options *o) +{ +return refs->be->fsck (refs, o); +} +void sanitize_refname_component (const char *refname,struct strbuf *out) {if (check_or_sanitize_refname (refname,REFNAME_ALLOW_ONELEVEL , out))
refs.h +8 -0
@@ -4,6 +4,7 @@#include "commit.h" #include "repository.h" +struct fsck_options ;struct object_id ;struct ref_store ;struct strbuf ;
@@ -541,6 +542,13 @@ int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_dat*/ int check_refname_format (const char *refname,int flags); +/* +* Check the reference database for consistency. Return 0 if refs and +* reflogs are consistent, and non-zero otherwise. The errors will be +* written to stderr. +*/ +int refs_fsck (struct ref_store *refs,struct fsck_options *o); +/* * Apply the rules from check_refname_format, but mutate the result until it * is acceptable, and place the result in "out".
refs/debug.c +11 -0
@@ -419,6 +419,15 @@ static int debug_reflog_expire(struct ref_store *ref_store, const char *refname,return res; } +static int debug_fsck (struct ref_store *ref_store, +struct fsck_options *o) +{ +struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store; +int res = drefs->refs->be->fsck (drefs->refs, o); +trace_printf_key (&trace_refs,"fsck: %d\n" , res); +return res; +} +struct ref_storage_be refs_be_debug = { .name ="debug" , .init =NULL ,
@@ -451,4 +460,6 @@ struct ref_storage_be refs_be_debug = { .create_reflog = debug_create_reflog, .delete_reflog = debug_delete_reflog, .reflog_expire = debug_reflog_expire, + + .fsck = debug_fsck, };
refs/files-backend.c +14 -1
@@ -3408,6 +3408,17 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,return ret; } +static int files_fsck (struct ref_store *ref_store, +struct fsck_options *o) +{ +int ret; +struct files_ref_store *refs = +files_downcast (ref_store,REF_STORE_READ ,"fsck" ); + + ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o); +return ret; +} +struct ref_storage_be refs_be_files = { .name ="files" , .init = files_ref_store_init,
@@ -3434,5 +3445,7 @@ struct ref_storage_be refs_be_files = { .reflog_exists = files_reflog_exists, .create_reflog = files_create_reflog, .delete_reflog = files_delete_reflog, - .reflog_expire = files_reflog_expire + .reflog_expire = files_reflog_expire, + + .fsck = files_fsck, };
refs/packed-backend.c +8 -0
@@ -1735,6 +1735,12 @@ static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_sreturn empty_ref_iterator_begin (); } +static int packed_fsck (struct ref_store *ref_store, +struct fsck_options *o) +{ +return 0 ; +} +struct ref_storage_be refs_be_packed = { .name ="packed" , .init = packed_ref_store_init,
@@ -1762,4 +1768,6 @@ struct ref_storage_be refs_be_packed = { .create_reflog = NULL, .delete_reflog = NULL, .reflog_expire = NULL, + + .fsck = packed_fsck, };
refs/refs-internal.h +6 -0
@@ -4,6 +4,7 @@#include "refs.h" #include "iterator.h" +struct fsck_options ;struct ref_transaction ;/*
@@ -650,6 +651,9 @@ typedef int read_raw_ref_fn(struct ref_store *ref_store, const char *refname,typedef int read_symbolic_ref_fn (struct ref_store *ref_store,const char *refname,struct strbuf *referent); +typedef int fsck_fn (struct ref_store *ref_store, +struct fsck_options *o); +struct ref_storage_be {const char *name;ref_store_init_fn *init;
@@ -677,6 +681,8 @@ struct ref_storage_be {create_reflog_fn *create_reflog;delete_reflog_fn *delete_reflog;reflog_expire_fn *reflog_expire; + +fsck_fn *fsck; };extern struct ref_storage_be refs_be_files;
refs/reftable-backend.c +8 -0
@@ -2303,6 +2303,12 @@ static int reftable_be_reflog_expire(struct ref_store *ref_store,return ret; } +static int reftable_be_fsck (struct ref_store *ref_store, +struct fsck_options *o) +{ +return 0 ; +} +struct ref_storage_be refs_be_reftable = { .name ="reftable" , .init = reftable_be_init,
@@ -2330,4 +2336,6 @@ struct ref_storage_be refs_be_reftable = { .create_reflog = reftable_be_create_reflog, .delete_reflog = reftable_be_delete_reflog, .reflog_expire = reftable_be_reflog_expire, + + .fsck = reftable_be_fsck, };
Signature · 1 line
2.45.2
PATCH v8 5/9 builtin/refs: add verify subcommand
In reply to shejialuoIntroduce a new subcommand "verify" in git-refs(1) to allow the user to check the reference database consistency.
--- Documentation/git-refs.txt | 13 +++++++++++ builtin/refs.c | 44 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+)
Documentation/git-refs.txt +13 -0
@@ -10,6 +10,7 @@ SYNOPSIS -------- [verse] 'git refs migrate' --ref-format=<format> [--dry-run] +'git refs verify' [--strict] [--verbose] DESCRIPTION -----------
@@ -22,6 +23,9 @@ COMMANDS migrate:: Migrate ref store between different formats. +verify:: + Verify reference database consistency. + OPTIONS -------
@@ -39,6 +43,15 @@ include::ref-storage-format.txt[] can be used to double check that the migration works as expected before performing the actual migration. +The following options are specific to 'git refs verify': + +--strict:: + Enable more strict checking, every WARN severity for the `Fsck Messages` + be seen as ERROR. See linkgit:git-fsck[1]. + +--verbose:: + When verifying the reference database consistency, be chatty. + KNOWN LIMITATIONS -----------------
builtin/refs.c +44 -0
69 lines not on this page · raw message
Signature · 1 line
2.45.2
PATCH v8 6/9 builtin/fsck: add `git-refs verify` child process
In reply to shejialuoIntroduce a new function "fsck_refs" that initializes and runs a child process to execute the "git-refs verify" command.
--- builtin/fsck.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+)
builtin/fsck.c +17 -0
@@ -897,6 +897,21 @@ static int check_pack_rev_indexes(struct repository *r, int show_progress)return res; } +static void fsck_refs (void ) +{ +struct child_process refs_verify =CHILD_PROCESS_INIT ; +child_process_init (&refs_verify); + refs_verify.git_cmd =1 ; +strvec_pushl (&refs_verify.args,"refs" ,"verify" ,NULL ); +if (verbose) +strvec_push (&refs_verify.args,"--verbose" ); +if (check_strict) +strvec_push (&refs_verify.args,"--strict" ); + +if (run_command (&refs_verify)) + errors_found |=ERROR_REFS ; +} +static char const *const fsck_usage[] = { N_("git fsck [--tags] [--root] [--unreachable] [--cache] [--no-reflogs]\n" " [--[no-]full] [--strict] [--verbose] [--lost-found]\n"
@@ -1066,6 +1081,8 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)check_connectivity (); +fsck_refs (); +if (the_repository->settings.core_commit_graph) {struct child_process commit_graph_verify =CHILD_PROCESS_INIT ;
Signature · 1 line
2.45.2
PATCH v8 7/9 files-backend: add unified interface for refs scanning
In reply to shejialuoFor refs and reflogs, we need to scan its corresponding directories to check every regular file or symbolic link which shares the same pattern. Introduce a unified interface for scanning directories for files-backend.
--- refs/files-backend.c | 77 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-)
refs/files-backend.c +76 -1
@@ -6,6 +6,7 @@#include "../gettext.h" #include "../hash.h" #include "../hex.h" +#include "../fsck.h" #include "../refs.h" #include "refs-internal.h" #include "ref-cache.h"
@@ -3408,6 +3409,78 @@ static int files_ref_store_remove_on_disk(struct ref_store *ref_store,return ret; } +/* +* For refs and reflogs, they share a unified interface when scanning +* the whole directory. This function is used as the callback for each +* regular file or symlink in the directory. +*/ +typedef int (*files_fsck_refs_fn )(struct fsck_options *o, +const char *gitdir, +const char *refs_check_dir, +struct dir_iterator *iter); + +static int files_fsck_refs_dir (struct ref_store *ref_store, +struct fsck_options *o, +const char *refs_check_dir, +files_fsck_refs_fn *fsck_refs_fns) +{ +const char *gitdir = ref_store->gitdir; +struct strbuf sb =STRBUF_INIT ; +struct dir_iterator *iter; +int iter_status; +int ret =0 ; + +strbuf_addf (&sb,"%s/%s" , gitdir, refs_check_dir); + + iter =dir_iterator_begin (sb.buf,0 ); + +if (!iter) { + ret =error_errno ("cannot open directory %s" , sb.buf); + goto out; + } + +while ((iter_status =dir_iterator_advance (iter)) ==ITER_OK ) { +if (S_ISDIR (iter->st.st_mode)) { +continue ; + }else if (S_ISREG (iter->st.st_mode) || +S_ISLNK (iter->st.st_mode)) { +if (o->verbose_refs) +fprintf_ln (stderr,"Checking %s/%s" , + refs_check_dir, iter->relative_path); +for (size_t i =0 ; fsck_refs_fns[i]; i++) { +if (fsck_refs_fns[i](o, gitdir, refs_check_dir, iter)) + ret =-1 ; + } + }else { + ret =error (_ ("unexpected file type for '%s'" ), + iter->basename); + } + } + +if (iter_status !=ITER_DONE ) + ret =error (_ ("failed to iterate over '%s'" ), sb.buf); + +out: +strbuf_release (&sb); +return ret; +} + +static int files_fsck_refs (struct ref_store *ref_store, +struct fsck_options *o) +{ +int ret; +files_fsck_refs_fn fsck_refs_fns[]= { +NULL + }; + +if (o->verbose_refs) +fprintf_ln (stderr,"Checking references consistency" ); + + ret =files_fsck_refs_dir (ref_store, o,"refs" , fsck_refs_fns); + +return ret; +} +static int files_fsck (struct ref_store *ref_store,struct fsck_options *o) {
@@ -3415,7 +3488,9 @@ static int files_fsck(struct ref_store *ref_store,struct files_ref_store *refs =files_downcast (ref_store,REF_STORE_READ ,"fsck" ); - ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o); + ret = refs->packed_ref_store->be->fsck (refs->packed_ref_store, o) + |files_fsck_refs (ref_store, o); +return ret; }
Signature · 1 line
2.45.2