status: show progress bar if refreshing the index takes too long
PATCH 1/1 status: show progress bar if refreshing the index takes too long
Refreshing the index is usually very fast, but it can still take a long time sometimes. Cold cache is one, or something else silly (*). In this case, it's good to show something to let the user know "git status" is not hanging, it's just busy doing something.
(*) I got called by my colleague because her "git status" took very
long and looked pretty much like hanging. After a bit of strace, it looks to me that git was trying to rehash every single file, and this was a big repository. This process could take minutes.
In this case, I think it was probably because she copied this repository to a new place and stat data did not match anymore. So git fell back to hashing.
---
I need to get this out of my head before I forget. I obviously think this is a good idea and could be done in more places, even just to deal with cold cache. The hint about "git status -uno" for example, could be accompanied by a progress bar for scanning for untracked files...
Another note about rehashing files as part of refresh. We probably could do better by hashing in parallel. Or perhaps not because having a big lock around object database pretty much kills performance, and if I remember correctly none of my core was 100% consumed (i.e. CPU bottleneck as an indication that multithread is a good idea...)
Anyway that's it! Weekend after a long week! I'll read mails and respond tomorrow.
builtin/am.c | 2 +- builtin/commit.c | 6 ++++-- cache.h | 7 +++++-- preload-index.c | 44 +++++++++++++++++++++++++++++++++++++++----- read-cache.c | 10 ++++++++++ sequencer.c | 2 +- 6 files changed, 60 insertions(+), 11 deletions(-)
builtin/am.c +1 -1
@@ -2324,7 +2324,7 @@ int cmd_am(int argc, const char **argv, const char *prefix)/* Ensure a valid committer ident can be constructed */ git_committer_info (IDENT_STRICT ); -if (read_index_preload (&the_index,NULL ) <0 ) +if (read_index_preload (&the_index,NULL ,0 ) <0 )die (_ ("failed to read the index" ));if (in_progress) {
builtin/commit.c +4 -2
@@ -1355,8 +1355,10 @@ int cmd_status(int argc, const char **argv, const char *prefix)PATHSPEC_PREFER_FULL , prefix, argv); -read_cache_preload (&s.pathspec); -refresh_index (&the_index,REFRESH_QUIET |REFRESH_UNMERGED , &s.pathspec,NULL ,NULL ); +read_index_preload (&the_index, &s.pathspec,REFRESH_PROGRESS ); +refresh_index (&the_index, +REFRESH_QUIET |REFRESH_UNMERGED |REFRESH_PROGRESS , + &s.pathspec,NULL ,NULL );if (use_optional_locks ()) fd =hold_locked_index (&index_lock,0 );
cache.h +5 -2
@@ -410,7 +410,7 @@ void validate_cache_entries(const struct index_state *istate);#define read_cache () read_index(&the_index)#define read_cache_from (path) read_index_from(&the_index, (path), (get_git_dir())) -#define read_cache_preload (pathspec) read_index_preload(&the_index, (pathspec)) +#define read_cache_preload (pathspec) read_index_preload(&the_index, (pathspec), 0)#define is_cache_unborn () is_index_unborn(&the_index)#define read_cache_unmerged () read_index_unmerged(&the_index)#define discard_cache () discard_index(&the_index)
@@ -659,7 +659,9 @@ extern int daemonize(void);/* Initialize and use the cache information */ struct lock_file ;extern int read_index (struct index_state *); -extern int read_index_preload (struct index_state *,const struct pathspec *pathspec); +extern int read_index_preload (struct index_state *, +const struct pathspec *pathspec, +unsigned int refresh_flags);extern int do_read_index (struct index_state *istate,const char *path,int must_exist);/* for testting only! */ extern int read_index_from(struct index_state *, const char *path,
@@ -814,6 +816,7 @@ extern void fill_stat_cache_info(struct cache_entry *ce, struct stat *st);#define REFRESH_IGNORE_MISSING 0x0008/* ignore non-existent */ #define REFRESH_IGNORE_SUBMODULES 0x0010/* ignore submodules */ #define REFRESH_IN_PORCELAIN 0x0020/* user friendly output, not "needs update" */ +#define REFRESH_PROGRESS 0x0040/* show progress bar if stderr is tty */ extern int refresh_index (struct index_state *,unsigned int flags,const struct pathspec *pathspec,char *seen,const char *header_msg);extern struct cache_entry *refresh_cache_entry (struct index_state *,struct cache_entry *,unsigned int );
preload-index.c +39 -5
115 lines not on this page · raw message
read-cache.c +10 -0
@@ -23,6 +23,7 @@#include "split-index.h" #include "utf8.h" #include "fsmonitor.h" +#include "progress.h" /* Mask for the name length in ce_flags in the on-disk index */
@@ -1477,6 +1478,11 @@ int refresh_index(struct index_state *istate, unsigned int flags,const char *added_fmt;const char *unmerged_fmt;uint64_t start =getnanotime (); +struct progress *progress =NULL ; + +if (flags &REFRESH_PROGRESS &&isatty (2 )) + progress =start_delayed_progress (_ ("Refresh index" ), + istate->cache_nr); modified_fmt = (in_porcelain ?"M\t%s\n" :"%s: needs update\n" ); deleted_fmt = (in_porcelain ?"D\t%s\n" :"%s: needs update\n" );
@@ -1516,6 +1522,8 @@ int refresh_index(struct index_state *istate, unsigned int flags, new_entry =refresh_cache_ent (istate, ce, options, &cache_errno, &changed);if (new_entry == ce)continue ; +if (progress) +display_progress (progress, i);if (!new_entry) {const char *fmt;
@@ -1547,6 +1555,8 @@ int refresh_index(struct index_state *istate, unsigned int flags,replace_index_entry (istate, i, new_entry); } +if (progress) +stop_progress (&progress);trace_performance_since (start,"refresh index" );return has_errors; }
sequencer.c +1 -1
@@ -1909,7 +1909,7 @@ static int read_and_refresh_cache(struct replay_opts *opts) {struct lock_file index_lock =LOCK_INIT ;int index_fd =hold_locked_index (&index_lock,0 ); -if (read_index_preload (&the_index,NULL ) <0 ) { +if (read_index_preload (&the_index,NULL ,0 ) <0 ) {rollback_lock_file (&index_lock);return error (_ ("git %s: failed to read the index" ),_ (action_name (opts)));
Signature · 1 line
2.19.0.rc0.337.ge906d732e7
Quoted 12 lines · Nguyễn Thái Ngọc Duy
Refreshing the index is usually very fast, but it can still take a long time sometimes. Cold cache is one, or something else silly (*). In this case, it's good to show something to let the user know "git status" is not hanging, it's just busy doing something.
Signed-off-by Nguyễn Thái Ngọc Duy <[email protected]>---
read-cache.c +2
-0@@ -1516,6 +1522,8 @@ int refresh_index(struct index_state *istate, unsigned int flags, +if (progress) +display_progress (progress, i);
read-cache.c int refresh_index(struct index_state *istate, unsigned int flags, · line 1556
Quoted 2 lines
read-cache.c +2
-0@@ -1547,6 +1555,8 @@ int refresh_index(struct index_state *istate, unsigned int flags, +if (progress) +stop_progress (&progress);
Nit: Both display_progress() and stop_progress() behave sanely when 'progress' is NULL, so no need for the conditional.
Quoted 6 lines · Eric Sunshine
Quoted 5 lines · Nguyễn Thái Ngọc Duy
Refreshing the index is usually very fast, but it can still take a long time sometimes. Cold cache is one, or something else silly (*). In this case, it's good to show something to let the user know "git status" is not hanging, it's just busy doing something.
Quoted 11 lines
read-cache.c int refresh_index(struct index_state *istate, unsigned int flags, · line 1556
Quoted 9 lines
Signed-off-by Nguyễn Thái Ngọc Duy <[email protected]>---
read-cache.c +4
-0@@ -1516,6 +1522,8 @@ int refresh_index(struct index_state *istate, unsigned int flags, +if (progress) +display_progress (progress, i);@@ -1547,6 +1555,8 @@ int refresh_index(struct index_state *istate, unsigned int flags, +if (progress) +stop_progress (&progress);Nit: Both
display_progress()andstop_progress()behave sanely when 'progress' is NULL, so no need for the conditional.
Don't forget this one in preload-index.c:preload_index():
+ if (pd.progress)
+ stop_progress(&pd.progress);
I found this extra one by creating the following rules in a Coccinelle script:
@@
expression e;
@@
- if (e) { stop_progress(&e); }
+ stop_progress(&e);
@@
expression e;
expression i;
@@
- if (e) { display_progress(e, i); }
+ display_progress(e, i);
Not sure if we want to put these in a .cocci script or not.
Thanks,
-Stolee
Quoted 13 lines · Derrick Stolee
Quoted 7 lines · Eric Sunshine
read-cache.c int refresh_index(struct index_state *istate, unsigned int flags,
Quoted 4 lines · Nguyễn Thái Ngọc Duy
read-cache.c +4
-0+if (progress) +display_progress (progress, i); +if (progress) +stop_progress (&progress);Nit: Both
display_progress()andstop_progress()behave sanely when 'progress' is NULL, so no need for the conditional.Don't forget this one in preload-index.c:preload_index():
preload-index.c +2
-0+if (pd.progress) +stop_progress (&pd.progress);
Quoted 4 lines
I found this extra one by creating the following rules in a Coccinelle script:
- if (e) {stop_progress(&e); }
- if (e) {display_progress(e, i); }
Not sure if we want to put these in a .cocci script or not.
If so, we'd want to match display_throughput() and stop_progress_msg(), as well.
PATCH v2 1/1 status: show progress bar if refreshing the index takes too long
In reply to Nguyễn Thái Ngọc DuyRefreshing the index is usually very fast, but it can still take a long time sometimes. Cold cache is one. Or copying a repo to a new place (*). It's good to show something to let the user know "git status" is not hanging, it's just busy doing something.
(*) In this case, all stat info in the index becomes invalid and git
falls back to rehashing all file content to see if there's any difference between updating stat info in the index. This is quite expensive. Even with a repo as small as git.git, it takes 3 seconds.
---
v2 changes
- call stop_progress() without checking if the progress_data object is NULL.
- disable progress bar when outputting porcelain formats
builtin/am.c | 2 +- builtin/commit.c | 10 ++++++++-- cache.h | 7 +++++-- preload-index.c | 50 +++++++++++++++++++++++++++++++++++++++++++----- read-cache.c | 12 ++++++++++++ sequencer.c | 2 +- 6 files changed, 72 insertions(+), 11 deletions(-)
builtin/am.c +1 -1
@@ -2324,7 +2324,7 @@ int cmd_am(int argc, const char **argv, const char *prefix)/* Ensure a valid committer ident can be constructed */ git_committer_info (IDENT_STRICT ); -if (read_index_preload (&the_index,NULL ) <0 ) +if (read_index_preload (&the_index,NULL ,0 ) <0 )die (_ ("failed to read the index" ));if (in_progress) {
builtin/commit.c +8 -2
@@ -1295,6 +1295,7 @@ int cmd_status(int argc, const char **argv, const char *prefix)static int no_renames =-1 ;static const char *rename_score_arg = (const char *)-1 ;static struct wt_status s; +unsigned int progress_flag =0 ;int fd;struct object_id oid;static struct option builtin_status_options[] = {
@@ -1355,8 +1356,13 @@ int cmd_status(int argc, const char **argv, const char *prefix)PATHSPEC_PREFER_FULL , prefix, argv); -read_cache_preload (&s.pathspec); -refresh_index (&the_index,REFRESH_QUIET |REFRESH_UNMERGED , &s.pathspec,NULL ,NULL ); +if (status_format !=STATUS_FORMAT_PORCELAIN && + status_format !=STATUS_FORMAT_PORCELAIN_V2 ) + progress_flag =REFRESH_PROGRESS ; +read_index_preload (&the_index, &s.pathspec, progress_flag); +refresh_index (&the_index, +REFRESH_QUIET |REFRESH_UNMERGED |progress_flag, + &s.pathspec,NULL ,NULL );if (use_optional_locks ()) fd =hold_locked_index (&index_lock,0 );
cache.h +5 -2
@@ -410,7 +410,7 @@ void validate_cache_entries(const struct index_state *istate);#define read_cache () read_index(&the_index)#define read_cache_from (path) read_index_from(&the_index, (path), (get_git_dir())) -#define read_cache_preload (pathspec) read_index_preload(&the_index, (pathspec)) +#define read_cache_preload (pathspec) read_index_preload(&the_index, (pathspec), 0)#define is_cache_unborn () is_index_unborn(&the_index)#define read_cache_unmerged () read_index_unmerged(&the_index)#define discard_cache () discard_index(&the_index)
@@ -659,7 +659,9 @@ extern int daemonize(void);/* Initialize and use the cache information */ struct lock_file ;extern int read_index (struct index_state *); -extern int read_index_preload (struct index_state *,const struct pathspec *pathspec); +extern int read_index_preload (struct index_state *, +const struct pathspec *pathspec, +unsigned int refresh_flags);extern int do_read_index (struct index_state *istate,const char *path,int must_exist);/* for testting only! */ extern int read_index_from(struct index_state *, const char *path,
@@ -814,6 +816,7 @@ extern void fill_stat_cache_info(struct cache_entry *ce, struct stat *st);#define REFRESH_IGNORE_MISSING 0x0008/* ignore non-existent */ #define REFRESH_IGNORE_SUBMODULES 0x0010/* ignore submodules */ #define REFRESH_IN_PORCELAIN 0x0020/* user friendly output, not "needs update" */ +#define REFRESH_PROGRESS 0x0040/* show progress bar if stderr is tty */ extern int refresh_index (struct index_state *,unsigned int flags,const struct pathspec *pathspec,char *seen,const char *header_msg);extern struct cache_entry *refresh_cache_entry (struct index_state *,struct cache_entry *,unsigned int );
preload-index.c +45 -5
126 lines not on this page · raw message
read-cache.c +12 -0
@@ -23,6 +23,7 @@#include "split-index.h" #include "utf8.h" #include "fsmonitor.h" +#include "progress.h" /* Mask for the name length in ce_flags in the on-disk index */
@@ -1477,6 +1478,11 @@ int refresh_index(struct index_state *istate, unsigned int flags,const char *added_fmt;const char *unmerged_fmt;uint64_t start =getnanotime (); +struct progress *progress =NULL ; + +if (flags &REFRESH_PROGRESS &&isatty (2 )) + progress =start_delayed_progress (_ ("Refresh index" ), + istate->cache_nr); modified_fmt = (in_porcelain ?"M\t%s\n" :"%s: needs update\n" ); deleted_fmt = (in_porcelain ?"D\t%s\n" :"%s: needs update\n" );
@@ -1516,6 +1522,8 @@ int refresh_index(struct index_state *istate, unsigned int flags, new_entry =refresh_cache_ent (istate, ce, options, &cache_errno, &changed);if (new_entry == ce)continue ; +if (progress) +display_progress (progress, i);if (!new_entry) {const char *fmt;
@@ -1547,6 +1555,10 @@ int refresh_index(struct index_state *istate, unsigned int flags,replace_index_entry (istate, i, new_entry); } +if (progress) { +display_progress (progress, istate->cache_nr); +stop_progress (&progress); + }trace_performance_since (start,"refresh index" );return has_errors; }
sequencer.c +1 -1
@@ -1909,7 +1909,7 @@ static int read_and_refresh_cache(struct replay_opts *opts) {struct lock_file index_lock =LOCK_INIT ;int index_fd =hold_locked_index (&index_lock,0 ); -if (read_index_preload (&the_index,NULL ) <0 ) { +if (read_index_preload (&the_index,NULL ,0 ) <0 ) {rollback_lock_file (&index_lock);return error (_ ("git %s: failed to read the index" ),_ (action_name (opts)));
Signature · 1 line
2.19.0.rc0.337.ge906d732e7