propose config-based hooks
Subsystems (2)
PATCH v6 16/17 run-command: allow capturing of collated output
In reply to Emily ShafferSome callers, for example server-side hooks which wish to relay hook output to clients across a transport, want to capture what would normally print to stderr and do something else with it. Allow that via a callback.
By calling the callback regardless of whether there's output available, we allow clients to send e.g. a keepalive if necessary.
Because we expose a strbuf, not a fd or FILE*, there's no need to create a temporary pipe or similar - we can just skip the print to stderr and instead hand it to the caller.
---
Notes:
Originally when writing this patch I attempted to use a pipe in memory - but managing its lifetime was actually pretty tricky, and I found I could achieve the same thing with less code by doing it this way. Critique welcome, including "no, you really need to do it with a pipe".
builtin/fetch.c | 2 +- builtin/submodule--helper.c | 2 +- hook.c | 1 + run-command.c | 33 +++++++++++++++++++++++++-------- run-command.h | 18 +++++++++++++++++- submodule.c | 2 +- t/helper/test-run-command.c | 25 ++++++++++++++++++++----- t/t0061-run-command.sh | 7 +++++++ 8 files changed, 73 insertions(+), 17 deletions(-)
builtin/fetch.c +1 -1
@@ -1647,7 +1647,7 @@ static int fetch_multiple(struct string_list *list, int max_children) result =run_processes_parallel_tr2 (max_children, &fetch_next_remote, &fetch_failed_to_start, -NULL , +NULL ,NULL , &fetch_finished, &state,"fetch" ,"parallel/fetch" );
builtin/submodule--helper.c +1 -1
@@ -2294,7 +2294,7 @@ static int update_submodules(struct submodule_update_clone *suc)int i;run_processes_parallel_tr2 (suc->max_jobs, update_clone_get_next_task, - update_clone_start_failure,NULL , + update_clone_start_failure,NULL ,NULL , update_clone_task_finished, suc,"submodule" ,"parallel/update" );
hook.c +1 -0
@@ -407,6 +407,7 @@ int run_hooks(const char *hookname, struct run_hooks_opt *options) pick_next_hook, notify_start_failure, options->feed_pipe, +NULL , notify_hook_finished, &cb_data, "hook",
run-command.c +25 -8
@@ -1549,6 +1549,7 @@ struct parallel_processes {get_next_task_fn get_next_task;start_failure_fn start_failure;feed_pipe_fn feed_pipe; +consume_sideband_fn consume_sideband;task_finished_fn task_finished;struct {
@@ -1614,6 +1615,7 @@ static void pp_init(struct parallel_processes *pp,get_next_task_fn get_next_task, start_failure_fn start_failure, feed_pipe_fn feed_pipe, + consume_sideband_fn consume_sideband, task_finished_fn task_finished, void *data) {
@@ -1634,6 +1636,7 @@ static void pp_init(struct parallel_processes *pp, pp->start_failure = start_failure ? start_failure : default_start_failure; pp->feed_pipe = feed_pipe ? feed_pipe : default_feed_pipe; pp->task_finished = task_finished ? task_finished : default_task_finished; + pp->consume_sideband = consume_sideband; pp->nr_processes =0 ; pp->output_owner =0 ;
@@ -1670,7 +1673,10 @@ static void pp_cleanup(struct parallel_processes *pp)* When get_next_task added messages to the buffer in its last * iteration, the buffered output is non empty. */ -strbuf_write (&pp->buffered_output, stderr); +if (pp->consume_sideband) + pp->consume_sideband (&pp->buffered_output, pp->data); +else +strbuf_write (&pp->buffered_output, stderr);strbuf_release (&pp->buffered_output);sigchain_pop_common ();
@@ -1786,9 +1792,13 @@ static void pp_buffer_stderr(struct parallel_processes *pp, int output_timeout)static void pp_output (struct parallel_processes *pp) {int i = pp->output_owner; +if (pp->children[i].state ==GIT_CP_WORKING && pp->children[i].err.len) { -strbuf_write (&pp->children[i].err, stderr); +if (pp->consume_sideband) + pp->consume_sideband (&pp->children[i].err, pp->data); +else +strbuf_write (&pp->children[i].err, stderr);strbuf_reset (&pp->children[i].err); } }
@@ -1827,11 +1837,15 @@ static int pp_collect_finished(struct parallel_processes *pp)strbuf_addbuf (&pp->buffered_output, &pp->children[i].err);strbuf_reset (&pp->children[i].err); }else { -strbuf_write (&pp->children[i].err, stderr); +/* Output errors, then all other finished child processes */ +if (pp->consume_sideband) { + pp->consume_sideband (&pp->children[i].err, pp->data); + pp->consume_sideband (&pp->buffered_output, pp->data); + }else { +strbuf_write (&pp->children[i].err, stderr); +strbuf_write (&pp->buffered_output, stderr); + }strbuf_reset (&pp->children[i].err); - -/* Output all other finished child processes */ -strbuf_write (&pp->buffered_output, stderr);strbuf_reset (&pp->buffered_output);/*
@@ -1855,6 +1869,7 @@ int run_processes_parallel(int n,get_next_task_fn get_next_task, start_failure_fn start_failure, feed_pipe_fn feed_pipe, + consume_sideband_fn consume_sideband, task_finished_fn task_finished, void *pp_cb) {
@@ -1865,7 +1880,7 @@ int run_processes_parallel(int n,sigchain_push (SIGPIPE ,SIG_IGN ); -pp_init (&pp, n, get_next_task, start_failure, feed_pipe, task_finished, pp_cb); +pp_init (&pp, n, get_next_task, start_failure, feed_pipe, consume_sideband, task_finished, pp_cb);while (1 ) { for (i = 0; i < spawn_cap && !pp.shutdown &&
@@ -1903,6 +1918,7 @@ int run_processes_parallel(int n,int run_processes_parallel_tr2 (int n,get_next_task_fn get_next_task,start_failure_fn start_failure,feed_pipe_fn feed_pipe, +consume_sideband_fn consume_sideband,task_finished_fn task_finished,void *pp_cb,const char *tr2_category,const char *tr2_label) {
@@ -1912,7 +1928,8 @@ int run_processes_parallel_tr2(int n, get_next_task_fn get_next_task, ((n < 1) ? online_cpus() : n)); result =run_processes_parallel (n, get_next_task, start_failure, - feed_pipe, task_finished, pp_cb); + feed_pipe, consume_sideband, + task_finished, pp_cb);trace2_region_leave (tr2_category, tr2_label,NULL );
run-command.h +17 -1
@@ -450,6 +450,20 @@ typedef int (*feed_pipe_fn)(struct strbuf *pipe,void *pp_cb, void *pp_task_cb); +/** +* If this callback is provided, instead of collating process output to stderr, +* they will be collated into a new pipe. consume_sideband_fn will be called +* repeatedly. When output is available on that pipe, it will be contained in +* 'output'. But it will be called with an empty 'output' too, to allow for +* keepalives or similar operations if necessary. +* +* pp_cb is the callback cookie as passed into run_processes_parallel. +* +* Since this callback is provided with the collated output, no task cookie is +* provided. +*/ +typedef void (*consume_sideband_fn )(struct strbuf *output,void *pp_cb); +/** * This callback is called on every child process that finished processing. *
@@ -485,10 +499,12 @@ int run_processes_parallel(int n, get_next_task_fn, start_failure_fn, feed_pipe_fn, + consume_sideband_fn, task_finished_fn, void *pp_cb);int run_processes_parallel_tr2 (int n,get_next_task_fn ,start_failure_fn , -feed_pipe_fn ,task_finished_fn ,void *pp_cb, +feed_pipe_fn ,consume_sideband_fn , +task_finished_fn ,void *pp_cb,const char *tr2_category,const char *tr2_label);#endif
submodule.c +1 -1
@@ -1638,7 +1638,7 @@ int fetch_populated_submodules(struct repository *r,run_processes_parallel_tr2 (max_parallel_jobs, get_next_submodule, fetch_start_failure, -NULL , +NULL ,NULL , fetch_finish, &spf,"submodule" ,"parallel/fetch" );
t/helper/test-run-command.c +20 -5
@@ -51,6 +51,16 @@ static int no_job(struct child_process *cp,return 0 ; } +static void test_consume_sideband (struct strbuf *output,void *cb) +{ +FILE *sideband; + + sideband =fopen ("./sideband" ,"a" ); + +strbuf_write (output, sideband); +fclose (sideband); +} + static int task_finished(int result, struct strbuf *err, void *pp_cb,
@@ -201,7 +211,7 @@ static int testsuite(int argc, const char **argv) suite.tests.nr, max_jobs); ret =run_processes_parallel (max_jobs, next_test, test_failed, - test_stdin, test_finished, &suite); + test_stdin,NULL , test_finished, &suite);if (suite.failed.nr >0 ) { ret =1 ;
@@ -429,23 +439,28 @@ int cmd__run_command(int argc, const char **argv)if (!strcmp (argv[1 ],"run-command-parallel" ))exit (run_processes_parallel (jobs, parallel_next, -NULL ,NULL ,NULL , &proc)); +NULL ,NULL ,NULL ,NULL , &proc));if (!strcmp (argv[1 ],"run-command-abort" ))exit (run_processes_parallel (jobs, parallel_next, -NULL ,NULL , task_finished, &proc)); +NULL ,NULL ,NULL , task_finished, &proc));if (!strcmp (argv[1 ],"run-command-no-jobs" ))exit (run_processes_parallel (jobs, no_job, -NULL ,NULL , task_finished, &proc)); +NULL ,NULL ,NULL , task_finished, &proc));if (!strcmp (argv[1 ],"run-command-stdin" )) { proc.in =-1 ; proc.no_stdin =0 ;exit (run_processes_parallel (jobs, parallel_next,NULL , - test_stdin,NULL , &proc)); + test_stdin,NULL ,NULL , &proc)); } +if (!strcmp (argv[1 ],"run-command-sideband" )) +exit (run_processes_parallel (jobs, parallel_next,NULL ,NULL , + test_consume_sideband,NULL , + &proc)); +fprintf (stderr,"check usage\n" );return 1 ; }
t/t0061-run-command.sh +7 -0
@@ -143,6 +143,13 @@ test_expect_success 'run_command runs in parallel with more tasks than jobs avai test_cmp expect actual ' +test_expect_success 'run_command can divert output' ' + test_when_finished rm sideband && + test-tool run-command run-command-sideband 3 sh -c "printf \"%s\n%s\n\" Hello World" 2>actual && + test_must_be_empty actual && + test_cmp expect sideband +' + cat >expect <<-EOF preloaded output of a child listening for stdin:
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
PATCH v6 17/17 hooks: allow callers to capture output
In reply to Emily ShafferSome server-side hooks will require capturing output to send over sideband instead of printing directly to stderr. Expose that capability.
---
Notes:
You can see this in practice in the conversions for some of the push hooks, like 'receive-pack'.
hook.c | 2 +- hook.h | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-)
hook.c +1 -1
@@ -407,7 +407,7 @@ int run_hooks(const char *hookname, struct run_hooks_opt *options) pick_next_hook, notify_start_failure, options->feed_pipe, -NULL , + options->consume_sideband, notify_hook_finished, &cb_data, "hook",
hook.h +10 -0
@@ -76,6 +76,14 @@ struct run_hooks_optfeed_pipe_fn feed_pipe;void *feed_pipe_ctx; +/* +* Populate this to capture output and prevent it from being printed to +* stderr. This will be passed directly through to +* run_command:run_parallel_processes(). See t/helper/test-run-command.c +* for an example. +*/ +consume_sideband_fn consume_sideband; +/* Number of threads to parallelize across */ int jobs;
@@ -103,6 +111,7 @@ struct hook_cb_data { .str_stdin = STRING_LIST_INIT_DUP, \ .feed_pipe = NULL, \ .feed_pipe_ctx = NULL, \ + .consume_sideband = NULL, \ .run_hookdir = configured_hookdir_opt() \ }
@@ -115,6 +124,7 @@ struct hook_cb_data { .str_stdin = STRING_LIST_INIT_DUP, \ .feed_pipe = NULL, \ .feed_pipe_ctx = NULL, \ + .consume_sideband = NULL, \ .run_hookdir = configured_hookdir_opt() \ }
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
Re: [PATCH v6 00/17] propose config-based hooks (part I)
In reply to Emily ShafferQuoted 13 lines · Emily Shaffer
Hi folks, and thanks for the patience - I ran into many, many last-mile challenges.
I haven't addressed many comments on the design doc yet - I was keen to get the "functionally complete" implementation and conversion to the list.
Next on my plate:
- Update the design doc to make sense with what's in the implementation.
- A blog post! How to set up new hooks, why they're neat, etc.
- We seem to have some Googlers interested in trying it out internally, so I'm hoping we'll gather and collate feedback from that soon too.
- And of course addressing comments on this series.
Quoted 2 lines
Thanks!
- Emily
This approach looks good to me. I'll look forward to seeing the updated design and the feedback from the internal tests.
Re: [PATCH v6 00/17] propose config-based hooks (part I)
Quoted 14 lines · Josh Steadmon
Quoted 13 lines · Emily Shaffer
Hi folks, and thanks for the patience - I ran into many, many last-mile challenges.
I haven't addressed many comments on the design doc yet - I was keen to get the "functionally complete" implementation and conversion to the list.
Next on my plate:
- Update the design doc to make sense with what's in the implementation.
- A blog post! How to set up new hooks, why they're neat, etc.
- We seem to have some Googlers interested in trying it out internally, so I'm hoping we'll gather and collate feedback from that soon too.
- And of course addressing comments on this series.
Quoted 5 lines
Quoted 2 lines
Thanks!
- EmilyThis approach looks good to me. I'll look forward to seeing the updated design and the feedback from the internal tests.
Thanks.
By the way, es/config-hooks does not seem to pass 5411 (at least) even as a standalone topic, and has been kicked out of 'seen' for some time. Has anybody took a look into the issue?
Re: [PATCH v6 00/17] propose config-based hooks (part I)
Quoted 20 lines · Junio C Hamano
Quoted 17 lines · Josh Steadmon
Quoted 15 lines · Emily Shaffer
Hi folks, and thanks for the patience - I ran into many, many last-mile challenges.
I haven't addressed many comments on the design doc yet - I was keen to get the "functionally complete" implementation and conversion to the list.
Next on my plate:
- Update the design doc to make sense with what's in the implementation.
- A blog post! How to set up new hooks, why they're neat, etc.
- We seem to have some Googlers interested in trying it out internally, so I'm hoping we'll gather and collate feedback from that soon too.
- And of course addressing comments on this series.Thanks!
- Emily
Quoted 8 lines
Quoted 2 lines
This approach looks good to me. I'll look forward to seeing the updated design and the feedback from the internal tests.
Thanks.
By the way, es/config-hooks does not seem to pass 5411 (at least) even as a standalone topic, and has been kicked out of 'seen' for some time. Has anybody took a look into the issue?
Yeah, I looked at it today. Looks like an issue with not paying attention to master->main default config, since I added a new test to the 5411 suite (which means it wouldn't have made a conflict for someone to say "ah yes, s/master/main/g"). I am tracking down couple of other CI errors today and will send a reroll today or tomorrow.
Re: [PATCH v6 00/17] propose config-based hooks (part I)
Quoted 9 lines · Emily Shaffer
Quoted 3 lines
By the way, es/config-hooks does not seem to pass 5411 (at least) even as a standalone topic, and has been kicked out of 'seen' for some time. Has anybody took a look into the issue?
Yeah, I looked at it today. Looks like an issue with not paying attention to
master->maindefault config, since I added a new test to the 5411 suite (which means it wouldn't have made a conflict for someone to say "ah yes, s/master/main/g"). I am tracking down couple of other CI errors today and will send a reroll today or tomorrow.
Thanks.
Re: [PATCH v6 00/17] propose config-based hooks (part I)
Quoted 7 lines · Junio C Hamano
Quoted 4 lines · Emily Shaffer
Quoted 3 lines
By the way, es/config-hooks does not seem to pass 5411 (at least) even as a standalone topic, and has been kicked out of 'seen' for some time. Has anybody took a look into the issue?
Quoted 7 lines
Quoted 5 lines
Yeah, I looked at it today. Looks like an issue with not paying attention to
master->maindefault config, since I added a new test to the 5411 suite (which means it wouldn't have made a conflict for someone to say "ah yes, s/master/main/g"). I am tracking down couple of other CI errors today and will send a reroll today or tomorrow.Thanks.
I don't have a reroll today. I have been trying to get to the bottom of a test which fails when built with clang but passes when built with gcc (t6030-bisect-porcelain.sh after patch 12 of the part II series) and have not made progress on that, let alone on the other tasks I wanted to do before sending the next version.
Next week I will only work one day, so I'd anticipate a reroll sometime the week following. Sorry for the wait - but I think even if I sent it with the fix for this t5411 failure, it would still break 'seen' because of whatever this clang vs. gcc problem is.
Hope you enjoy your holidays.
- Emily
Re: [PATCH v6 00/17] propose config-based hooks (part I)
Quoted 5 lines · Emily Shaffer
I don't have a reroll today. I have been trying to get to the bottom of a test which fails when built with clang but passes when built with gcc (
t6030-bisect-porcelain.shafter patch 12 of the part II series) and have not made progress on that, let alone on the other tasks I wanted to do before sending the next version.
Thanks for an interim report. No need to rush.
Quoted 1 line
Hope you enjoy your holidays.
You too, and have fun.
v7 cover propose config-based hooks (part I)
In reply to Emily ShafferSince v6:
- Converted 'enum hookdir_opt' to UPPER_SNAKE
- Coccinelle fix in the hook destructor
- Fixed a bug where builtin/hook.c wasn't running the default git config setup and therefore missed hooks in core.hooksPath when it was set. (These hooks would still run except when invoked by 'git hook run' as the config was called by the processes which invoked the hook library.)
CI run: https://github.com/nasamuffin/git/actions/runs/436864964
Thanks!
- Emily
Emily Shaffer (17):
doc: propose hooks managed by the config
hook: scaffolding for git-hook subcommand
hook: add list command
hook: include hookdir hook in list
hook: respect hook.runHookDir
hook: implement hookcmd.<name>.skip
parse-options: parse into strvec
hook: add 'run' subcommand
hook: replace find_hook() with hook_exists()
hook: support passing stdin to hooks
run-command: allow stdin for run_processes_parallel
hook: allow parallel hook execution
hook: allow specifying working directory for hooks
run-command: add stdin callback for parallelization
hook: provide stdin by string_list or callback
run-command: allow capturing of collated output
hooks: allow callers to capture output
.gitignore | 1 + Documentation/Makefile | 1 + Documentation/config/hook.txt | 19 + Documentation/git-hook.txt | 117 +++++ Documentation/technical/api-parse-options.txt | 5 + .../technical/config-based-hooks.txt | 355 +++++++++++++++ Makefile | 2 + builtin.h | 1 + builtin/bugreport.c | 4 +- builtin/fetch.c | 1 + builtin/hook.c | 176 ++++++++ builtin/submodule--helper.c | 2 +- command-list.txt | 1 + git.c | 1 + hook.c | 416 ++++++++++++++++++ hook.h | 154 +++++++ parse-options-cb.c | 16 + parse-options.h | 4 + run-command.c | 85 +++- run-command.h | 31 ++ submodule.c | 1 + t/helper/test-run-command.c | 46 +- t/t0061-run-command.sh | 37 ++ t/t1360-config-based-hooks.sh | 256 +++++++++++ 24 files changed, 1717 insertions(+), 15 deletions(-) create mode 100644 Documentation/config/hook.txt create mode 100644 Documentation/git-hook.txt create mode 100644 Documentation/technical/config-based-hooks.txt create mode 100644 builtin/hook.c create mode 100644 hook.c create mode 100644 hook.h create mode 100755 t/t1360-config-based-hooks.sh
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
PATCH v7 1/17 doc: propose hooks managed by the config
Begin a design document for config-based hooks, managed via git-hook. Focus on an overview of the implementation and motivation for design decisions. Briefly discuss the alternatives considered before this point. Also, attempt to redefine terms to fit into a multihook world.
---
Notes:
Since v6, checked for inconsistencies with implementation and added lots of caveats about whether 'git hook add' and 'git hook edit' will ever materialize.
Hopefully this reflects reality now; please review accordingly.
Since v6, checked for inconsistencies with implementation and added lots of caveats about whether 'git hook add' and 'git hook edit' will ever materialize.
Hopefully this reflects reality now; please review accordingly.
Since v4, addressed comments from Jonathan Tan about wording. However, I have not addressed AEvar's comments or done a full re-review of this document. I wanted to get the rest of the series out for initial review first.
- Emily
Since v4, addressed comments from Jonathan Tan about wording.
Documentation/Makefile | 1 + .../technical/config-based-hooks.txt | 355 ++++++++++++++++++ 2 files changed, 356 insertions(+) create mode 100644 Documentation/technical/config-based-hooks.txt
Documentation/Makefile +1 -0
@@ -81,6 +81,7 @@ SP_ARTICLES += $(API_DOCS) TECH_DOCS += MyFirstContribution TECH_DOCS += MyFirstObjectWalk TECH_DOCS += SubmittingPatches +TECH_DOCS += technical/config-based-hooks TECH_DOCS += technical/hash-function-transition TECH_DOCS += technical/http-protocol TECH_DOCS += technical/index-format
Documentation/technical/config-based-hooks.txt new file +355 -0
355 lines not on this page · raw message
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
Re: [PATCH v7 01/17] doc: propose hooks managed by the config
Quoted 3 lines · Emily Shaffer
Since v4, addressed comments from Jonathan Tan about wording. However, I have not addressed AEvar's comments or done a full re-review of this document. I wanted to get the rest of the series out for initial review first.
As you note here and in a couple of other patch notes a lot of the current state was based on my v4 commentary. I see we have parallel hook execution by default now, woohoo!
I've been keeping an eye on this series, but have been kicking the can down the road on reviewing it.
Skimming it now I think the state of it looks mostly good now when viewing the end result, I think it's mainly got one big problem, but the good news is that it's relatively easy to solve :)
Which is that I think it's really hard to follow along with it because 01/17 starts with a big design doc that's partially outdated, and partially saying things that aren't in or should be in either a user-facing doc or commit message.
And then individual patches (e.g. 12/17) either don't have tests associated with them to test the feature they add, don't update/add docs, or the docs are at the very beginning.
I think we should aim to mostly or entirely get rid of Documentation/technical/config-based-hooks.txt, it was more of a "what about this design?" document in the beginning.
In a series we'd apply most or all of it should really be in end-user doc (and stuff like "Future work" can just be noted in commit messages as we go along).
So long story short, I started trying to review this, but found myself trying to reply to one patch and then grabbing docs from 01/17, or (e.g. for the parallel stuff) not having tests and starting to come up with them myself.
So I thought I'd send this E-Mail instead as prodding to maybe convince you to re-roll it again to make it easier to follow along in a piecemeal fashion.
As noted before I'm happy to help with this series if needed. I just thought I'd send this first given that it's been a month since the last submission, perhaps you've got some more local WIP changes by now...
Re: [PATCH v7 01/17] doc: propose hooks managed by the config
Quoted 15 lines · Ævar Arnfjörð Bjarmason
Quoted 3 lines · Emily Shaffer
Since v4, addressed comments from Jonathan Tan about wording. However, I have not addressed AEvar's comments or done a full re-review of this document. I wanted to get the rest of the series out for initial review first.
As you note here and in a couple of other patch notes a lot of the current state was based on my v4 commentary. I see we have parallel hook execution by default now, woohoo!
I've been keeping an eye on this series, but have been kicking the can down the road on reviewing it.
Quoted 8 lines
Skimming it now I think the state of it looks mostly good now when viewing the end result, I think it's mainly got one big problem, but the good news is that it's relatively easy to solve :)
Which is that I think it's really hard to follow along with it because 01/17 starts with a big design doc that's partially outdated, and partially saying things that aren't in or should be in either a user-facing doc or commit message.
Sure.
Quoted 4 lines
And then individual patches (e.g. 12/17) either don't have tests associated with them to test the feature they add, don't update/add docs, or the docs are at the very beginning.
Thanks, sure.
Quoted 4 lines
I think we should aim to mostly or entirely get rid of
Documentation/technical/config-based-hooks.txt, it was more of a "what about this design?" document in the beginning.
I'm not 100% sure that I agree - there are a couple other design docs in Documentation/technical which I still refer to from time to time, e.g. for sparse checkout. But I *do* agree that there's a lot of info there that needs to be in end-user docs.
Quoted 9 lines
In a series we'd apply most or all of it should really be in end-user doc (and stuff like "Future work" can just be noted in commit messages as we go along).
So long story short, I started trying to review this, but found myself trying to reply to one patch and then grabbing docs from 01/17, or (e.g. for the parallel stuff) not having tests and starting to come up with them myself.
Yeah. A related issue I could imagine, although not what you mentioned here, is needing to do the same thing between part I and part II of the series, as I often added some functionality late in part I and then used it first late in part II. I don't think this is worth reordering for, but probably better notes would be handy.
Quoted 8 lines
So I thought I'd send this E-Mail instead as prodding to maybe convince you to re-roll it again to make it easier to follow along in a piecemeal fashion.
As noted before I'm happy to help with this series if needed. I just thought I'd send this first given that it's been a month since the last submission, perhaps you've got some more local WIP changes by now...
The biggest help for me would be review focused on nits, code style, missed free(), etc. - I still haven't gotten that kind of review yet, and I feel the series as it is now is pretty much "feature complete".
In fact, and I'll mention this in reply to the cover letter in a moment, we picked this series up as-is from Junio's branch in 'gitster/git' and have been using it at Google for a couple of weeks now - primarily with users running their legacy hooks living in hookdir, but also with a subset of users encouraged to try out the config functionality.
Thanks for the bump.
- Emily
Re: [PATCH v7 01/17] doc: propose hooks managed by the config
In reply to Emily ShafferDocumentation/technical/config-based-hooks.txt line 139
Quoted 3 lines · Emily Shaffer
Documentation/technical/config-based-hooks.txt new file +3
-0+ways, providing an avenue to deprecate these "legacy" hooks if desired. The +handling is based on a config `hook.runHookDir`, which is checked against a +number of cases:
Don't we want to also warn when the setting "no" or something similar prevents the legacy hook from running, to help users who wonder why their hook scripts are not running? I.e.
Quoted 1 line
+- "no": the legacy hook will not be run
+- "warn-no": Git will print a warning to stderr before ignoring the + legacy hook
Documentation/technical/config-based-hooks.txt line 150
Quoted 9 lines
+- "interactive": Git will prompt the user before running the legacy hook +- "warn": Git will print a warning to stderr before running the legacy hook +- "yes" (default): Git will silently run the legacy hookDocumentation/technical/config-based-hooks.txt new file +5
-0+In case this list is expanded in the future, if a value for `hook.runHookDir` is +given which Git does not recognize, Git should discard that config entry. For +example, if "warn" was specified at system level and "junk" was specified at +global level, Git would resolve the value to "warn"; if the only time the config +was set was to "junk", Git would use the default value of "yes".
Hmph, instead of complaining "value 'junk' is not recognized" and erroring out? Why?
Documentation/technical/config-based-hooks.txt line 203
Quoted 6 lines
+[[stage-3]]
+==== Stage 3Documentation/technical/config-based-hooks.txt new file +4
-0+ +`.git/hooks` is removed from the template and the hook directory is considered +deprecated. To avoid breaking older repos, the default of `hook.runHookDir` is +not changed, and `find_hook()` is not removed.
Presumably, we'll have documentation somewhere that instructs users (who were taught by slashdot and other site to add certain scripts under their .git/hooks/) how to do the equivalent without adding scripts in .git/hooks/ directory and instead using the config mechanism (e.g. "when told to add script X in .git/hooks/, read such an instruction as if telling you to do Y instead") by the time this happens? It probably makes sense to do so as part of stage-2, at which point the users are _ready_ to migrate.
Documentation/technical/config-based-hooks.txt line 218
Quoted 11 lines
+[[security]]
+=== Security and repo configDocumentation/technical/config-based-hooks.txt new file +9
-0+ +Part of the motivation behind this refactor is to mitigate hooks as an attack +vector;footnote:[https://lore.kernel.org/git/[email protected]/] +however, as the design stands, users can still provide hooks in the repo-level +config, which is included when a repo is zipped and sent elsewhere. The +security of the repo-level config is still under discussion; this design +generally assumes the repo-level config is secure, which is not true yet. The +goal is to avoid an overcomplicated design to work around a problem which has +ceased to exist.
I doubt we want to claim anything about security as part of this series. As you say in the paragraph, .git/config and .git/hooks/ are equally (un)protected and if we decide to punt .git/config security, then not moving away from .git/hooks would not hurt security-wise, either (in other words, security is not a viable motivation behind this series).
And if we stop advertising 'security merit' that does not exist, what remains? Isn't the biggest selling point that an identical set of hook configuration can be shared among multiple repositories, and it allows more than one hook scripts to be triggered by a single "hook event"? There may be other good things we should be able to sell the new mechanism to our users, and we do stress on them, which is done in the motivation section. So...
Quoted 3 lines
+.Comparison of alternatives
+|===
+|Feature |Config-based hooks |Hook directories |Status quo
Sorry, but I did not find this table particularly convincing.
The only thing I sense is a hand-wavy desire that "we could make it better than everybody else if we work on it in this area", which can apply equally for other approaches---they could enhance what they already have (e.g. "discoverability & documentation").
As a list of "these are the points we aspire to do better than other people", I think it is an excellent idea to have a table like this here in the documentation. But that is not a "comparison".
Quoted 8 lines
+[[execution-ordering]]
+=== Execution orderingDocumentation/technical/config-based-hooks.txt new file +6
-0+ +We may find that config order is insufficient for some users; for example, +config order makes it difficult to add a new hook to the system or global config +which runs at the end of the hook list. A new ordering schema should be: + +1) Specified by a `hook.order` config, so that users will not unexpectedly see
Documentation/technical/config-based-hooks.txt line 304
Quoted 10 lines
Documentation/technical/config-based-hooks.txt new file +10
-0+their order change; + +2) Either dependency or numerically based. + +Dependency-based ordering is prone to classic linked-list problems, like a +cycles and handling of missing dependencies. But, it paves the way for enabling +parallelization if some tasks truly depend on others. + +Numerical ordering makes it tricky for Git to generate suggested ordering +numbers for each command, but is easy to determine a definitive order.
OK.
Have we decided what we do for hooks whose interface is to feed their input from their standard input? The current system, I think, just feeds the single hook by writing into a pipe to it, but if we were to drive multiple hooks, we'd need to write the same thing to each of these hook programs?
Do we have a plan to deal with hooks whose outcome is not just "yes/no", e.g. "proc-receive" hook that munges the list of refs to be updated and the new values for them, or "applypatch-msg" that munges the incoming proposed commit log message? Does the second hook work on the result of the first hook? Do the two hooks work on the vanilla state and their output have to agree with each other?
Documentation/technical/config-based-hooks.txt
Quoted 6 lines
+[[parallelization]]
+=== Parallelization with dependenciesDocumentation/technical/config-based-hooks.txt new file +4
-0+ +Currently hooks use a naive parallelization scheme or are run in series. But if +one hook depends on another's output, then users will want to specify those +dependencies.
An untold assumption here is that the questions I asked earlier on having more than one hooks that is not just yes/no is something readers know the same answer, and that answer is "the outcome of the first hook is passed along (as if it were the input given by Git directly, if the first hook did not exist), to the second hook. It should be spelled out somewhere before the execution ordering section, I think.
Re: [PATCH v7 01/17] doc: propose hooks managed by the config
Quoted 11 lines · Junio C Hamano
Documentation/technical/config-based-hooks.txt line 139
Quoted 3 lines · Emily Shaffer
Documentation/technical/config-based-hooks.txt new file +3
-0+ways, providing an avenue to deprecate these "legacy" hooks if desired. The +handling is based on a config `hook.runHookDir`, which is checked against a +number of cases:Don't we want to also warn when the setting "no" or something similar prevents the legacy hook from running, to help users who wonder why their hook scripts are not running? I.e.
Quoted 4 lines
Quoted 1 line
+- "no": the legacy hook will not be run
+- "warn-no": Git will print a warning to stderr before ignoring the + legacy hook
Yeah, I think you are right. Jonathan N suggested such an option as "error" (as opposed to warning); I'll add it here plus to the enum and take your description verbatim. Thanks.
Quoted 5 lines
Quoted 4 lines
+- "interactive": Git will prompt the user before running the legacy hook +- "warn": Git will print a warning to stderr before running the legacy hook +- "yes" (default): Git will silently run the legacy hook
Quoted 8 lines
Documentation/technical/config-based-hooks.txt line 150
Quoted 5 lines
Documentation/technical/config-based-hooks.txt new file +5
-0+In case this list is expanded in the future, if a value for `hook.runHookDir` is +given which Git does not recognize, Git should discard that config entry. For +example, if "warn" was specified at system level and "junk" was specified at +global level, Git would resolve the value to "warn"; if the only time the config +was set was to "junk", Git would use the default value of "yes".Hmph, instead of complaining "value 'junk' is not recognized" and erroring out? Why?
I think for the exact case you're describing above - where I forgot some useful combination of "run/don't run" and "warn/don't warn" (or, one could forsee, "first" e.g. to run the legacy hook early on instead of late, or "only" e.g. config hooks don't exist, etc etc), we add this flag, and some enterprise (like Jonathan N's team) distributes the new config to folks' system configs before 100% of machines are using the newer Git executable. It's a long shot :) but I'd rather be flexible than inflexible.
I will update the design doc anyway - I ended up implementing it as "complain about 'junk' and then do default behavior".
Quoted 16 lines
Documentation/technical/config-based-hooks.txt line 203
Quoted 7 lines
+[[stage-3]]
+==== Stage 3Documentation/technical/config-based-hooks.txt new file +4
-0+ +`.git/hooks` is removed from the template and the hook directory is considered +deprecated. To avoid breaking older repos, the default of `hook.runHookDir` is +not changed, and `find_hook()` is not removed.Presumably, we'll have documentation somewhere that instructs users (who were taught by slashdot and other site to add certain scripts under their .git/hooks/) how to do the equivalent without adding scripts in .git/hooks/ directory and instead using the config mechanism (e.g. "when told to add script X in .git/hooks/, read such an instruction as if telling you to do Y instead") by the time this happens? It probably makes sense to do so as part of stage-2, at which point the users are _ready_ to migrate.
Hm. Where should we distribute such a documentation? git-hook.txt? githooks.txt? I think it's a good idea, sure.
Quoted 11 lines
Quoted 11 lines
+[[security]]
+=== Security and repo configDocumentation/technical/config-based-hooks.txt new file +8
-0+ +Part of the motivation behind this refactor is to mitigate hooks as an attack +vector;footnote:[https://lore.kernel.org/git/[email protected]/] +however, as the design stands, users can still provide hooks in the repo-level +config, which is included when a repo is zipped and sent elsewhere. The +security of the repo-level config is still under discussion; this design +generally assumes the repo-level config is secure, which is not true yet. The +goal is to avoid an overcomplicated design to work around a problem which has
Quoted 8 lines
Quoted 1 line
+ceased to exist.
I doubt we want to claim anything about security as part of this series. As you say in the paragraph, .git/config and .git/hooks/ are equally (un)protected and if we decide to punt .git/config security, then not moving away from .git/hooks would not hurt security-wise, either (in other words, security is not a viable motivation behind this series).
Yeah, I think you are right that despite my best efforts to spin it that way, it just isn't a security win :) Ah well...
Quoted 8 lines
And if we stop advertising 'security merit' that does not exist, what remains? Isn't the biggest selling point that an identical set of hook configuration can be shared among multiple repositories, and it allows more than one hook scripts to be triggered by a single "hook event"? There may be other good things we should be able to sell the new mechanism to our users, and we do stress on them, which is done in the motivation section. So...
Ok.
I would like to keep this header and make it more clear that we didn't fully succeed in the wish to make zip attacks impossible, but I will remove it from the motivations section at the top. I also added a little more clarity (I hope) on the other pieces in the motivation section.
Quoted 7 lines
Quoted 4 lines
+.Comparison of alternatives
+|===
+|Feature |Config-based hooks |Hook directories |Status quoSorry, but I did not find this table particularly convincing.
Quoted 8 lines
The only thing I sense is a hand-wavy desire that "we could make it better than everybody else if we work on it in this area", which can apply equally for other approaches---they could enhance what they already have (e.g. "discoverability & documentation").
As a list of "these are the points we aspire to do better than other people", I think it is an excellent idea to have a table like this here in the documentation. But that is not a "comparison".
Ok. I'll see what I can do to frame it better.
Quoted 22 lines
Quoted 9 lines
+[[execution-ordering]]
+=== Execution orderingDocumentation/technical/config-based-hooks.txt new file +6
-0+ +We may find that config order is insufficient for some users; for example, +config order makes it difficult to add a new hook to the system or global config +which runs at the end of the hook list. A new ordering schema should be: + +1) Specified by a `hook.order` config, so that users will not unexpectedly seeDocumentation/technical/config-based-hooks.txt line 304
Quoted 10 lines
Documentation/technical/config-based-hooks.txt new file +10
-0+their order change; + +2) Either dependency or numerically based. + +Dependency-based ordering is prone to classic linked-list problems, like a +cycles and handling of missing dependencies. But, it paves the way for enabling +parallelization if some tasks truly depend on others. + +Numerical ordering makes it tricky for Git to generate suggested ordering +numbers for each command, but is easy to determine a definitive order.OK.
Quoted 5 lines
Have we decided what we do for hooks whose interface is to feed their input from their standard input? The current system, I think, just feeds the single hook by writing into a pipe to it, but if we were to drive multiple hooks, we'd need to write the same thing to each of these hook programs?
Yep. There are a few ways to do it:
1. Provide a file to be used as stdin. This way we just hook up that file's fd to each hook process's stdin fd. (Example:
https://lore.kernel.org/git/[email protected]
in builtin/am.c:run_post_rewrite_hook())
2. Provide a string_list in the run_hooks_opt struct; hook.c treats each entry in the string_list as a line to stdin, separated by a newline, and "replays" the list to each hook in order. (Example:
https://lore.kernel.org/git/[email protected])
3. Set up your own more complicated version of (2) by writing your own callback to provide "next line of stdin" based on a context pointer. The context pointer is per-hook-task. (Example:
https://lore.kernel.org/git/[email protected])
Quoted 7 lines
Do we have a plan to deal with hooks whose outcome is not just "yes/no", e.g. "proc-receive" hook that munges the list of refs to be updated and the new values for them, or "applypatch-msg" that munges the incoming proposed commit log message? Does the second hook work on the result of the first hook? Do the two hooks work on the vanilla state and their output have to agree with each other?
The only hook I found this way was 'proc-receive' itself - and in the end, because it's somewhat interactive with two-way communication with the caller, it didn't seem like there was a good way to reason about multiple hooks, like you say. So in the 'proc-receive' case, we disallow multiple hooks. See
https://lore.kernel.org/git/[email protected].
Quoted 15 lines
Documentation/technical/config-based-hooks.txt
Quoted 7 lines
+[[parallelization]]
+=== Parallelization with dependenciesDocumentation/technical/config-based-hooks.txt new file +4
-0+ +Currently hooks use a naive parallelization scheme or are run in series. But if +one hook depends on another's output, then users will want to specify those +dependencies.An untold assumption here is that the questions I asked earlier on having more than one hooks that is not just yes/no is something readers know the same answer, and that answer is "the outcome of the first hook is passed along (as if it were the input given by Git directly, if the first hook did not exist), to the second hook. It should be spelled out somewhere before the execution ordering section, I think.
I'll make an explicit callout about that case, thanks.
- Emily
PATCH v7 2/17 hook: scaffolding for git-hook subcommand
In reply to Emily ShafferIntroduce infrastructure for a new subcommand, git-hook, which will be used to ease config-based hook management. This command will handle parsing configs to compose a list of hooks to run for a given event, as well as adding or modifying hook configs in an interactive fashion.
---
Notes:
Since v4, mainly changed to RUN_SETUP_GENTLY so that 'git hook list' can be executed outside of a repo.
.gitignore | 1 + Documentation/git-hook.txt | 20 ++++++++++++++++++++ Makefile | 1 + builtin.h | 1 + builtin/hook.c | 21 +++++++++++++++++++++ command-list.txt | 1 + git.c | 1 + t/t1360-config-based-hooks.sh | 11 +++++++++++ 8 files changed, 57 insertions(+) create mode 100644 Documentation/git-hook.txt create mode 100644 builtin/hook.c create mode 100755 t/t1360-config-based-hooks.sh
.gitignore +1 -0
@@ -76,6 +76,7 @@ /git-grep /git-hash-object /git-help +/git-hook /git-http-backend /git-http-fetch /git-http-push
Documentation/git-hook.txt new file +20 -0
@@ -0,0 +1,20 @@ +git-hook(1) +=========== + +NAME +---- +git-hook - Manage configured hooks + +SYNOPSIS +-------- +[verse] +'git hook' + +DESCRIPTION +----------- +A placeholder command. Later, you will be able to list, add, and modify hooks +with this command. + +GIT +--- +Part of the linkgit:git[1] suite
Makefile +1 -0
@@ -1101,6 +1101,7 @@ BUILTIN_OBJS += builtin/get-tar-commit-id.o BUILTIN_OBJS += builtin/grep.o BUILTIN_OBJS += builtin/hash-object.o BUILTIN_OBJS += builtin/help.o +BUILTIN_OBJS += builtin/hook.o BUILTIN_OBJS += builtin/index-pack.o BUILTIN_OBJS += builtin/init-db.o BUILTIN_OBJS += builtin/interpret-trailers.o
builtin.h +1 -0
@@ -163,6 +163,7 @@ int cmd_get_tar_commit_id(int argc, const char **argv, const char *prefix);int cmd_grep (int argc,const char **argv,const char *prefix);int cmd_hash_object (int argc,const char **argv,const char *prefix);int cmd_help (int argc,const char **argv,const char *prefix); +int cmd_hook (int argc,const char **argv,const char *prefix);int cmd_index_pack (int argc,const char **argv,const char *prefix);int cmd_init_db (int argc,const char **argv,const char *prefix);int cmd_interpret_trailers (int argc,const char **argv,const char *prefix);
builtin/hook.c new file +21 -0
@@ -0,0 +1,21 @@ +#include "cache.h" + +#include "builtin.h" +#include "parse-options.h" + +static const char *const builtin_hook_usage[] = { +N_ ("git hook" ), +NULL +}; + +int cmd_hook (int argc,const char **argv,const char *prefix) +{ +struct option builtin_hook_options[] = { +OPT_END (), + }; + + argc =parse_options (argc, argv, prefix, builtin_hook_options, + builtin_hook_usage,0 ); + +return 0 ; +}
command-list.txt +1 -0
@@ -103,6 +103,7 @@ git-grep mainporcelain info git-gui mainporcelain git-hash-object plumbingmanipulators git-help ancillaryinterrogators complete +git-hook mainporcelain git-http-backend synchingrepositories git-http-fetch synchelpers git-http-push synchelpers
git.c +1 -0
@@ -525,6 +525,7 @@ static struct cmd_struct commands[] = { { "grep", cmd_grep, RUN_SETUP_GENTLY }, { "hash-object", cmd_hash_object }, { "help", cmd_help }, + { "hook", cmd_hook, RUN_SETUP_GENTLY }, { "index-pack", cmd_index_pack, RUN_SETUP_GENTLY | NO_PARSEOPT }, { "init", cmd_init_db }, { "init-db", cmd_init_db },
t/t1360-config-based-hooks.sh new file +11 -0
@@ -0,0 +1,11 @@ +#!/bin/bash + +test_description='config-managed multihooks, including git-hook command' + +. ./test-lib.sh + +test_expect_success 'git hook command does not crash' ' + git hook +' + +test_done
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
PATCH v7 3/17 hook: add list command
In reply to Emily ShafferTeach 'git hook list <hookname>', which checks the known configs in order to create an ordered list of hooks to run on a given hook event.
Multiple commands can be specified for a given hook by providing multiple "hook.<hookname>.command = <path-to-hook>" lines. Hooks will be run in config order. If more properties need to be set on a given hook in the future, commands can also be specified by providing "hook.<hookname>.command = <hookcmd-name>", as well as a "[hookcmd <hookcmd-name>]" subsection; at minimum, this subsection must contain a "hookcmd.<hookcmd-name>.command = <path-to-hook>" line.
For example:
$ git config --list | grep ^hook hook.pre-commit.command=baz hook.pre-commit.command=~/bar.sh hookcmd.baz.command=~/baz/from/hookcmd.sh
$ git hook list pre-commit global: ~/baz/from/hookcmd.sh local: ~/bar.sh
---
Notes:
Since v4, updated the sample in the commit message to reflect reality better.
Since v4, more work on the documentation. Also a slight change to the output format (space instead of tab).
Documentation/config/hook.txt | 9 +++ Documentation/git-hook.txt | 59 ++++++++++++++++- Makefile | 1 + builtin/hook.c | 56 +++++++++++++++-- hook.c | 115 ++++++++++++++++++++++++++++++++++ hook.h | 26 ++++++++ t/t1360-config-based-hooks.sh | 81 +++++++++++++++++++++++- 7 files changed, 338 insertions(+), 9 deletions(-) create mode 100644 Documentation/config/hook.txt create mode 100644 hook.c create mode 100644 hook.h
Documentation/config/hook.txt new file +9 -0
@@ -0,0 +1,9 @@ +hook.<command>.command:: + A command to execute during the <command> hook event. This can be an + executable on your device, a oneliner for your shell, or the name of a + hookcmd. See linkgit:git-hook[1]. + +hookcmd.<name>.command:: + A command to execute during a hook for which <name> has been specified + as a command. This can be an executable on your device or a oneliner for + your shell. See linkgit:git-hook[1].
Documentation/git-hook.txt +56 -3
68 lines not on this page · raw message
Makefile +1 -0
@@ -904,6 +904,7 @@ LIB_OBJS += grep.o LIB_OBJS += hashmap.o LIB_OBJS += help.o LIB_OBJS += hex.o +LIB_OBJS += hook.o LIB_OBJS += ident.o LIB_OBJS += json-writer.o LIB_OBJS += kwset.o
builtin/hook.c +52 -4
73 lines not on this page · raw message
hook.c new file +115 -0
115 lines not on this page · raw message
hook.h new file +26 -0
@@ -0,0 +1,26 @@ +#include "config.h" +#include "list.h" +#include "strbuf.h" + +struct hook +{ +struct list_head list; +/* +* Config file which holds the hook.*.command definition. +* (This has nothing to do with the hookcmd.<name>.* configs.) +*/ +enum config_scope origin; +/* The literal command to run. */ +struct strbuf command; +}; + +/* +* Provides a linked list of 'struct hook' detailing commands which should run +* in response to the 'hookname' event, in execution order. +*/ +struct list_head *hook_list (const struct strbuf *hookname); + +/* Free memory associated with a 'struct hook' */ +void free_hook (struct hook *ptr); +/* Empties the list at 'head', calling 'free_hook()' on each entry */ +void clear_hook_list (struct list_head *head);
t/t1360-config-based-hooks.sh +79 -2
87 lines not on this page · raw message
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
Re: [PATCH v7 03/17] hook: add list command
Quoted 10 lines
Teach 'git hook list <hookname>', which checks the known configs in order to create an ordered list of hooks to run on a given hook event.
Multiple commands can be specified for a given hook by providing multiple "hook.<hookname>.command = <path-to-hook>" lines. Hooks will be run in config order. If more properties need to be set on a given hook in the future, commands can also be specified by providing "hook.<hookname>.command = <hookcmd-name>", as well as a "[hookcmd <hookcmd-name>]" subsection; at minimum, this subsection must contain a "hookcmd.<hookcmd-name>.command = <path-to-hook>" line.
I learned later that this isn't true - in patch 6, the commit message and one of the tests therein describe being able to skip a previously inline command by making a hookcmd section of the same name and just specifying "skip = true" there (without any command).
Maybe just delete the "at minimum" part.
builtin/hook.c line 17
Quoted 5 lines
builtin/hook.c +3
-1+static int list (int argc,const char **argv,const char *prefix) { -struct option builtin_hook_options[] = { +struct list_head *head, *pos; +struct hook *item;
You asked for review on nits too so here's one: "item" should be declared in the list_for_each block. (That also makes it easier to see that we don't need to free it.)
hook.c line 4
Quoted 10 lines
hook.c new file +4
-0@@ -0,0 +1,115 @@ +#include "cache.h" + +#include "hook.h" +#include "config.h"
Usually we put all the includes together without any intervening blank lines.
Quoted 5 lines
hook.c new file +5
-0+static void append_or_move_hook (struct list_head *head,const char *command) +{ +struct list_head *pos =NULL , *tmp =NULL ; +struct hook *to_add =NULL ; +
hook.c line 28
Quoted 10 lines
hook.c new file +10
-0+/* +* remove the prior entry with this command; we'll replace it at the +* end. +*/ +list_for_each_safe (pos, tmp, head) { + struct hook *it = list_entry(pos, struct hook, list); +if (!strcmp (it->command.buf, command)) { +list_del (pos); +/* we'll simply move the hook to the end */ + to_add = it;
"break" here?
hook.c line 34
Quoted 6 lines
hook.c new file +6
-0+ } + } + +if (!to_add) { +/* adding a new hook, not moving an old one */ + to_add =xmalloc (sizeof (struct hook ));
Style is to write sizeof(*to_add), I think.
[snip]
hook.c line 61
Quoted 3 lines
hook.c new file +3
-0+struct hook_config_cb +{ +struct strbuf *hookname;
struct declarations have "{" not on a line on its own.
Also, "hookname" could just be a char *?
Quoted 6 lines
hook.c new file +6
-0+struct list_head *list; +}; + +static int hook_config_lookup (const char *key,const char *value,void *cb_data) +{ +struct hook_config_cb *data = cb_data;
hook.c line 77
Quoted 10 lines
hook.c new file +10
-0+const char *hook_key = data->hookname->buf; +struct list_head *head = data->list; + +if (!strcmp (key, hook_key)) { +const char *command = value; +struct strbuf hookcmd_name =STRBUF_INIT ; + +/* Check if a hookcmd with that name exists. */ +strbuf_addf (&hookcmd_name,"hookcmd.%s.command" , command); +git_config_get_value (hookcmd_name.buf, &command);
So we don't really care whether git_config_get_value returns 0 or 1 as long as it doesn't touch "command" if there is no such hookcmd. That fits with how git_config_get_value() is documented, so that's great. Perhaps better to document as:
If a hookcmd.%s.command config exists, replace the command with the value of that config. (If not, do nothing - git_config_get_value() is documented to not overwrite the value argument in this case.)
Quoted 11 lines
hook.c new file +11
-0+ +if (!command) { +strbuf_release (&hookcmd_name); +BUG ("git_config_get_value overwrote a string it shouldn't have" ); + } + +/* +* TODO: implement an option-getting callback, e.g. +* get configs by pattern hookcmd.$value.* +* for each key+value, do_callback(key, value, cb_data) +*/
hook.c line 98
Quoted 10 lines
hook.c new file +10
-0+ +append_or_move_hook (head, command); + +strbuf_release (&hookcmd_name); + } + +return 0 ; +} + +struct list_head *hook_list (const struct strbuf * hookname)
"const char *hookname" should suffice?
Also, search for "* " and replace with " *" where applicable (also in the .h file).
hook.c line 111
Quoted 13 lines
hook.c new file +13
-0+{ +struct strbuf hook_key =STRBUF_INIT ; +struct list_head *hook_head =xmalloc (sizeof (struct list_head )); +struct hook_config_cb cb_data = { &hook_key, hook_head }; + +INIT_LIST_HEAD (hook_head); + +if (!hookname) +return NULL ; + +strbuf_addf (&hook_key,"hook.%s.command" , hookname->buf); + +git_config (hook_config_lookup, (void *)&cb_data);
Do we need this void* cast?
Quoted 14 lines
hook.c new file +4
-0+ +strbuf_release (&hook_key); +return hook_head; +}hook.h new file +4
-0@@ -0,0 +1,26 @@ +#include "config.h" +#include "list.h" +#include "strbuf.h" +
hook.h line 14
Quoted 10 lines
hook.h new file +10
-0+struct hook +{ +struct list_head list; +/* +* Config file which holds the hook.*.command definition. +* (This has nothing to do with the hookcmd.<name>.* configs.) +*/ +enum config_scope origin; +/* The literal command to run. */ +struct strbuf command;
"char *" would suffice?
The tests look fine.
Re: [PATCH v7 03/17] hook: add list command
Quoted 12 lines · Jonathan Tan
Quoted 11 lines
Teach 'git hook list <hookname>', which checks the known configs in order to create an ordered list of hooks to run on a given hook event.
Multiple commands can be specified for a given hook by providing multiple "hook.<hookname>.command = <path-to-hook>" lines. Hooks will be run in config order. If more properties need to be set on a given hook in the future, commands can also be specified by providing "hook.<hookname>.command = <hookcmd-name>", as well as a "[hookcmd <hookcmd-name>]" subsection; at minimum, this subsection must contain a "hookcmd.<hookcmd-name>.command = <path-to-hook>" line.
Quoted 6 lines
I learned later that this isn't true - in patch 6, the commit message and one of the tests therein describe being able to skip a previously inline command by making a hookcmd section of the same name and just specifying "skip = true" there (without any command).
Maybe just delete the "at minimum" part.
Thanks, nice catch. I dropped "at minimum" and s/must/should/. Since I received feedback during the dogfood internally that there's no documentation on "hookcmd.<thing>.skip" whatsoever, I'll try to make a more detailed overview of hookcmd acting as an alias in the user-facing docs in that commit.
Quoted 10 lines
builtin/hook.c line 17
Quoted 6 lines
builtin/hook.c +3
-1+static int list (int argc,const char **argv,const char *prefix) { -struct option builtin_hook_options[] = { +struct list_head *head, *pos; +struct hook *item;You asked for review on nits too so here's one: "item" should be declared in the
list_for_eachblock. (That also makes it easier to see that we don't need to free it.)
Ok, done.
Quoted 14 lines
hook.c line 4
Quoted 11 lines
hook.c new file +4
-0@@ -0,0 +1,115 @@ +#include "cache.h" + +#include "hook.h" +#include "config.h" Usually we put all the includes together without any intervening blank lines.
Done.
Quoted 10 lines
Quoted 10 lines
hook.c new file +9
-0+static void append_or_move_hook (struct list_head *head,const char *command) +{ +struct list_head *pos =NULL , *tmp =NULL ; +struct hook *to_add =NULL ; + +/* +* remove the prior entry with this command; we'll replace it at the +* end. +*/
Quoted 8 lines
hook.c line 28
Quoted 6 lines
hook.c new file +6
-0+list_for_each_safe (pos, tmp, head) { + struct hook *it = list_entry(pos, struct hook, list); +if (!strcmp (it->command.buf, command)) { +list_del (pos); +/* we'll simply move the hook to the end */ + to_add = it;"break" here?
I think it's safe to do so, but I think I left it out in case duplicates do make it into the list somehow. But if they're always being inserted via "append_or_move_hook()" that should not be an issue, so I'll add the break.
In fact, fixing this exposed a bug. Later, I add using 'pos' instead of 'head':
list_add_tail(&to_add->list, pos);
When I'm guaranteed to iterate to the end of the list, that works fine, because in this implementation the last element of the list has "->next" set back to "head". But when 'pos' isn't sure to be at the end of the list, it breaks the list. Whoops.. :)
Quoted 9 lines
hook.c line 34
Quoted 7 lines
hook.c new file +6
-0+ } + } + +if (!to_add) { +/* adding a new hook, not moving an old one */ + to_add =xmalloc (sizeof (struct hook ));Style is to write
sizeof(*to_add), I think.
Sure.
Quoted 8 lines
[snip]
hook.c line 61
Quoted 3 lines
hook.c new file +3
-0+struct hook_config_cb +{ +struct strbuf *hookname;struct declarations have "{" not on a line on its own.
Sure, I can change it. Although, 'git grep -E "struct \w+$"' tells me there are a few other offenders :) But the count (60ish minus doc/relnotes references) is much lower than 'git grep -E "struct \w+ \{$"' (800+ matches) so I'm definitely doing it wrong :D
Quoted 2 lines
Also, "hookname" could just be a char *?
Hrmph. I think my C++ background is showing - to me, calling ".buf" (which is cheap) a few times is a small price to pay to get length info and so on for free. "hookname" can come from the user - "git hook (list|run) repo-special-magic-hook" - so I worry about leaving it as a raw char* with no size info associated. Am I being too paranoid?
Quoted 23 lines
Quoted 7 lines
hook.c new file +6
-0+struct list_head *list; +}; + +static int hook_config_lookup (const char *key,const char *value,void *cb_data) +{ +struct hook_config_cb *data = cb_data;hook.c line 77
Quoted 10 lines
hook.c new file +10
-0+const char *hook_key = data->hookname->buf; +struct list_head *head = data->list; + +if (!strcmp (key, hook_key)) { +const char *command = value; +struct strbuf hookcmd_name =STRBUF_INIT ; + +/* Check if a hookcmd with that name exists. */ +strbuf_addf (&hookcmd_name,"hookcmd.%s.command" , command); +git_config_get_value (hookcmd_name.buf, &command);So we don't really care whether
git_config_get_valuereturns 0 or 1 as long as it doesn't touch "command" if there is no such hookcmd. That fits with howgit_config_get_value()is documented, so that's great. Perhaps better to document as:
Quoted 3 lines
If a hookcmd.%s.command config exists, replace the command with the value of that config. (If not, do nothing -
git_config_get_value()is documented to not overwrite the value argument in this case.)
OK. Thanks, done.
Quoted 25 lines
Quoted 12 lines
hook.c new file +11
-0+ +if (!command) { +strbuf_release (&hookcmd_name); +BUG ("git_config_get_value overwrote a string it shouldn't have" ); + } + +/* +* TODO: implement an option-getting callback, e.g. +* get configs by pattern hookcmd.$value.* +* for each key+value, do_callback(key, value, cb_data) +*/ hook.c line 98
Quoted 10 lines
hook.c new file +10
-0+ +append_or_move_hook (head, command); + +strbuf_release (&hookcmd_name); + } + +return 0 ; +} + +struct list_head *hook_list (const struct strbuf * hookname)"const char *hookname" should suffice?
Quoted 2 lines
Also, search for "* " and replace with " *" where applicable (also in the .h file).
See above.
Quoted 16 lines
Quoted 4 lines
hook.c new file +3
-0+{ +struct strbuf hook_key =STRBUF_INIT ; +struct list_head *hook_head =xmalloc (sizeof (struct list_head ));hook.c line 111
Quoted 10 lines
hook.c new file +10
-0+struct hook_config_cb cb_data = { &hook_key, hook_head }; + +INIT_LIST_HEAD (hook_head); + +if (!hookname) +return NULL ; + +strbuf_addf (&hook_key,"hook.%s.command" , hookname->buf); + +git_config (hook_config_lookup, (void *)&cb_data);Do we need this void* cast?
Nope. Dropped, thanks.
Quoted 19 lines
Quoted 19 lines
hook.c new file +4
-0+ +strbuf_release (&hook_key); +return hook_head; +}hook.h new file +8
-0@@ -0,0 +1,26 @@ +#include "config.h" +#include "list.h" +#include "strbuf.h" + +struct hook +{ +struct list_head list; +/*
Quoted 8 lines
hook.h line 14
Quoted 6 lines
hook.h new file +6
-0+* Config file which holds the hook.*.command definition. +* (This has nothing to do with the hookcmd.<name>.* configs.) +*/ +enum config_scope origin; +/* The literal command to run. */ +struct strbuf command;"char *" would suffice?
Since 'command' comes directly from user input and is executed, I'm nervous about using a char* instead of a strbuf even more so than for the hookname.
Quoted 2 lines
The tests look fine.
Thanks for the nitpicky review, it was great! :)
- Emily
PATCH v7 4/17 hook: include hookdir hook in list
In reply to Emily ShafferHistorically, hooks are declared by placing an executable into $GIT_DIR/hooks/$HOOKNAME (or $HOOKDIR/$HOOKNAME). Although hooks taken from the config are more featureful than hooks placed in the $HOOKDIR, those hooks should not stop working for users who already have them.
Legacy hooks should be run directly, not in shell. We know that they are a path to an executable, not a oneliner script - and running them directly takes care of path quoting concerns for us for free.
---
Notes:
Newly split into its own commit since v4, and taking place much sooner.
An unfortunate side effect of adding this support *before* the hook.runHookDir support is that the labels on the list are not clear - because we aren't yet flagging which hooks are from the hookdir versus the config. I suppose we could move the addition of that field to the struct hook up to this patch, but it didn't make a lot of sense to me to do it just for cosmetic purposes.
builtin/hook.c | 18 ++++++++++++++---- hook.c | 15 +++++++++++++++ hook.h | 1 + t/t1360-config-based-hooks.sh | 19 +++++++++++++++++++ 4 files changed, 49 insertions(+), 4 deletions(-)
builtin/hook.c +14 -4
@@ -16,6 +16,7 @@ static int list(int argc, const char **argv, const char *prefix)struct list_head *head, *pos;struct hook *item;struct strbuf hookname =STRBUF_INIT ; +struct strbuf hookdir_annotation =STRBUF_INIT ;struct option list_options[] = {OPT_END (),
@@ -42,10 +43,17 @@ static int list(int argc, const char **argv, const char *prefix)list_for_each (pos, head) { item = list_entry(pos, struct hook, list); -if (item) -printf ("%s: %s\n" , -config_scope_name (item->origin), - item->command.buf); +if (item) { +/* Don't translate 'hookdir' - it matches the config */ +printf ("%s: %s%s\n" , + (item->from_hookdir + ?"hookdir" + :config_scope_name (item->origin)), + item->command.buf, + (item->from_hookdir + ? hookdir_annotation.buf + :"" )); + } }clear_hook_list (head);
@@ -62,6 +70,8 @@ int cmd_hook(int argc, const char **argv, const char *prefix)if (argc <2 )usage_with_options (builtin_hook_usage, builtin_hook_options); +git_config (git_default_config,NULL ); +if (!strcmp (argv[1 ],"list" ))return list (argc -1 , argv +1 , prefix);
hook.c +15 -0
@@ -2,6 +2,7 @@#include "hook.h" #include "config.h" +#include "run-command.h" void free_hook (struct hook *ptr) {
@@ -34,6 +35,7 @@ static void append_or_move_hook(struct list_head *head, const char *command) to_add =xmalloc (sizeof (struct hook ));strbuf_init (&to_add->command,0 );strbuf_addstr (&to_add->command, command); + to_add->from_hookdir =0 ; }/* re-set the scope so we show where an override was specified */
@@ -100,6 +102,7 @@ struct list_head* hook_list(const struct strbuf* hookname)struct strbuf hook_key =STRBUF_INIT ;struct list_head *hook_head =xmalloc (sizeof (struct list_head ));struct hook_config_cb cb_data = { &hook_key, hook_head }; +const char *legacy_hook_path =NULL ;INIT_LIST_HEAD (hook_head);
@@ -110,6 +113,18 @@ struct list_head* hook_list(const struct strbuf* hookname)git_config (hook_config_lookup, (void *)&cb_data); +if (have_git_dir ()) + legacy_hook_path =find_hook (hookname->buf); + +/* Unconditionally add legacy hook, but annotate it. */ +if (legacy_hook_path) { +struct hook *legacy_hook; + +append_or_move_hook (hook_head,absolute_path (legacy_hook_path)); + legacy_hook = list_entry(hook_head->prev, struct hook, list); + legacy_hook->from_hookdir =1 ; + } +strbuf_release (&hook_key);return hook_head; }
hook.h +1 -0
@@ -12,6 +12,7 @@ struct hookenum config_scope origin;/* The literal command to run. */ struct strbuf command; +int from_hookdir; };/*
t/t1360-config-based-hooks.sh +19 -0
@@ -23,6 +23,14 @@ setup_hookcmd () { test_config_global hookcmd.abc.command "/path/abc" --add } +setup_hookdir () { + mkdir .git/hooks + write_script .git/hooks/pre-commit <<-EOF + echo \"Legacy Hook\" + EOF + test_when_finished rm -rf .git/hooks +} + test_expect_success 'git hook rejects commands without a mode' ' test_must_fail git hook pre-commit '
@@ -85,4 +93,15 @@ test_expect_success 'git hook list reorders on duplicate commands' ' test_cmp expected actual ' +test_expect_success 'git hook list shows hooks from the hookdir' ' + setup_hookdir && + + cat >expected <<-EOF && + hookdir: $(pwd)/.git/hooks/pre-commit + EOF + + git hook list pre-commit >actual && + test_cmp expected actual +' + test_done
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
Re: [PATCH v7 04/17] hook: include hookdir hook in list
Quoted 4 lines
Historically, hooks are declared by placing an executable into $GIT_DIR/hooks/$HOOKNAME (or $HOOKDIR/$HOOKNAME). Although hooks taken from the config are more featureful than hooks placed in the $HOOKDIR, those hooks should not stop working for users who already have them.
Maybe explicitly add that we're listing them in the list with a "hookdir:" prefix.
Quoted 3 lines
Legacy hooks should be run directly, not in shell. We know that they are a path to an executable, not a oneliner script - and running them directly takes care of path quoting concerns for us for free.
Not sure what this paragraph is doing here.
builtin/hook.c static int list(int argc, const char **argv, const char *prefix) · line 19
Quoted 9 lines
builtin/hook.c +1
-0@@ -16,6 +16,7 @@ static int list(int argc, const char **argv, const char *prefix)struct list_head *head, *pos;struct hook *item;struct strbuf hookname =STRBUF_INIT ; +struct strbuf hookdir_annotation =STRBUF_INIT ;
Right now this is never set? Maybe hold off on adding this until we set something.
builtin/hook.c struct list_head* hook_list(const struct strbuf* hookname) · line 123
Quoted 12 lines
builtin/hook.c +8
-0@@ -110,6 +113,18 @@ struct list_head* hook_list(const struct strbuf* hookname)git_config (hook_config_lookup, (void *)&cb_data); +if (have_git_dir ()) + legacy_hook_path =find_hook (hookname->buf); + +/* Unconditionally add legacy hook, but annotate it. */ +if (legacy_hook_path) { +struct hook *legacy_hook; + +append_or_move_hook (hook_head,absolute_path (legacy_hook_path));
Both find_hook() and absolute_path() use static buffers to hold their return values, which makes me a bit nervous. Perhaps put them all under the same "if (have_git_dir())" so that it's clearer that we're not supposed to insert code arbitrarily between their invocation and their usage.
hook.h struct hook · line 15
Quoted 9 lines
hook.h +1
-0@@ -12,6 +12,7 @@ struct hookenum config_scope origin;/* The literal command to run. */ struct strbuf command; +int from_hookdir;
unsigned from_hookdir : 1?
The tests look good.
Re: [PATCH v7 04/17] hook: include hookdir hook in list
Quoted 8 lines · Jonathan Tan
Quoted 5 lines
Historically, hooks are declared by placing an executable into $GIT_DIR/hooks/$HOOKNAME (or $HOOKDIR/$HOOKNAME). Although hooks taken from the config are more featureful than hooks placed in the $HOOKDIR, those hooks should not stop working for users who already have them.
Maybe explicitly add that we're listing them in the list with a "hookdir:" prefix.
Sure.
Quoted 6 lines
Quoted 4 lines
Legacy hooks should be run directly, not in shell. We know that they are a path to an executable, not a oneliner script - and running them directly takes care of path quoting concerns for us for free.
Not sure what this paragraph is doing here.
Yep, this is an artifact of the review process (explaining why I didn't do something weird, which I did in an earlier version, but now it doesn't make sense to mention it at all). Deleted.
Quoted 13 lines
builtin/hook.c static int list(int argc, const char **argv, const char *prefix) · line 19
Quoted 10 lines
builtin/hook.c +1
-0@@ -16,6 +16,7 @@ static int list(int argc, const char **argv, const char *prefix)struct list_head *head, *pos;struct hook *item;struct strbuf hookname =STRBUF_INIT ; +struct strbuf hookdir_annotation =STRBUF_INIT ;Right now this is never set? Maybe hold off on adding this until we set something.
Yeah, that makes sense. Will do.
Quoted 5 lines
Quoted 5 lines
builtin/hook.c +0
-0@@ -110,6 +113,18 @@ struct list_head* hook_list(const struct strbuf* hookname)git_config (hook_config_lookup, (void *)&cb_data);
Quoted 14 lines
builtin/hook.c struct list_head* hook_list(const struct strbuf* hookname) · line 123
Quoted 8 lines
builtin/hook.c +8
-0+if (have_git_dir ()) + legacy_hook_path =find_hook (hookname->buf); + +/* Unconditionally add legacy hook, but annotate it. */ +if (legacy_hook_path) { +struct hook *legacy_hook; + +append_or_move_hook (hook_head,absolute_path (legacy_hook_path));Both
find_hook()andabsolute_path()use static buffers to hold their return values, which makes me a bit nervous. Perhaps put them all under the same "if (have_git_dir())" so that it's clearer that we're not supposed to insert code arbitrarily between their invocation and their usage.
Oh, that's a cool way to indicate that. Thanks, I did that, and learned something new!
Quoted 12 lines
hook.h struct hook · line 15
Quoted 10 lines
hook.h +1
-0@@ -12,6 +12,7 @@ struct hookenum config_scope origin;/* The literal command to run. */ struct strbuf command; +int from_hookdir;unsigned
from_hookdir: 1?
Sure. It doesn't make a difference now but I see that would be nice for futureproofing.
Quoted 2 lines
The tests look good.
Thanks.
PATCH v7 5/17 hook: respect hook.runHookDir
In reply to Emily ShafferInclude hooks specified in the hook directory in the list of hooks to run. These hooks do need to be treated differently from config-specified ones - they do not need to run in a shell, and later on may be disabled or warned about based on a config setting.
Because they are at least as local as the local config, we'll run them last - to keep the hook execution order from global to local.
---
Notes:
Newly split into its own commit since v4, and taking place much sooner.
An unfortunate side effect of adding this support *before* the hook.runHookDir support is that the labels on the list are not clear - because we aren't yet flagging which hooks are from the hookdir versus the config. I suppose we could move the addition of that field to the struct hook up to this patch, but it didn't make a lot of sense to me to do it just for cosmetic purposes.
Documentation/config/hook.txt | 5 ++++ builtin/hook.c | 54 +++++++++++++++++++++++++++++++++-- hook.c | 21 ++++++++++++++ hook.h | 15 ++++++++++ t/t1360-config-based-hooks.sh | 43 ++++++++++++++++++++++++++++ 5 files changed, 135 insertions(+), 3 deletions(-)
Documentation/config/hook.txt +5 -0
@@ -7,3 +7,8 @@ hookcmd.<name>.command:: A command to execute during a hook for which <name> has been specified as a command. This can be an executable on your device or a oneliner for your shell. See linkgit:git-hook[1]. + +hook.runHookDir:: + Controls how hooks contained in your hookdir are executed. Can be any of + "yes", "warn", "interactive", or "no". Defaults to "yes". See + linkgit:git-hook[1] and linkgit:git-config[1] "core.hooksPath").
builtin/hook.c +51 -3
79 lines not on this page · raw message
hook.c +21 -0
@@ -97,6 +97,27 @@ static int hook_config_lookup(const char *key, const char *value, void *cb_data)return 0 ; } +enum hookdir_opt configured_hookdir_opt (void ) +{ +const char *key; +if (git_config_get_value ("hook.runhookdir" , &key)) +return HOOKDIR_YES ;/* by default, just run it. */ + +if (!strcmp (key,"no" )) +return HOOKDIR_NO ; + +if (!strcmp (key,"yes" )) +return HOOKDIR_YES ; + +if (!strcmp (key,"warn" )) +return HOOKDIR_WARN ; + +if (!strcmp (key,"interactive" )) +return HOOKDIR_INTERACTIVE ; + +return HOOKDIR_UNKNOWN ; +} +struct list_head *hook_list (const struct strbuf * hookname) {struct strbuf hook_key =STRBUF_INIT ;
hook.h +15 -0
@@ -21,6 +21,21 @@ struct hook*/ struct list_head *hook_list (const struct strbuf *hookname); +enum hookdir_opt +{ +HOOKDIR_NO , +HOOKDIR_WARN , +HOOKDIR_INTERACTIVE , +HOOKDIR_YES , +HOOKDIR_UNKNOWN , +}; + +/* +* Provides the hookdir_opt specified in the config without consulting any +* command line arguments. +*/ +enum hookdir_opt configured_hookdir_opt (void ); +/* Free memory associated with a 'struct hook' */ void free_hook (struct hook *ptr);/* Empties the list at 'head', calling 'free_hook()' on each entry */
t/t1360-config-based-hooks.sh +43 -0
47 lines not on this page · raw message
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
Re: [PATCH v7 05/17] hook: respect hook.runHookDir
Quoted 7 lines
Include hooks specified in the hook directory in the list of hooks to run. These hooks do need to be treated differently from config-specified ones - they do not need to run in a shell, and later on may be disabled or warned about based on a config setting.
Because they are at least as local as the local config, we'll run them last - to keep the hook execution order from global to local.
This commit message doesn't seem to match the code change. Firstly, we're teaching hook.runHookDir, not respecting it (since it did not exist before this commit), and it's about showing it in "list" and not about running it at all.
Perhaps just "hook: teach hook.runHookDir" as the subject and as the body:
For now, this just affects the output of "git hook list". In the future, this will affect the behavior of "git hook run" and when Git runs hooks before or after its operations.
builtin/hook.c static int list(int argc, const char **argv, const char *prefix) · line 54
Quoted 9 lines
builtin/hook.c +9
-0+switch (should_run_hookdir) { +case HOOKDIR_NO : +strbuf_addstr (&hookdir_annotation,_ (" (will not run)" )); +break ; +case HOOKDIR_INTERACTIVE : +strbuf_addstr (&hookdir_annotation,_ (" (will prompt)" )); +break ; +case HOOKDIR_WARN : +case HOOKDIR_UNKNOWN :
Hmm...UNKNOWN is the same as WARN? This doesn't agree with what is said in patch 1:
+In case this list is expanded in the future, if a value for hook.runHookDir is +given which Git does not recognize, Git should discard that config entry. For +example, if "warn" was specified at system level and "junk" was specified at +global level, Git would resolve the value to "warn"; if the only time the config +was set was to "junk", Git would use the default value of "yes".
But having said that, I would prefer if Git just errored out in this case.
The rest looks good.
Re: [PATCH v7 05/17] hook: respect hook.runHookDir
Quoted 6 lines · Jonathan Tan
Quoted 6 lines
Include hooks specified in the hook directory in the list of hooks to run. These hooks do need to be treated differently from config-specified ones - they do not need to run in a shell, and later on may be disabled or warned about based on a config setting.
Quoted 7 lines
Quoted 2 lines
Because they are at least as local as the local config, we'll run them last - to keep the hook execution order from global to local.
This commit message doesn't seem to match the code change. Firstly, we're teaching hook.runHookDir, not respecting it (since it did not exist before this commit), and it's about showing it in "list" and not about running it at all.
Yeah, thanks for noticing. Now that I'm rereading it, it looks like this is the old commit message for "include hookdir hook in list". Yikes.
Quoted 21 lines
Perhaps just "hook: teach hook.runHookDir" as the subject and as the body:
For now, this just affects the output of "git hook list". In the future, this will affect the behavior of "git hook run" and when Git runs hooks before or after its operations.
builtin/hook.c static int list(int argc, const char **argv, const char *prefix) · line 54
Quoted 9 lines
builtin/hook.c +9
-0+switch (should_run_hookdir) { +case HOOKDIR_NO : +strbuf_addstr (&hookdir_annotation,_ (" (will not run)" )); +break ; +case HOOKDIR_INTERACTIVE : +strbuf_addstr (&hookdir_annotation,_ (" (will prompt)" )); +break ; +case HOOKDIR_WARN : +case HOOKDIR_UNKNOWN :Hmm...UNKNOWN is the same as WARN? This doesn't agree with what is said in patch 1:
Quoted 8 lines
+In case this list is expanded in the future, if a value for
hook.runHookDiris +given which Git does not recognize, Git should discard that config entry. For +example, if "warn" was specified at system level and "junk" was specified at +global level, Git would resolve the value to "warn"; if the only time the config +was set was to "junk", Git would use the default value of "yes".But having said that, I would prefer if Git just errored out in this case.
Eh, I think I'd still like it to be tolerant. Thanks for keeping me honest :) I added a test to enforce the tolerant behavior, too.
Quoted 2 lines
The rest looks good.
Thanks.
PATCH v7 6/17 hook: implement hookcmd.<name>.skip
In reply to Emily ShafferIf a user wants a specific repo to skip execution of a hook which is set at a global or system level, they can now do so by specifying 'skip' in their repo config:
~/.gitconfig
[hook.pre-commit]
command = skippable-oneliner
command = skippable-hookcmd
[hookcmd.skippable-hookcmd]
command = foo.sh
$GIT_DIR/.git/config
[hookcmd.skippable-oneliner]
skip = true
[hookcmd.skippable-hookcmd]
skip = true
Later it may make sense to add an option like "hookcmd.<name>.<hook-event>-skip" - but for simplicity, let's start with a universal skip setting like this.
---
Notes:
In addition to being handy for turning off global hooks one project doesn't care about, this setting will be necessary much later for the 'proc-receive' hook, which can only cope with up to one hook being specified.
New since v4.
During the Google team's review club I was reminded about this whole 'skip' option I never implemented. It's true that it's impossible to exclude a given hook without this; however, I think I have some more work to do on it, so consider it RFC for now and tell me what you think :)
- Emily
During the Google team's review club this week I was reminded about this whole 'skip' option I never implemented. It's true that it's impossible to exclude a given hook without this; however, I think we have some more work to do on it, so consider it RFC for now and tell me what you think :)
- Emily
hook.c | 37 +++++++++++++++++++++++++---------- t/t1360-config-based-hooks.sh | 23 ++++++++++++++++++++++ 2 files changed, 50 insertions(+), 10 deletions(-)
hook.c +27 -10
@@ -12,23 +12,24 @@ void free_hook(struct hook *ptr) } } -static void append_or_move_hook (struct list_head *head,const char *command) +static struct hook *find_hook_by_command (struct list_head *head,const char *command) {struct list_head *pos =NULL , *tmp =NULL ; -struct hook *to_add =NULL ; +struct hook *found =NULL ; -/* -* remove the prior entry with this command; we'll replace it at the -* end. -*/ list_for_each_safe (pos, tmp, head) { struct hook *it = list_entry(pos, struct hook, list);if (!strcmp (it->command.buf, command)) {list_del (pos); -/* we'll simply move the hook to the end */ - to_add = it; + found = it; } } +return found; +} + +static void append_or_move_hook (struct list_head *head,const char *command) +{ +struct hook *to_add =find_hook_by_command (head, command);if (!to_add) {/* adding a new hook, not moving an old one */
@@ -41,7 +42,7 @@ static void append_or_move_hook(struct list_head *head, const char *command)/* re-set the scope so we show where an override was specified */ to_add->origin =current_config_scope (); -list_add_tail (&to_add->list, pos); +list_add_tail (&to_add->list, head); }static void remove_hook (struct list_head *to_remove)
@@ -73,8 +74,18 @@ static int hook_config_lookup(const char *key, const char *value, void *cb_data)if (!strcmp (key, hook_key)) {const char *command = value;struct strbuf hookcmd_name =STRBUF_INIT ; +int skip =0 ; + +/* +* Check if we're removing that hook instead. Hookcmds are +* removed by name, and inlined hooks are removed by command +* content. +*/ +strbuf_addf (&hookcmd_name,"hookcmd.%s.skip" , command); +git_config_get_bool (hookcmd_name.buf, &skip);/* Check if a hookcmd with that name exists. */ +strbuf_reset (&hookcmd_name);strbuf_addf (&hookcmd_name,"hookcmd.%s.command" , command);git_config_get_value (hookcmd_name.buf, &command);
@@ -89,7 +100,13 @@ static int hook_config_lookup(const char *key, const char *value, void *cb_data)* for each key+value, do_callback(key, value, cb_data) */ -append_or_move_hook (head, command); +if (skip) { +struct hook *to_remove =find_hook_by_command (head, command); +if (to_remove) +remove_hook (&(to_remove->list)); + }else { +append_or_move_hook (head, command); + }strbuf_release (&hookcmd_name); }
t/t1360-config-based-hooks.sh +23 -0
@@ -132,6 +132,29 @@ test_expect_success 'hook.runHookDir = warn is respected by list' ' test_i18ncmp expected actual ' +test_expect_success 'git hook list removes skipped hookcmd' ' + setup_hookcmd && + test_config hookcmd.abc.skip "true" --add && + + cat >expected <<-EOF && + no commands configured for hook '\''pre-commit'\'' + EOF + + git hook list pre-commit >actual && + test_i18ncmp expected actual +' + +test_expect_success 'git hook list removes skipped inlined hook' ' + setup_hooks && + test_config hookcmd."$ROOT/path/ghi".skip "true" --add && + + cat >expected <<-EOF && + global: $ROOT/path/def + EOF + + git hook list pre-commit >actual && + test_cmp expected actual +' test_expect_success 'hook.runHookDir = interactive is respected by list' ' setup_hookdir &&
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
Re: [PATCH v7 06/17] hook: implement hookcmd.<name>.skip
Quoted 3 lines
If a user wants a specific repo to skip execution of a hook which is set at a global or system level, they can now do so by specifying 'skip' in their repo config:
Usually the present tense describes the situation before the commit, so maybe s/they can now do so/they will be able to do so/.
hook.c void free_hook(struct hook *ptr) · line 15
Quoted 2 lines
hook.c +1
-1-static void append_or_move_hook (struct list_head *head,const char *command) +static struct hook *find_hook_by_command (struct list_head *head,const char *command)
"* " -> " *"
[snip tests]
For the tests, I thought of the case in which we skip a hookcmd that was never specified as a hook, but that's probably not very useful.
Re: [PATCH v7 06/17] hook: implement hookcmd.<name>.skip
Quoted 7 lines · Jonathan Tan
Quoted 4 lines
If a user wants a specific repo to skip execution of a hook which is set at a global or system level, they can now do so by specifying 'skip' in their repo config:
Usually the present tense describes the situation before the commit, so maybe s/they can now do so/they will be able to do so/.
Sure.
Quoted 5 lines
hook.c void free_hook(struct hook *ptr) · line 15
Quoted 3 lines
hook.c +1
-1-static void append_or_move_hook (struct list_head *head,const char *command) +static struct hook *find_hook_by_command (struct list_head *head,const char *command)"* " -> " *"
Thanks.
Quoted 5 lines
[snip tests]
For the tests, I thought of the case in which we skip a hookcmd that was never specified as a hook, but that's probably not very useful.
Ah, it might be useful to make sure we don't choke trying to remove something that isn't there - I'll add one.
By the way, I got feedback from Googlers using config hooks that "skip" isn't actually documented anywhere public-facing. For v8 I've added a section on it to Documentation/git-hook.txt as well as to Documentation/config/hook.txt.
- Emily
PATCH v7 7/17 parse-options: parse into strvec
In reply to Emily Shafferparse-options already knows how to read into a string_list, and it knows how to read into an strvec as a passthrough (that is, including the argument as well as its value). string_list and strvec serve similar purposes but are somewhat painful to convert between; so, let's teach parse-options to read values of string arguments directly into an strvec without preserving the argument name.
This is useful if collecting generic arguments to pass through to another command, for example, 'git hook run --arg "--quiet" --arg "--format=pretty" some-hook'. The resulting strvec would contain { "--quiet", "--format=pretty" }.
The implementation is based on that of OPT_STRING_LIST.
---
Notes:
Since v4, fixed one or two more places where I missed the argv_array->strvec
rename.Documentation/technical/api-parse-options.txt | 5 +++++ parse-options-cb.c | 16 ++++++++++++++++ parse-options.h | 4 ++++ 3 files changed, 25 insertions(+)
Documentation/technical/api-parse-options.txt +5 -0
@@ -173,6 +173,11 @@ There are some macros to easily define options: The string argument is stored as an element in `string_list`. Use of `--no-option` will clear the list of preceding values. +`OPT_STRVEC(short, long, &struct strvec, arg_str, description)`:: + Introduce an option with a string argument. + The string argument is stored as an element in `strvec`. + Use of `--no-option` will clear the list of preceding values. + `OPT_INTEGER(short, long, &int_var, description)`:: Introduce an option with integer argument. The integer is put into `int_var`.
parse-options-cb.c +16 -0
@@ -207,6 +207,22 @@ int parse_opt_string_list(const struct option *opt, const char *arg, int unset)return 0 ; } +int parse_opt_strvec (const struct option *opt,const char *arg,int unset) +{ +struct strvec *v = opt->value; + +if (unset) { +strvec_clear (v); +return 0 ; + } + +if (!arg) +return -1 ; + +strvec_push (v, arg); +return 0 ; +} +int parse_opt_noop_cb (const struct option *opt,const char *arg,int unset) {return 0 ;
parse-options.h +4 -0
@@ -177,6 +177,9 @@ struct option {#define OPT_STRING_LIST (s, l, v, a, h) \ { OPTION_CALLBACK, (s), (l), (v), (a), \ (h), 0, &parse_opt_string_list } +#define OPT_STRVEC (s, l, v, a, h) \ + { OPTION_CALLBACK, (s), (l), (v), (a), \ + (h), 0, &parse_opt_strvec }#define OPT_UYN (s, l, v, h) { OPTION_CALLBACK, (s), (l), (v), NULL, \ (h), PARSE_OPT_NOARG, &parse_opt_tertiary }#define OPT_EXPIRY_DATE (s, l, v, h) \
@@ -296,6 +299,7 @@ int parse_opt_commits(const struct option *, const char *, int);int parse_opt_commit (const struct option *,const char *,int );int parse_opt_tertiary (const struct option *,const char *,int );int parse_opt_string_list (const struct option *,const char *,int ); +int parse_opt_strvec (const struct option *,const char *,int );int parse_opt_noop_cb (const struct option *,const char *,int ); enum parse_opt_result parse_opt_unknown_cb(struct parse_opt_ctx_t *ctx, const struct option *,
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
PATCH v7 8/17 hook: add 'run' subcommand
In reply to Emily ShafferIn order to enable hooks to be run as an external process, by a standalone Git command, or by tools which wrap Git, provide an external means to run all configured hook commands for a given hook event.
For now, the hook commands will run in config order, in series. As alternate ordering or parallelism is supported in the future, we should add knobs to use those to the command line as well.
As with the legacy hook implementation, all stdout generated by hook commands is redirected to stderr. Piping from stdin is not yet supported.
Legacy hooks (those present in $GITDIR/hooks) are run at the end of the execution list. For now, there is no way to disable them.
Users may wish to provide hook commands like 'git config hook.pre-commit.command "~/linter.sh --pre-commit"'. To enable this, config-defined hooks are run in a shell. (Since hooks in $GITDIR/hooks can't be specified with included arguments or paths which need expansion like this, they are run without a shell instead.)
---
Notes:
Since v4, updated the docs, and did less local application of single quotes. In order for hookdir hooks to run successfully with a space in the path, though, they must not be run with 'sh -c'. So we can treat the hookdir hooks specially, and warn users via doc about special considerations for configured hooks with spaces in their path.
Documentation/git-hook.txt | 31 +++++++++- builtin/hook.c | 48 ++++++++++++++- hook.c | 112 ++++++++++++++++++++++++++++++++++ hook.h | 32 ++++++++++ t/t1360-config-based-hooks.sh | 65 +++++++++++++++++++- 5 files changed, 281 insertions(+), 7 deletions(-)
Documentation/git-hook.txt +29 -2
@@ -9,11 +9,12 @@ SYNOPSIS -------- [verse] 'git hook' list <hook-name> +'git hook' run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] <hook-name> DESCRIPTION ----------- -You can list configured hooks with this command. Later, you will be able to run, -add, and modify hooks with this command. +You can list and run configured hooks with this command. Later, you will be able +to add and modify hooks with this command. This command parses the default configuration files for sections `hook` and `hookcmd`. `hook` is used to describe the commands which will be run during a
@@ -64,6 +65,32 @@ in the order they should be run, and print the config scope where the relevant `hook.<hook-name>.command` was specified, not the `hookcmd` (if applicable). This output is human-readable and the format is subject to change over time. +run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] `<hook-name>`:: + +Runs hooks configured for `<hook-name>`, in the same order displayed by `git +hook list`. Hooks configured this way are run prepended with `sh -c`, so paths +containing special characters or spaces should be wrapped in single quotes: +`command = '/my/path with spaces/script.sh' some args`. + +OPTIONS +------- +--run-hookdir:: + Overrides the hook.runHookDir config. Must be 'yes', 'warn', + 'interactive', or 'no'. Specifies how to handle hooks located in the Git + hook directory (core.hooksPath). + +-a:: +--arg:: + Only valid for `run`. ++ +Specify arguments to pass to every hook that is run. + +-e:: +--env:: + Only valid for `run`. ++ +Specify environment variables to set for every hook that is run. + CONFIGURATION ------------- include::config/hook.txt[]
builtin/hook.c +46 -2
77 lines not on this page · raw message
hook.c +112 -0
127 lines not on this page · raw message
hook.h +32 -0
@@ -1,6 +1,7 @@#include "config.h" #include "list.h" #include "strbuf.h" +#include "strvec.h" struct hook {
@@ -36,6 +37,37 @@ enum hookdir_opt*/ enum hookdir_opt configured_hookdir_opt (void ); +struct run_hooks_opt +{ +/* Environment vars to be set for each hook */ +struct strvec env; + +/* Args to be passed to each hook */ +struct strvec args; + +/* +* How should the hookdir be handled? +* Leave the RUN_HOOKS_OPT_INIT default in most cases; this only needs +* to be overridden if the user can override it at the command line. +*/ +enum hookdir_opt run_hookdir; +}; + +#define RUN_HOOKS_OPT_INIT { \ + .env = STRVEC_INIT, \ + .args = STRVEC_INIT, \ + .run_hookdir = configured_hookdir_opt() \ +} + +void run_hooks_opt_init (struct run_hooks_opt *o); +void run_hooks_opt_clear (struct run_hooks_opt *o); + +/* +* Runs all hooks associated to the 'hookname' event in order. Each hook will be +* passed 'env' and 'args'. +*/ +int run_hooks (const char *hookname,struct run_hooks_opt *options); +/* Free memory associated with a 'struct hook' */ void free_hook (struct hook *ptr);/* Empties the list at 'head', calling 'free_hook()' on each entry */
t/t1360-config-based-hooks.sh +62 -3
89 lines not on this page · raw message
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
Re: [PATCH v7 08/17] hook: add 'run' subcommand
Quoted 8 lines
In order to enable hooks to be run as an external process, by a standalone Git command, or by tools which wrap Git, provide an external means to run all configured hook commands for a given hook event.
For now, the hook commands will run in config order, in series. As alternate ordering or parallelism is supported in the future, we should add knobs to use those to the command line as well.
Quoted 6 lines
As with the legacy hook implementation, all stdout generated by hook commands is redirected to stderr. Piping from stdin is not yet supported.
Legacy hooks (those present in $GITDIR/hooks) are run at the end of the execution list. For now, there is no way to disable them.
Not true anymore now that we have hook.runHookDir :-)
Documentation/git-hook.txt in the order they should be run, and print the config scope where the relevant · line 73
Quoted 10 lines
Documentation/git-hook.txt +6
-0@@ -64,6 +65,32 @@ in the order they should be run, and print the config scope where the relevant `hook.<hook-name>.command` was specified, not the `hookcmd` (if applicable). This output is human-readable and the format is subject to change over time. +run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] `<hook-name>`:: + +Runs hooks configured for `<hook-name>`, in the same order displayed by `git +hook list`. Hooks configured this way are run prepended with `sh -c`, so paths +containing special characters or spaces should be wrapped in single quotes: +`command = '/my/path with spaces/script.sh' some args`.
I learned recently that this may not work the way I expect [1], so you might want to specifically call this out for someone who knows how run-command and running-with-shell works.
Quoted 13 lines
hook.c +9
-0@@ -135,6 +136,56 @@ enum hookdir_opt configured_hookdir_opt(void)return HOOKDIR_UNKNOWN ; } +static int should_include_hookdir (const char *path,enum hookdir_opt cfg) +{ +struct strbuf prompt =STRBUF_INIT ; +/* +* If the path doesn't exist, don't bother adding the empty hook and +* don't bother checking the config or prompting the user. +*/ +if (!path) +return 0 ;
hook.c enum hookdir_opt configured_hookdir_opt(void) · line 157
Quoted 10 lines
hook.c +10
-0+ +switch (cfg) + { +case HOOKDIR_NO : +return 0 ; +case HOOKDIR_UNKNOWN : +fprintf (stderr, +_ ("Unrecognized value for 'hook.runHookDir'. " +"Is there a typo? " )); +/* FALLTHROUGH */
Same comment (about UNKNOWN and defaulting to WARN instead of YES) as in one of the previous patches.
Quoted 30 lines
hook.c +30
-0+case HOOKDIR_WARN : +fprintf (stderr,_ ("Running legacy hook at '%s'\n" ), + path); +return 1 ; +case HOOKDIR_INTERACTIVE : +do { +/* +* TRANSLATORS: Make sure to include [Y] and [n] +* in your translation. Only English input is +* accepted. Default option is "yes". +*/ +fprintf (stderr,_ ("Run '%s'? [Yn] " ), path); +git_read_line_interactively (&prompt); +strbuf_tolower (&prompt); +if (starts_with (prompt.buf,"n" )) { +strbuf_release (&prompt); +return 0 ; + }else if (starts_with (prompt.buf,"y" )) { +strbuf_release (&prompt); +return 1 ; + } +/* otherwise, we didn't understand the input */ + }while (prompt.len);/* an empty reply means "Yes" */ +strbuf_release (&prompt); +return 1 ; +case HOOKDIR_YES : +default : +return 1 ; + } +}
[snip]
Quoted 12 lines
hook.c +12
-0+int run_hooks (const char *hookname,struct run_hooks_opt *options) +{ +struct strbuf hookname_str =STRBUF_INIT ; +struct list_head *to_run, *pos =NULL , *tmp =NULL ; +int rc =0 ; + +if (!options) +BUG ("a struct run_hooks_opt must be provided to run_hooks" ); + +strbuf_addstr (&hookname_str, hookname); + + to_run =hook_list (&hookname_str);
hook.c struct list_head* hook_list(const struct strbuf* hookname) · line 255
Quoted 10 lines
hook.c +10
-0+ +list_for_each_safe (pos, tmp, to_run) { +struct child_process hook_proc =CHILD_PROCESS_INIT ; + struct hook *hook = list_entry(pos, struct hook, list); + + hook_proc.env = options->env.v; + hook_proc.no_stdin =1 ; + hook_proc.stdout_to_stderr =1 ; + hook_proc.trace2_hook_name = hook->command.buf; + hook_proc.use_shell =1 ;
I think this is based on run_hook_ve() in run-command.c - could we refactor that to avoid duplication of code?
Quoted 25 lines
hook.c +25
-0+ +if (hook->from_hookdir) { +if (!should_include_hookdir (hook->command.buf, options->run_hookdir)) +continue ; +/* +* Commands from the config could be oneliners, but we know +* for certain that hookdir commands are not. +*/ + hook_proc.use_shell =0 ; + } + +/* add command */ +strvec_push (&hook_proc.args, hook->command.buf); + +/* +* add passed-in argv, without expanding - let the user get back +* exactly what they put in +*/ +strvec_pushv (&hook_proc.args, options->args.v); + + rc |=run_command (&hook_proc); + } + +return rc; +}
[snip]
Quoted 11 lines
hook.h +11
-0+struct run_hooks_opt +{ +/* Environment vars to be set for each hook */ +struct strvec env; + +/* Args to be passed to each hook */ +struct strvec args; + +/* +* How should the hookdir be handled? +* Leave the RUN_HOOKS_OPT_INIT default in most cases; this only needs
hook.h enum hookdir_opt · line 60
Quoted 10 lines
hook.h +10
-0+* to be overridden if the user can override it at the command line. +*/ +enum hookdir_opt run_hookdir; +}; + +#define RUN_HOOKS_OPT_INIT { \ + .env = STRVEC_INIT, \ + .args = STRVEC_INIT, \ + .run_hookdir = configured_hookdir_opt() \ +}
I don't think we have function invocations in our declarations like this. Maybe stick to just using run_hooks_opt_init().
[snip tests]
The tests look good.
Re: [PATCH v7 08/17] hook: add 'run' subcommand
Quoted 9 lines · Jonathan Tan
Quoted 9 lines
In order to enable hooks to be run as an external process, by a standalone Git command, or by tools which wrap Git, provide an external means to run all configured hook commands for a given hook event.
For now, the hook commands will run in config order, in series. As alternate ordering or parallelism is supported in the future, we should add knobs to use those to the command line as well.
Quoted 8 lines
Quoted 6 lines
As with the legacy hook implementation, all stdout generated by hook commands is redirected to stderr. Piping from stdin is not yet supported.
Legacy hooks (those present in $GITDIR/hooks) are run at the end of the execution list. For now, there is no way to disable them.
Not true anymore now that we have hook.runHookDir :-)
Updated.
Quoted 5 lines
Quoted 5 lines
Documentation/git-hook.txt +0
-0@@ -64,6 +65,32 @@ in the order they should be run, and print the config scope where the relevant `hook.<hook-name>.command` was specified, not the `hookcmd` (if applicable). This output is human-readable and the format is subject to change over time.
Quoted 10 lines
Documentation/git-hook.txt in the order they should be run, and print the config scope where the relevant · line 73
Quoted 6 lines
Documentation/git-hook.txt +6
-0+run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] `<hook-name>`:: + +Runs hooks configured for `<hook-name>`, in the same order displayed by `git +hook list`. Hooks configured this way are run prepended with `sh -c`, so paths +containing special characters or spaces should be wrapped in single quotes: +`command = '/my/path with spaces/script.sh' some args`.I learned recently that this may not work the way I expect [1], so you might want to specifically call this out for someone who knows how run-command and running-with-shell works.
I think it might be good enough to say "may be prepended" instead - the quoting advice (wrap your paths) still holds.
Quoted 21 lines
[1] https://lore.kernel.org/git/[email protected]/
Quoted 18 lines
hook.c +14
-0@@ -135,6 +136,56 @@ enum hookdir_opt configured_hookdir_opt(void)return HOOKDIR_UNKNOWN ; } +static int should_include_hookdir (const char *path,enum hookdir_opt cfg) +{ +struct strbuf prompt =STRBUF_INIT ; +/* +* If the path doesn't exist, don't bother adding the empty hook and +* don't bother checking the config or prompting the user. +*/ +if (!path) +return 0 ; + +switch (cfg) + { +case HOOKDIR_NO : +return 0 ;
Quoted 8 lines
hook.c enum hookdir_opt configured_hookdir_opt(void) · line 157
Quoted 5 lines
hook.c +5
-0+case HOOKDIR_UNKNOWN : +fprintf (stderr, +_ ("Unrecognized value for 'hook.runHookDir'. " +"Is there a typo? " )); +/* FALLTHROUGH */ Same comment (about UNKNOWN and defaulting to WARN instead of YES) as in one of the previous patches.
Like in the previous patch, I opted to make it match the design doc (UNKNOWN matches default, aka YES). I left the typo warning, though, as it might be useful for someone trying to debug why "itneractive" isn't working as they expect.
Quoted 51 lines
Quoted 31 lines
hook.c +30
-0+case HOOKDIR_WARN : +fprintf (stderr,_ ("Running legacy hook at '%s'\n" ), + path); +return 1 ; +case HOOKDIR_INTERACTIVE : +do { +/* +* TRANSLATORS: Make sure to include [Y] and [n] +* in your translation. Only English input is +* accepted. Default option is "yes". +*/ +fprintf (stderr,_ ("Run '%s'? [Yn] " ), path); +git_read_line_interactively (&prompt); +strbuf_tolower (&prompt); +if (starts_with (prompt.buf,"n" )) { +strbuf_release (&prompt); +return 0 ; + }else if (starts_with (prompt.buf,"y" )) { +strbuf_release (&prompt); +return 1 ; + } +/* otherwise, we didn't understand the input */ + }while (prompt.len);/* an empty reply means "Yes" */ +strbuf_release (&prompt); +return 1 ; +case HOOKDIR_YES : +default : +return 1 ; + } +}[snip]
Quoted 17 lines
hook.c +17
-0+int run_hooks (const char *hookname,struct run_hooks_opt *options) +{ +struct strbuf hookname_str =STRBUF_INIT ; +struct list_head *to_run, *pos =NULL , *tmp =NULL ; +int rc =0 ; + +if (!options) +BUG ("a struct run_hooks_opt must be provided to run_hooks" ); + +strbuf_addstr (&hookname_str, hookname); + + to_run =hook_list (&hookname_str); + +list_for_each_safe (pos, tmp, to_run) { +struct child_process hook_proc =CHILD_PROCESS_INIT ; + struct hook *hook = list_entry(pos, struct hook, list); +
Quoted 8 lines
hook.c struct list_head* hook_list(const struct strbuf* hookname) · line 255
Quoted 5 lines
hook.c +5
-0+ hook_proc.env = options->env.v; + hook_proc.no_stdin =1 ; + hook_proc.stdout_to_stderr =1 ; + hook_proc.trace2_hook_name = hook->command.buf; + hook_proc.use_shell =1 ;I think this is based on
run_hook_ve()inrun-command.c- could we refactor that to avoid duplication of code?
Hm. At the end of part II of this series "run_hook_ve()" is deleted entirely; the implementation of "run_hooks" diverges significantly as the series progresses (supporting stdin/stdout, etc) so I'd rather not try to keep one of them based on the other, as I think it'll be more complicated than it seems in this patch.
Quoted 45 lines
Quoted 26 lines
hook.c +25
-0+ +if (hook->from_hookdir) { +if (!should_include_hookdir (hook->command.buf, options->run_hookdir)) +continue ; +/* +* Commands from the config could be oneliners, but we know +* for certain that hookdir commands are not. +*/ + hook_proc.use_shell =0 ; + } + +/* add command */ +strvec_push (&hook_proc.args, hook->command.buf); + +/* +* add passed-in argv, without expanding - let the user get back +* exactly what they put in +*/ +strvec_pushv (&hook_proc.args, options->args.v); + + rc |=run_command (&hook_proc); + } + +return rc; +}[snip]
Quoted 16 lines
hook.h +16
-0+struct run_hooks_opt +{ +/* Environment vars to be set for each hook */ +struct strvec env; + +/* Args to be passed to each hook */ +struct strvec args; + +/* +* How should the hookdir be handled? +* Leave the RUN_HOOKS_OPT_INIT default in most cases; this only needs +* to be overridden if the user can override it at the command line. +*/ +enum hookdir_opt run_hookdir; +}; +
Quoted 8 lines
hook.h enum hookdir_opt · line 60
Quoted 5 lines
hook.h +5
-0+#define RUN_HOOKS_OPT_INIT { \ + .env = STRVEC_INIT, \ + .args = STRVEC_INIT, \ + .run_hookdir = configured_hookdir_opt() \ +}I don't think we have function invocations in our declarations like this. Maybe stick to just using
run_hooks_opt_init().
Sure.
Quoted 4 lines
[snip tests]
The tests look good.
Thanks for the detailed review.
- Emily
PATCH v7 9/17 hook: replace find_hook() with hook_exists()
In reply to Emily ShafferAdd a helper to easily determine whether any hooks exist for a given hook event.
Many callers want to check whether some state could be modified by a hook; that check should include the config-based hooks as well. Optimize by checking the config directly. Since commands which execute hooks might want to take args to replace 'hook.runHookDir', let 'hook_exists()' mirror the behavior of 'hook.runHookDir'.
--- builtin/bugreport.c | 4 ++-- hook.c | 15 +++++++++++++++ hook.h | 9 +++++++++ 3 files changed, 26 insertions(+), 2 deletions(-)
builtin/bugreport.c +2 -2
@@ -3,7 +3,7 @@#include "strbuf.h" #include "help.h" #include "compat/compiler.h" -#include "run-command.h" +#include "hook.h" static void get_system_info (struct strbuf *sys_info)
@@ -82,7 +82,7 @@ static void get_populated_hooks(struct strbuf *hook_info, int nongit) }for (i =0 ; i <ARRAY_SIZE (hook); i++) -if (find_hook (hook[i])) +if (hook_exists (hook[i],configured_hookdir_opt ()))strbuf_addf (hook_info,"%s\n" , hook[i]); }
hook.c +15 -0
@@ -225,6 +225,21 @@ void run_hooks_opt_init(struct run_hooks_opt *o) o->run_hookdir =configured_hookdir_opt (); } +int hook_exists (const char *hookname,enum hookdir_opt should_run_hookdir) +{ +const char *value =NULL ;/* throwaway */ +struct strbuf hook_key =STRBUF_INIT ; + +int could_run_hookdir = (should_run_hookdir ==HOOKDIR_INTERACTIVE || + should_run_hookdir ==HOOKDIR_WARN || + should_run_hookdir ==HOOKDIR_YES ) + && !!find_hook (hookname); + +strbuf_addf (&hook_key,"hook.%s.command" , hookname); + +return (!git_config_get_value (hook_key.buf, &value)) || could_run_hookdir; +} +void run_hooks_opt_clear (struct run_hooks_opt *o) {strvec_clear (&o->env);
hook.h +9 -0
@@ -62,6 +62,15 @@ struct run_hooks_optvoid run_hooks_opt_init (struct run_hooks_opt *o);void run_hooks_opt_clear (struct run_hooks_opt *o); +/* +* Returns 1 if any hooks are specified in the config or if a hook exists in the +* hookdir. Typically, invoke hook_exsts() like: +* hook_exists(hookname, configured_hookdir_opt()); +* Like with run_hooks, if you take a --run-hookdir flag, reflect that +* user-specified behavior here instead. +*/ +int hook_exists (const char *hookname,enum hookdir_opt should_run_hookdir); +/* * Runs all hooks associated to the 'hookname' event in order. Each hook will be * passed 'env' and 'args'.
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
Re: [PATCH v7 09/17] hook: replace find_hook() with hook_exists()
Quoted 8 lines
Add a helper to easily determine whether any hooks exist for a given hook event.
Many callers want to check whether some state could be modified by a hook; that check should include the config-based hooks as well. Optimize by checking the config directly. Since commands which execute hooks might want to take args to replace 'hook.runHookDir', let '
hook_exists()' mirror the behavior of 'hook.runHookDir'.
The text makes sense, but the title might better be "introduce hook_exists()" instead of "replace", since find_hook() is still around.
Also maybe briefly mention the future plans - e.g. in the future, no code will use find_hook() except <whatever the hook-internal functions are>, because all of them will use hook_exists() and run_hook().
hook.h struct run_hooks_opt · line 72
Quoted 8 lines
hook.h +8
-0+/* +* Returns 1 if any hooks are specified in the config or if a hook exists in the +* hookdir. Typically, invoke hook_exsts() like: +* hook_exists(hookname, configured_hookdir_opt()); +* Like with run_hooks, if you take a --run-hookdir flag, reflect that +* user-specified behavior here instead. +*/ +int hook_exists (const char *hookname,enum hookdir_opt should_run_hookdir);
I wonder if enum hookdir_opt should support a "unspecified" instead, in which case hook_exists() will automatically read the config (instead of relying on the caller to call configured_hookdir_opt()), but I see that this patch set is version 7 and perhaps this design point has already been discussed.
Re: [PATCH v7 09/17] hook: replace find_hook() with hook_exists()
Quoted 17 lines · Jonathan Tan
Quoted 9 lines
Add a helper to easily determine whether any hooks exist for a given hook event.
Many callers want to check whether some state could be modified by a hook; that check should include the config-based hooks as well. Optimize by checking the config directly. Since commands which execute hooks might want to take args to replace 'hook.runHookDir', let '
hook_exists()' mirror the behavior of 'hook.runHookDir'.The text makes sense, but the title might better be "introduce
hook_exists()" instead of "replace", sincefind_hook()is still around.Also maybe briefly mention the future plans - e.g. in the future, no code will use
find_hook()except <whatever the hook-internal functions are>, because all of them will usehook_exists()andrun_hook().
Quoted 14 lines
hook.h struct run_hooks_opt · line 72
Quoted 8 lines
hook.h +8
-0+/* +* Returns 1 if any hooks are specified in the config or if a hook exists in the +* hookdir. Typically, invoke hook_exsts() like: +* hook_exists(hookname, configured_hookdir_opt()); +* Like with run_hooks, if you take a --run-hookdir flag, reflect that +* user-specified behavior here instead. +*/ +int hook_exists (const char *hookname,enum hookdir_opt should_run_hookdir);I wonder if enum
hookdir_optshould support a "unspecified" instead, in which casehook_exists()will automatically read the config (instead of relying on the caller to callconfigured_hookdir_opt()), but I see that this patch set is version 7 and perhaps this design point has already been discussed.
No, I don't think it has been. So you mean something like:
enum hookdir_opt
{
HOOKDIR_NO,
HOOKDIR_WARN,
HOOKDIR_INTERACTIVE,
HOOKDIR_YES,
HOOKDIR_USE_CFG,
HOOKDIR_UNKNOWN,
};(name subject to quibbling) and then reimaginingconfigured_hookdir_opt() to something like:
enum hookdir_opt resolve_hookdir_opt(enum hookdir_opt o) {
if (o != HOOKDIR_USE_CFG)
return o; /* former contents of configured_hookdir_opt here */
}
I like that, if nobody has complaints.
- Emily
Re: [PATCH v7 09/17] hook: replace find_hook() with hook_exists()
In reply to Jonathan TanQuoted 4 lines · Jonathan Tan
Quoted 4 lines
Add a helper to easily determine whether any hooks exist for a given hook event.
Quoted 8 lines
Quoted 5 lines
Many callers want to check whether some state could be modified by a hook; that check should include the config-based hooks as well. Optimize by checking the config directly. Since commands which execute hooks might want to take args to replace 'hook.runHookDir', let '
hook_exists()' mirror the behavior of 'hook.runHookDir'.The text makes sense, but the title might better be "introduce
hook_exists()" instead of "replace", sincefind_hook()is still around.
Yep. Also, removed the one instance I did replace in this commit for some reason (builtin/bugreport.c). Not sure why that's here and not later on in part II.
Quoted 4 lines
Also maybe briefly mention the future plans - e.g. in the future, no code will use
find_hook()except <whatever the hook-internal functions are>, because all of them will usehook_exists()andrun_hook().
Nice, good point - I'll do that.
Quoted 15 lines
hook.h struct run_hooks_opt · line 72
Quoted 9 lines
hook.h +8
-0+/* +* Returns 1 if any hooks are specified in the config or if a hook exists in the +* hookdir. Typically, invoke hook_exsts() like: +* hook_exists(hookname, configured_hookdir_opt()); +* Like with run_hooks, if you take a --run-hookdir flag, reflect that +* user-specified behavior here instead. +*/ +int hook_exists (const char *hookname,enum hookdir_opt should_run_hookdir);I wonder if enum
hookdir_optshould support a "unspecified" instead, in which casehook_exists()will automatically read the config (instead of relying on the caller to callconfigured_hookdir_opt()), but I see that this patch set is version 7 and perhaps this design point has already been discussed.
Nope, and I think that's a pretty neat idea. I've done so.
PATCH v7 10/17 hook: support passing stdin to hooks
In reply to Emily ShafferSome hooks (such as post-rewrite) need to take input via stdin. Previously, callers provided stdin to hooks by setting run-command.h:child_process.in, which takes a FD. Callers would open the file in question themselves before calling run-command(). However, since we will now need to seek to the front of the file and read it again for every hook which runs, hook.h:run_command() takes a path and handles FD management itself. Since this file is opened for read only, it should not prevent later parallel execution support.
On the frontend, this is supported by asking for a file path, rather than by reading stdin. Reading directly from stdin would involve caching the entire stdin (to memory or to disk) and reading it back from the beginning to each hook. We'd want to support cases like insufficient memory or storage for the file. While this may prove useful later, for now the path of least resistance is to just ask the user to make this interim file themselves.
--- Documentation/git-hook.txt | 11 +++++++++-- builtin/hook.c | 5 ++++- hook.c | 7 ++++++- hook.h | 9 +++++++-- t/t1360-config-based-hooks.sh | 24 ++++++++++++++++++++++++ 5 files changed, 50 insertions(+), 6 deletions(-)
Documentation/git-hook.txt +9 -2
@@ -9,7 +9,8 @@ SYNOPSIS -------- [verse] 'git hook' list <hook-name> -'git hook' run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] <hook-name> +'git hook' run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] [--to-stdin=<path>] + <hook-name> DESCRIPTION -----------
@@ -65,7 +66,7 @@ in the order they should be run, and print the config scope where the relevant `hook.<hook-name>.command` was specified, not the `hookcmd` (if applicable). This output is human-readable and the format is subject to change over time. -run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] `<hook-name>`:: +run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] [--to-stdin=<path>] `<hook-name>`:: Runs hooks configured for `<hook-name>`, in the same order displayed by `git hook list`. Hooks configured this way are run prepended with `sh -c`, so paths
@@ -91,6 +92,12 @@ Specify arguments to pass to every hook that is run. + Specify environment variables to set for every hook that is run. +--to-stdin:: + Only valid for `run`. ++ +Specify a file which will be streamed into stdin for every hook that is run. +Each hook will receive the entire file from beginning to EOF. + CONFIGURATION ------------- include::config/hook.txt[]
builtin/hook.c +4 -1
@@ -9,7 +9,8 @@static const char *const builtin_hook_usage[] = {N_ ("git hook list <hookname>" ), -N_ ("git hook run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] <hookname>" ), +N_ ("git hook run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...]" +"[--to-stdin=<path>] <hookname>" ),NULL };
@@ -97,6 +98,8 @@ static int run(int argc, const char **argv, const char *prefix) N_("environment variables for hook to use")),OPT_STRVEC ('a' ,"arg" , &opt.args,N_ ("args" ),N_ ("argument to pass to hook" )), +OPT_STRING (0 ,"to-stdin" , &opt.path_to_stdin,N_ ("path" ), +N_ ("file to read into hooks' stdin" )), OPT_END(), };
hook.c +6 -1
@@ -263,8 +263,13 @@ int run_hooks(const char *hookname, struct run_hooks_opt *options)struct child_process hook_proc =CHILD_PROCESS_INIT ; struct hook *hook = list_entry(pos, struct hook, list); +/* reopen the file for stdin; run_command closes it. */ +if (options->path_to_stdin) + hook_proc.in =xopen (options->path_to_stdin,O_RDONLY ); +else + hook_proc.no_stdin =1 ; + hook_proc.env = options->env.v; - hook_proc.no_stdin =1 ; hook_proc.stdout_to_stderr =1 ; hook_proc.trace2_hook_name = hook->command.buf; hook_proc.use_shell =1 ;
hook.h +7 -2
@@ -51,11 +51,15 @@ struct run_hooks_opt* to be overridden if the user can override it at the command line. */ enum hookdir_opt run_hookdir; + +/* Path to file which should be piped to stdin for each hook */ +const char *path_to_stdin; };#define RUN_HOOKS_OPT_INIT { \ - .env = STRVEC_INIT, \ + .env = STRVEC_INIT, \ .args = STRVEC_INIT, \ + .path_to_stdin = NULL, \ .run_hookdir = configured_hookdir_opt() \ }
@@ -73,7 +77,8 @@ int hook_exists(const char *hookname, enum hookdir_opt should_run_hookdir);/* * Runs all hooks associated to the 'hookname' event in order. Each hook will be -* passed 'env' and 'args'. +* passed 'env' and 'args'. The file at 'stdin_path' will be closed and reopened +* for each hook that runs. */ int run_hooks (const char *hookname,struct run_hooks_opt *options);
t/t1360-config-based-hooks.sh +24 -0
@@ -229,4 +229,28 @@ test_expect_success 'out-of-repo runs excluded' ' nongit test_must_fail git hook run pre-commit ' +test_expect_success 'stdin to multiple hooks' ' + git config --add hook.test.command "xargs -P1 -I% echo a%" && + git config --add hook.test.command "xargs -P1 -I% echo b%" && + test_when_finished "test_unconfig hook.test.command" && + + cat >input <<-EOF && + 1 + 2 + 3 + EOF + + cat >expected <<-EOF && + a1 + a2 + a3 + b1 + b2 + b3 + EOF + + git hook run --to-stdin=input test 2>actual && + test_cmp expected actual +' + test_done
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
PATCH v7 11/17 run-command: allow stdin for run_processes_parallel
In reply to Emily ShafferWhile it makes sense not to inherit stdin from the parent process to avoid deadlocking, it's not necessary to completely ban stdin to children. An informed user should be able to configure stdin safely. By setting some_child.process.no_stdin=1 before calling get_next_task() we provide a reasonable default behavior but enable users to set up stdin streaming for themselves during the callback.
some_child.process.stdout_to_stderr, however, remains unmodifiable by get_next_task() - the rest of the run_processes_parallel() API depends on child output in stderr.
--- run-command.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
run-command.c +3 -1
@@ -1683,6 +1683,9 @@ static int pp_start_one(struct parallel_processes *pp)if (i == pp->max_processes)BUG ("bookkeeping is hard" ); +/* disallow by default, but allow users to set up stdin if they wish */ + pp->children[i].process.no_stdin =1 ; + code = pp->get_next_task(&pp->children[i].process, &pp->children[i].err, pp->data,
@@ -1694,7 +1697,6 @@ static int pp_start_one(struct parallel_processes *pp) } pp->children[i].process.err =-1 ; pp->children[i].process.stdout_to_stderr =1 ; - pp->children[i].process.no_stdin =1 ;if (start_command (&pp->children[i].process)) { code = pp->start_failure(&pp->children[i].err,
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
Re: [PATCH v7 11/17] run-command: allow stdin for run_processes_parallel
run-command.c static int pp_start_one(struct parallel_processes *pp) · line 1688
Quoted 11 lines
run-command.c +3
-0@@ -1683,6 +1683,9 @@ static int pp_start_one(struct parallel_processes *pp)if (i == pp->max_processes)BUG ("bookkeeping is hard" ); +/* disallow by default, but allow users to set up stdin if they wish */ + pp->children[i].process.no_stdin =1 ; +
This makes sense. May be worth a more detailed comment, e.g.:
By default, do not inherit stdin from the parent process. (If not, all children would share it!) Users may overwrite this by having the get_next_task function assign 0 to no_stdin and an appropriate integer to in.
Re: [PATCH v7 11/17] run-command: allow stdin for run_processes_parallel
Quoted 15 lines · Jonathan Tan
run-command.c static int pp_start_one(struct parallel_processes *pp) · line 1688
Quoted 12 lines
run-command.c +3
-0@@ -1683,6 +1683,9 @@ static int pp_start_one(struct parallel_processes *pp)if (i == pp->max_processes)BUG ("bookkeeping is hard" ); +/* disallow by default, but allow users to set up stdin if they wish */ + pp->children[i].process.no_stdin =1 ; +This makes sense. May be worth a more detailed comment, e.g.:
Quoted 4 lines
By default, do not inherit stdin from the parent process. (If not, all children would share it!) Users may overwrite this by having the
get_next_taskfunction assign 0 tono_stdinand an appropriate integer to in.
Thanks, took it slightly modified:
/*
* By default, do not inherit stdin from the parent process - otherwise,
* all children would share stdin! Users may overwrite this to provide
* something to the child's stdin by having their 'get_next_task'
* callback assign 0 to .no_stdin and an appropriate integer to .in. */
- Emily
PATCH v7 12/17 hook: allow parallel hook execution
In reply to Emily ShafferIn many cases, there's no reason not to allow hooks to execute in parallel. run_processes_parallel() is well-suited - it's a task queue that runs its housekeeping in series, which means users don't need to worry about thread safety on their callback data. True multithreaded execution with the async_* functions isn't necessary here. Synchronous hook execution can be achieved by only allowing 1 job to run at a time.
Teach run_hooks() to use that function for simple hooks which don't require stdin or capture of stderr.
---
Notes:
Per AEvar's request - parallel hook execution on day zero.
In most ways run_processes_parallel() worked great for me - but it didn't have great support for hooks where we pipe to and from. I had to add this support later in the series.
Since I modified an existing and in-use library I'd appreciate a keen look on these patches.
- Emily
Documentation/config/hook.txt | 5 ++ Documentation/git-hook.txt | 14 +++- builtin/hook.c | 6 +- hook.c | 142 ++++++++++++++++++++++++++-------- hook.h | 28 ++++++- 5 files changed, 157 insertions(+), 38 deletions(-)
Documentation/config/hook.txt +5 -0
@@ -12,3 +12,8 @@ hook.runHookDir:: Controls how hooks contained in your hookdir are executed. Can be any of "yes", "warn", "interactive", or "no". Defaults to "yes". See linkgit:git-hook[1] and linkgit:git-config[1] "core.hooksPath"). + +hook.jobs:: + Specifies how many hooks can be run simultaneously during parallelized + hook execution. If unspecified, defaults to the number of processors on + the current system.
Documentation/git-hook.txt +12 -2
@@ -10,7 +10,7 @@ SYNOPSIS [verse] 'git hook' list <hook-name> 'git hook' run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] [--to-stdin=<path>] - <hook-name> + [(-j|--jobs) <n>] <hook-name> DESCRIPTION -----------
@@ -66,7 +66,7 @@ in the order they should be run, and print the config scope where the relevant `hook.<hook-name>.command` was specified, not the `hookcmd` (if applicable). This output is human-readable and the format is subject to change over time. -run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] [--to-stdin=<path>] `<hook-name>`:: +run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] [--to-stdin=<path>] [(-j|--jobs)<n>] `<hook-name>`:: Runs hooks configured for `<hook-name>`, in the same order displayed by `git hook list`. Hooks configured this way are run prepended with `sh -c`, so paths
@@ -98,6 +98,16 @@ Specify environment variables to set for every hook that is run. Specify a file which will be streamed into stdin for every hook that is run. Each hook will receive the entire file from beginning to EOF. +-j:: +--jobs:: + Only valid for `run`. ++ +Specify how many hooks to run simultaneously. If this flag is not specified, use +the value of the `hook.jobs` config. If the config is not specified, use the +number of CPUs on the current system. Some hooks may be ineligible for +parallelization: for example, 'commit-msg' intends hooks modify the commit +message body and cannot be parallelized. + CONFIGURATION ------------- include::config/hook.txt[]
builtin/hook.c +4 -2
@@ -10,7 +10,7 @@static const char *const builtin_hook_usage[] = {N_ ("git hook list <hookname>" ),N_ ("git hook run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...]" -"[--to-stdin=<path>] <hookname>" ), +"[--to-stdin=<path>] [(-j|--jobs) <count>] <hookname>" ),NULL };
@@ -90,7 +90,7 @@ static int list(int argc, const char **argv, const char *prefix)static int run (int argc,const char **argv,const char *prefix) {struct strbuf hookname =STRBUF_INIT ; -struct run_hooks_opt opt =RUN_HOOKS_OPT_INIT ; +struct run_hooks_opt opt =RUN_HOOKS_OPT_INIT_ASYNC ;int rc =0 ;struct option run_options[] = {
@@ -100,6 +100,8 @@ static int run(int argc, const char **argv, const char *prefix) N_("argument to pass to hook")),OPT_STRING (0 ,"to-stdin" , &opt.path_to_stdin,N_ ("path" ),N_ ("file to read into hooks' stdin" )), +OPT_INTEGER ('j' ,"jobs" , &opt.jobs, +N_ ("run up to <n> hooks simultaneously" )), OPT_END(), };
hook.c +109 -33
173 lines not on this page · raw message
hook.h +27 -1
@@ -37,6 +37,9 @@ enum hookdir_opt*/ enum hookdir_opt configured_hookdir_opt (void ); +/* Provides the number of threads to use for parallel hook execution. */ +int configured_hook_jobs (void ); +struct run_hooks_opt {/* Environment vars to be set for each hook */
@@ -54,15 +57,38 @@ struct run_hooks_opt/* Path to file which should be piped to stdin for each hook */ const char *path_to_stdin; + +/* Number of threads to parallelize across */ +int jobs; }; -#define RUN_HOOKS_OPT_INIT { \ +/* +* Callback provided to feed_pipe_fn and consume_sideband_fn. +*/ +struct hook_cb_data { +int rc; +struct list_head *head; +struct list_head *run_me; +struct run_hooks_opt *options; +}; + +#define RUN_HOOKS_OPT_INIT_SYNC { \ .env = STRVEC_INIT, \ .args = STRVEC_INIT, \ .path_to_stdin = NULL, \ + .jobs = 1, \ .run_hookdir = configured_hookdir_opt() \ } +#define RUN_HOOKS_OPT_INIT_ASYNC { \ + .env = STRVEC_INIT, \ + .args = STRVEC_INIT, \ + .path_to_stdin = NULL, \ + .jobs = configured_hook_jobs(), \ + .run_hookdir = configured_hookdir_opt() \ +} + +void run_hooks_opt_init (struct run_hooks_opt *o);void run_hooks_opt_clear (struct run_hooks_opt *o);
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
Re: [PATCH v7 12/17] hook: allow parallel hook execution
Quoted 10 lines
In many cases, there's no reason not to allow hooks to execute in parallel.
run_processes_parallel()is well-suited - it's a task queue that runs its housekeeping in series, which means users don't need to worry about thread safety on their callback data. True multithreaded execution with the async_* functions isn't necessary here. Synchronous hook execution can be achieved by only allowing 1 job to run at a time.Teach
run_hooks()to use that function for simple hooks which don't require stdin or capture of stderr.
Which hooks would be run in parallel, and which hooks in series? I don't see code that distinguishes between them.
Quoted 7 lines
Signed-off-by Emily Shaffer <[email protected]>---
Notes:
Per AEvar's request - parallel hook execution on day zero.
Quoted 6 lines
In most ways
run_processes_parallel()worked great for me - but it didn't have great support for hooks where we pipe to and from. I had to add this support later in the series.Since I modified an existing and in-use library I'd appreciate a keen look on these patches.
What is the existing and in-use library that you're modifying?
Quoted 36 lines
hook.c +32
-0@@ -246,11 +255,96 @@ void run_hooks_opt_clear(struct run_hooks_opt *o)strvec_clear (&o->args); } + +static int pick_next_hook (struct child_process *cp, +struct strbuf *out, +void *pp_cb, +void **pp_task_cb) +{ +struct hook_cb_data *hook_cb = pp_cb; + + struct hook *hook = list_entry(hook_cb->run_me, struct hook, list); + +if (hook_cb->head == hook_cb->run_me) +return 0 ; + + cp->env = hook_cb->options->env.v; + cp->stdout_to_stderr =1 ; + cp->trace2_hook_name = hook->command.buf; + +/* reopen the file for stdin; run_command closes it. */ +if (hook_cb->options->path_to_stdin) { + cp->no_stdin =0 ; + cp->in =xopen (hook_cb->options->path_to_stdin,O_RDONLY ); + }else { + cp->no_stdin =1 ; + } + +/* +* Commands from the config could be oneliners, but we know +* for certain that hookdir commands are not. +*/ +if (hook->from_hookdir) + cp->use_shell =0 ; +else
hook.c void run_hooks_opt_clear(struct run_hooks_opt *o) · line 299
Quoted 10 lines
hook.c +10
-0+ cp->use_shell =1 ; + +/* add command */ +strvec_push (&cp->args, hook->command.buf); + +/* +* add passed-in argv, without expanding - let the user get back +* exactly what they put in +*/ +strvec_pushv (&cp->args, hook_cb->options->args.v);
I just skimmed over this setup-process-for-hook part - it would have been much clearer if it was refactored into its own function before this patch (or better yet, written as its own function in the first place). As it is, there are some unnecessary rewritings - e.g. setting stdin after env, and the use_shell setup.
Quoted 1 line
hook.h +0
-0
[snip]
hook.h struct run_hooks_opt · line 73
Quoted 9 lines
hook.h +9
-0+/* +* Callback provided to feed_pipe_fn and consume_sideband_fn. +*/ +struct hook_cb_data { +int rc; +struct list_head *head; +struct list_head *run_me; +struct run_hooks_opt *options; +};
Could this be in hook.c instead?
Also, I think it's clearer if run_me was a struct hook, and set to NULL when iteration reaches the end. If you disagree, I think it needs some documentation (e.g. "the embedded linked list part of the hook that must be run next; if equal to head, then iteration has ended" or something like that).
hook.h struct run_hooks_opt · line 81
Quoted 7 lines
hook.h +2
-0+#define RUN_HOOKS_OPT_INIT_SYNC { \ .env = STRVEC_INIT, \ .args = STRVEC_INIT, \ .path_to_stdin = NULL, \ + .jobs = 1, \ .run_hookdir = configured_hookdir_opt() \ }
This is not used anywhere.
Re: [PATCH v7 12/17] hook: allow parallel hook execution
Quoted 9 lines · Jonathan Tan
Quoted 9 lines
In many cases, there's no reason not to allow hooks to execute in parallel.
run_processes_parallel()is well-suited - it's a task queue that runs its housekeeping in series, which means users don't need to worry about thread safety on their callback data. True multithreaded execution with the async_* functions isn't necessary here. Synchronous hook execution can be achieved by only allowing 1 job to run at a time.
Quoted 5 lines
Quoted 2 lines
Teach
run_hooks()to use that function for simple hooks which don't require stdin or capture of stderr.Which hooks would be run in parallel, and which hooks in series? I don't see code that distinguishes between them.
It's up to the caller, who can set run_hooks_opt.jobs. In part II of this series I made a guess at which ones should run in parallel or in series and specified it in Documentation/githooks.txt.
Quoted 8 lines
Quoted 8 lines
Signed-off-by Emily Shaffer <[email protected]>---
Notes:
Per AEvar's request - parallel hook execution on day zero.
Quoted 8 lines
Quoted 6 lines
In most ways
run_processes_parallel()worked great for me - but it didn't have great support for hooks where we pipe to and from. I had to add this support later in the series.Since I modified an existing and in-use library I'd appreciate a keen look on these patches.
What is the existing and in-use library that you're modifying?
Hm, this note wasn't super specific. From this point onwards in the series I make changes to the run-command.h:run_processes_parallel() library, although not in this commit itself. I think I meant "from here on out, help me look at run-command.h".
I'll try to make the note a little better next series, sorry for the confusion :) :)
Quoted 45 lines
Quoted 45 lines
hook.c +40
-0@@ -246,11 +255,96 @@ void run_hooks_opt_clear(struct run_hooks_opt *o)strvec_clear (&o->args); } + +static int pick_next_hook (struct child_process *cp, +struct strbuf *out, +void *pp_cb, +void **pp_task_cb) +{ +struct hook_cb_data *hook_cb = pp_cb; + + struct hook *hook = list_entry(hook_cb->run_me, struct hook, list); + +if (hook_cb->head == hook_cb->run_me) +return 0 ; + + cp->env = hook_cb->options->env.v; + cp->stdout_to_stderr =1 ; + cp->trace2_hook_name = hook->command.buf; + +/* reopen the file for stdin; run_command closes it. */ +if (hook_cb->options->path_to_stdin) { + cp->no_stdin =0 ; + cp->in =xopen (hook_cb->options->path_to_stdin,O_RDONLY ); + }else { + cp->no_stdin =1 ; + } + +/* +* Commands from the config could be oneliners, but we know +* for certain that hookdir commands are not. +*/ +if (hook->from_hookdir) + cp->use_shell =0 ; +else + cp->use_shell =1 ; + +/* add command */ +strvec_push (&cp->args, hook->command.buf); + +/* +* add passed-in argv, without expanding - let the user get back +* exactly what they put in
Quoted 8 lines
Review void run_hooks_opt_clear(struct run_hooks_opt *o) · line 299
Quoted 2 lines
(quoted diff) +2
-0+ */ + strvec_pushv(&cp->args, hook_cb->options->args.v);I just skimmed over this setup-process-for-hook part - it would have been much clearer if it was refactored into its own function before this patch (or better yet, written as its own function in the first place). As it is, there are some unnecessary rewritings - e.g. setting stdin after env, and the
use_shellsetup.
Yeah, that makes sense. Will see if I can change it for next round :)
Quoted 5 lines
Quoted 2 lines
hook.h +0
-0[snip]
Quoted 11 lines
hook.h struct run_hooks_opt · line 73
Quoted 9 lines
hook.h +9
-0+/* +* Callback provided to feed_pipe_fn and consume_sideband_fn. +*/ +struct hook_cb_data { +int rc; +struct list_head *head; +struct list_head *run_me; +struct run_hooks_opt *options; +};Could this be in
hook.cinstead?
It ends up being needed publicly by
https://lore.kernel.org/git/[email protected]
(receive-pack: convert receive hooks to hook.h), which writes its own stdin provider callback. (In a later commit, a "void* options" gets added to this struct.)
At that point it's needed because the run-command callback structure can provide one context pointer for the overall work queue, and one context pointer for the individual task; this one is the "overall work queue" pointer.
From hook.h's perspective, the entire hook_cb_data is needed for pick_next_hook; but run-command.h:run_processes_parallel() doesn't have a way to tease out a smaller amount of the context pointer for various callbacks. If we wanted to obfuscate "hook_cb_data" we'd need to add another indirection and call back to hook.h first, who could then tease out the client-provided context and then call the client callback, but to me it sounds unnecessarily complex.
Quoted 6 lines
Also, I think it's clearer if
run_mewas a struct hook, and set to NULL when iteration reaches the end. If you disagree, I think it needs some documentation (e.g. "the embedded linked list part of the hook that must be run next; if equal to head, then iteration has ended" or something like that).
Yeah, I don't see a huge reason not to do that, sure.
Quoted 10 lines
hook.h struct run_hooks_opt · line 81
Quoted 8 lines
hook.h +2
-0+#define RUN_HOOKS_OPT_INIT_SYNC { \ .env = STRVEC_INIT, \ .args = STRVEC_INIT, \ .path_to_stdin = NULL, \ + .jobs = 1, \ .run_hookdir = configured_hookdir_opt() \ }This is not used anywhere.
It is used in part II by hooks which are not able to be parallelized.
- Emily
PATCH v7 13/17 hook: allow specifying working directory for hooks
In reply to Emily ShafferHooks like "post-checkout" require that hooks have a different working directory than the initial process. Pipe that directly through to struct child_process.
Because we can just run 'git -C <some-dir> hook run ...' it shouldn't be necessary to pipe this option through the frontend. In fact, this reduces the possibility of users running hooks which affect some part of the filesystem outside of the repo in question.
---
Notes:
Needed later for "post-checkout" conversion.
hook.c | 1 + hook.h | 5 +++++ 2 files changed, 6 insertions(+)
hook.c +1 -0
@@ -271,6 +271,7 @@ static int pick_next_hook(struct child_process *cp, cp->env = hook_cb->options->env.v; cp->stdout_to_stderr =1 ; cp->trace2_hook_name = hook->command.buf; + cp->dir = hook_cb->options->dir;/* reopen the file for stdin; run_command closes it. */ if (hook_cb->options->path_to_stdin) {
hook.h +5 -0
@@ -60,6 +60,9 @@ struct run_hooks_opt/* Number of threads to parallelize across */ int jobs; + +/* Path to initial working directory for subprocess */ +const char *dir; };/*
@@ -77,6 +80,7 @@ struct hook_cb_data { .args = STRVEC_INIT, \ .path_to_stdin = NULL, \ .jobs = 1, \ + .dir = NULL, \ .run_hookdir = configured_hookdir_opt() \ }
@@ -85,6 +89,7 @@ struct hook_cb_data { .args = STRVEC_INIT, \ .path_to_stdin = NULL, \ .jobs = configured_hook_jobs(), \ + .dir = NULL, \ .run_hookdir = configured_hookdir_opt() \ }
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
PATCH v7 14/17 run-command: add stdin callback for parallelization
In reply to Emily ShafferIf a user of the run_processes_parallel() API wants to pipe a large amount of information to stdin of each parallel command, that information could exceed the buffer of the pipe allocated for that process's stdin. Generally this is solved by repeatedly writing to child_process.in between calls to start_command() and finish_command(); run_processes_parallel() did not provide users an opportunity to access child_process at that time.
Because the data might be extremely large (for example, a list of all refs received during a push from a client) simply taking a string_list or strbuf is not as scalable as using a callback; the rest of the run_processes_parallel() API also uses callbacks, so making this feature match the rest of the API reduces mental load on the user.
--- builtin/fetch.c | 1 + builtin/submodule--helper.c | 2 +- hook.c | 1 + run-command.c | 54 +++++++++++++++++++++++++++++++++++-- run-command.h | 17 +++++++++++- submodule.c | 1 + t/helper/test-run-command.c | 31 ++++++++++++++++++--- t/t0061-run-command.sh | 30 +++++++++++++++++++++ 8 files changed, 129 insertions(+), 8 deletions(-)
builtin/fetch.c +1 -0
@@ -1647,6 +1647,7 @@ static int fetch_multiple(struct string_list *list, int max_children) result =run_processes_parallel_tr2 (max_children, &fetch_next_remote, &fetch_failed_to_start, +NULL , &fetch_finished, &state,"fetch" ,"parallel/fetch" );
builtin/submodule--helper.c +1 -1
@@ -2294,7 +2294,7 @@ static int update_submodules(struct submodule_update_clone *suc)int i;run_processes_parallel_tr2 (suc->max_jobs, update_clone_get_next_task, - update_clone_start_failure, + update_clone_start_failure,NULL , update_clone_task_finished, suc,"submodule" ,"parallel/update" );
hook.c +1 -0
@@ -368,6 +368,7 @@ int run_hooks(const char *hookname, struct run_hooks_opt *options) run_processes_parallel_tr2(options->jobs, pick_next_hook, notify_start_failure, + NULL, notify_hook_finished, &cb_data, "hook",
run-command.c +52 -2
125 lines not on this page · raw message
run-command.h +16 -1
@@ -436,6 +436,20 @@ typedef int (*start_failure_fn)(struct strbuf *out,void *pp_cb, void *pp_task_cb); +/** +* This callback is called repeatedly on every child process who requests +* start_command() to create a pipe by setting child_process.in < 0. +* +* pp_cb is the callback cookie as passed into run_processes_parallel, and +* pp_task_cb is the callback cookie as passed into get_next_task_fn. +* The contents of 'send' will be read into the pipe and passed to the pipe. +* +* Return nonzero to close the pipe. +*/ +typedef int (*feed_pipe_fn )(struct strbuf *pipe, +void *pp_cb, +void *pp_task_cb); +/** * This callback is called on every child process that finished processing. *
@@ -470,10 +484,11 @@ typedef int (*task_finished_fn)(int result,int run_processes_parallel (int n,get_next_task_fn ,start_failure_fn , +feed_pipe_fn ,task_finished_fn ,void *pp_cb);int run_processes_parallel_tr2 (int n,get_next_task_fn ,start_failure_fn , -task_finished_fn ,void *pp_cb, +feed_pipe_fn ,task_finished_fn ,void *pp_cb,const char *tr2_category,const char *tr2_label);#endif
submodule.c +1 -0
@@ -1638,6 +1638,7 @@ int fetch_populated_submodules(struct repository *r,run_processes_parallel_tr2 (max_parallel_jobs, get_next_submodule, fetch_start_failure, +NULL , fetch_finish, &spf,"submodule" ,"parallel/fetch" );
t/helper/test-run-command.c +27 -4
@@ -32,8 +32,13 @@ static int parallel_next(struct child_process *cp,return 0 ;strvec_pushv (&cp->args, d->argv); + cp->in = d->in; + cp->no_stdin = d->no_stdin;strbuf_addstr (err,"preloaded output of a child\n" ); number_callbacks++; + + *task_cb =xmalloc (sizeof (int )); + *(int *)(*task_cb) =2 ;return 1 ; }
@@ -55,6 +60,17 @@ static int task_finished(int result,return 1 ; } +static int test_stdin (struct strbuf *pipe,void *cb,void *task_cb) +{ +int *lines_remaining = task_cb; + +if (*lines_remaining) +strbuf_addf (pipe,"sample stdin %d\n" , --(*lines_remaining)); + +return !(*lines_remaining); +} + +struct testsuite {struct string_list tests, failed;int next;
@@ -185,7 +201,7 @@ static int testsuite(int argc, const char **argv) suite.tests.nr, max_jobs); ret =run_processes_parallel (max_jobs, next_test, test_failed, - test_finished, &suite); + test_stdin, test_finished, &suite);if (suite.failed.nr >0 ) { ret =1 ;
@@ -413,15 +429,22 @@ int cmd__run_command(int argc, const char **argv)if (!strcmp (argv[1 ],"run-command-parallel" ))exit (run_processes_parallel (jobs, parallel_next, -NULL ,NULL , &proc)); +NULL ,NULL ,NULL , &proc));if (!strcmp (argv[1 ],"run-command-abort" ))exit (run_processes_parallel (jobs, parallel_next, -NULL , task_finished, &proc)); +NULL ,NULL , task_finished, &proc));if (!strcmp (argv[1 ],"run-command-no-jobs" ))exit (run_processes_parallel (jobs, no_job, -NULL , task_finished, &proc)); +NULL ,NULL , task_finished, &proc)); + +if (!strcmp (argv[1 ],"run-command-stdin" )) { + proc.in =-1 ; + proc.no_stdin =0 ; +exit (run_processes_parallel (jobs, parallel_next,NULL , + test_stdin,NULL , &proc)); + }fprintf (stderr,"check usage\n" );return 1 ;
t/t0061-run-command.sh +30 -0
@@ -143,6 +143,36 @@ test_expect_success 'run_command runs in parallel with more tasks than jobs avai test_cmp expect actual ' +cat >expect <<-EOF +preloaded output of a child +listening for stdin: +sample stdin 1 +sample stdin 0 +preloaded output of a child +listening for stdin: +sample stdin 1 +sample stdin 0 +preloaded output of a child +listening for stdin: +sample stdin 1 +sample stdin 0 +preloaded output of a child +listening for stdin: +sample stdin 1 +sample stdin 0 +EOF + +test_expect_success 'run_command listens to stdin' ' + write_script stdin-script <<-\EOF && + echo "listening for stdin:" + while read line; do + echo "$line" + done + EOF + test-tool run-command run-command-stdin 2 ./stdin-script 2>actual && + test_cmp expect actual +' + cat >expect <<-EOF preloaded output of a child asking for a quick stop
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
Re: [PATCH v7 14/17] run-command: add stdin callback for parallelization
Quoted 7 lines
If a user of the
run_processes_parallel()API wants to pipe a large amount of information to stdin of each parallel command, that information could exceed the buffer of the pipe allocated for that process's stdin. Generally this is solved by repeatedly writing to child_process.in between calls tostart_command()andfinish_command();run_processes_parallel()did not provide users an opportunity to accesschild_processat that time.
[snip]
Quoted 12 lines
run-command.h +4
-0@@ -436,6 +436,20 @@ typedef int (*start_failure_fn)(struct strbuf *out,void *pp_cb, void *pp_task_cb); +/** +* This callback is called repeatedly on every child process who requests +* start_command() to create a pipe by setting child_process.in < 0. +*
run-command.h typedef int (*start_failure_fn)(struct strbuf *out, · line 452
Quoted 10 lines
run-command.h +10
-0+* pp_cb is the callback cookie as passed into run_processes_parallel, and +* pp_task_cb is the callback cookie as passed into get_next_task_fn. +* The contents of 'send' will be read into the pipe and passed to the pipe. +* +* Return nonzero to close the pipe. +*/ +typedef int (*feed_pipe_fn )(struct strbuf *pipe, +void *pp_cb, +void *pp_task_cb); +
As you mention above in the commit message, I think the clearest API to support what we need is to just have a callback (that has access to child_process) that is executed between process start and finish.
As it is, I think this callback is too specific in that it takes a struct strbuf. I think that this struct strbuf will just end up being unnecessary copying much of the time, when the user could have just written to the fd directly.
Re: [PATCH v7 14/17] run-command: add stdin callback for parallelization
Quoted 38 lines · Jonathan Tan
Quoted 8 lines
If a user of the
run_processes_parallel()API wants to pipe a large amount of information to stdin of each parallel command, that information could exceed the buffer of the pipe allocated for that process's stdin. Generally this is solved by repeatedly writing to child_process.in between calls tostart_command()andfinish_command();run_processes_parallel()did not provide users an opportunity to accesschild_processat that time.[snip]
Quoted 12 lines
run-command.h +4
-0@@ -436,6 +436,20 @@ typedef int (*start_failure_fn)(struct strbuf *out,void *pp_cb, void *pp_task_cb); +/** +* This callback is called repeatedly on every child process who requests +* start_command() to create a pipe by setting child_process.in < 0. +* run-command.h typedef int (*start_failure_fn)(struct strbuf *out, · line 452
Quoted 10 lines
run-command.h +10
-0+* pp_cb is the callback cookie as passed into run_processes_parallel, and +* pp_task_cb is the callback cookie as passed into get_next_task_fn. +* The contents of 'send' will be read into the pipe and passed to the pipe. +* +* Return nonzero to close the pipe. +*/ +typedef int (*feed_pipe_fn )(struct strbuf *pipe, +void *pp_cb, +void *pp_task_cb); +As you mention above in the commit message, I think the clearest API to support what we need is to just have a callback (that has access to
child_process) that is executed between process start and finish.
Quoted 4 lines
As it is, I think this callback is too specific in that it takes a struct strbuf. I think that this struct strbuf will just end up being unnecessary copying much of the time, when the user could have just written to the fd directly.
Since the rest of the run_processes_parallel() API passes strings around with strbufs, I'd prefer to leave it as-is to match the general API expectations and style.
- Emily
Re: [PATCH v7 14/17] run-command: add stdin callback for parallelization
Quoted 24 lines
Quoted 19 lines
Quoted 4 lines
run-command.h +4
-0+/** +* This callback is called repeatedly on every child process who requests +* start_command() to create a pipe by setting child_process.in < 0. +* run-command.h typedef int (*start_failure_fn)(struct strbuf *out, · line 452
Quoted 10 lines
run-command.h +10
-0+* pp_cb is the callback cookie as passed into run_processes_parallel, and +* pp_task_cb is the callback cookie as passed into get_next_task_fn. +* The contents of 'send' will be read into the pipe and passed to the pipe. +* +* Return nonzero to close the pipe. +*/ +typedef int (*feed_pipe_fn )(struct strbuf *pipe, +void *pp_cb, +void *pp_task_cb); +As you mention above in the commit message, I think the clearest API to support what we need is to just have a callback (that has access to
child_process) that is executed between process start and finish.Quoted 4 lines
As it is, I think this callback is too specific in that it takes a struct strbuf. I think that this struct strbuf will just end up being unnecessary copying much of the time, when the user could have just written to the fd directly.
Quoted 5 lines
Since the rest of the
run_processes_parallel()API passes strings around with strbufs, I'd prefer to leave it as-is to match the general API expectations and style.- Emily
By the rest of the run_processes_parallel() API, do you mean get_next_task_fn, start_failure_fn, and task_finished_fn? If yes, I think that it makes sense for them to be strbuf, because buffering is needed to avoid outputs from individual child processes interleaving, but that's not true here.
Having said that, this is an internal API so we could just leave it as-is and then refactor it if we ever need something more flexible.
Re: [PATCH v7 14/17] run-command: add stdin callback for parallelization
Quoted 31 lines · Jonathan Tan
Quoted 25 lines
Quoted 20 lines
Quoted 5 lines
run-command.h +4
-0+/** +* This callback is called repeatedly on every child process who requests +* start_command() to create a pipe by setting child_process.in < 0. +* run-command.h typedef int (*start_failure_fn)(struct strbuf *out, · line 452
Quoted 10 lines
run-command.h +10
-0+* pp_cb is the callback cookie as passed into run_processes_parallel, and +* pp_task_cb is the callback cookie as passed into get_next_task_fn. +* The contents of 'send' will be read into the pipe and passed to the pipe. +* +* Return nonzero to close the pipe. +*/ +typedef int (*feed_pipe_fn )(struct strbuf *pipe, +void *pp_cb, +void *pp_task_cb); +As you mention above in the commit message, I think the clearest API to support what we need is to just have a callback (that has access to
child_process) that is executed between process start and finish.Quoted 4 lines
As it is, I think this callback is too specific in that it takes a struct strbuf. I think that this struct strbuf will just end up being unnecessary copying much of the time, when the user could have just written to the fd directly.
Quoted 5 lines
Since the rest of the
run_processes_parallel()API passes strings around with strbufs, I'd prefer to leave it as-is to match the general API expectations and style.- Emily
Quoted 8 lines
By the rest of the
run_processes_parallel()API, do you meanget_next_task_fn,start_failure_fn, andtask_finished_fn? If yes, I think that it makes sense for them to be strbuf, because buffering is needed to avoid outputs from individual child processes interleaving, but that's not true here.Having said that, this is an internal API so we could just leave it as-is and then refactor it if we ever need something more flexible.
Yeah, with that in mind I'll leave it as it is. I don't like the idea of directly exposing the child's pipe via callback; to me it feels like bad object-oriented design, but that maybe doesn't apply here :)
Anyway, like you say, we can change it later if someone doesn't like this, no problem.
PATCH v7 15/17 hook: provide stdin by string_list or callback
In reply to Emily ShafferIn cases where a hook requires only a small amount of information via stdin, it should be simple for users to provide a string_list alone. But in more complicated cases where the stdin is too large to hold in memory, let's provide a callback the users can populate line after line with instead.
--- hook.c | 39 ++++++++++++++++++++++++++++++++++++++- hook.h | 25 +++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-)
hook.c +38 -1
@@ -9,6 +9,7 @@ void free_hook(struct hook *ptr) {if (ptr) {strbuf_release (&ptr->command); +free (ptr->feed_pipe_cb_data);free (ptr); } }
@@ -38,6 +39,7 @@ static void append_or_move_hook(struct list_head *head, const char *command)strbuf_init (&to_add->command,0 );strbuf_addstr (&to_add->command, command); to_add->from_hookdir =0 ; + to_add->feed_pipe_cb_data =NULL ; }/* re-set the scope so we show where an override was specified */
@@ -253,9 +255,32 @@ void run_hooks_opt_clear(struct run_hooks_opt *o) {strvec_clear (&o->env);strvec_clear (&o->args); +string_list_clear (&o->str_stdin,0 ); } +static int pipe_from_string_list (struct strbuf *pipe,void *pp_cb,void *pp_task_cb) +{ +int *item_idx; +struct hook *ctx = pp_task_cb; +struct string_list *to_pipe = &((struct hook_cb_data *)pp_cb)->options->str_stdin; + +/* Bootstrap the state manager if necessary. */ +if (!ctx->feed_pipe_cb_data) { + ctx->feed_pipe_cb_data =xmalloc (sizeof (unsigned int )); + *(int *)ctx->feed_pipe_cb_data =0 ; + } + + item_idx = ctx->feed_pipe_cb_data; + +if (*item_idx < to_pipe->nr) { +strbuf_addf (pipe,"%s\n" , to_pipe->items[*item_idx].string); + (*item_idx)++; +return 0 ; + } +return 1 ; +} + static int pick_next_hook(struct child_process *cp, struct strbuf *out, void *pp_cb,
@@ -277,6 +302,10 @@ static int pick_next_hook(struct child_process *cp,if (hook_cb->options->path_to_stdin) { cp->no_stdin =0 ; cp->in =xopen (hook_cb->options->path_to_stdin,O_RDONLY ); + }else if (hook_cb->options->feed_pipe) { +/* ask for start_command() to make a pipe for us */ + cp->in =-1 ; + cp->no_stdin =0 ; }else { cp->no_stdin =1 ; }
@@ -350,6 +379,14 @@ int run_hooks(const char *hookname, struct run_hooks_opt *options)if (!options)BUG ("a struct run_hooks_opt must be provided to run_hooks" ); +if ((options->path_to_stdin && options->str_stdin.nr) || + (options->path_to_stdin && options->feed_pipe) || + (options->str_stdin.nr && options->feed_pipe)) +BUG ("choose only one method to populate stdin" ); + +if (options->str_stdin.nr) + options->feed_pipe = &pipe_from_string_list; +strbuf_addstr (&hookname_str, hookname); to_run =hook_list (&hookname_str);
@@ -368,7 +405,7 @@ int run_hooks(const char *hookname, struct run_hooks_opt *options) run_processes_parallel_tr2(options->jobs, pick_next_hook, notify_start_failure, - NULL, + options->feed_pipe, notify_hook_finished, &cb_data, "hook",
hook.h +25 -0
@@ -2,6 +2,7 @@#include "list.h" #include "strbuf.h" #include "strvec.h" +#include "run-command.h" struct hook {
@@ -14,6 +15,12 @@ struct hook/* The literal command to run. */ struct strbuf command;int from_hookdir; + +/* +* Use this to keep state for your feed_pipe_fn if you are using +* run_hooks_opt.feed_pipe. Otherwise, do not touch it. +*/ +void *feed_pipe_cb_data; };/*
@@ -57,12 +64,24 @@ struct run_hooks_opt/* Path to file which should be piped to stdin for each hook */ const char *path_to_stdin; +/* Pipe each string to stdin, separated by newlines */ +struct string_list str_stdin; +/* +* Callback and state pointer to ask for more content to pipe to stdin. +* Will be called repeatedly, for each hook. See +* hook.c:pipe_from_stdin() for an example. Keep per-hook state in +* hook.feed_pipe_cb_data (per process). Keep initialization context in +* feed_pipe_ctx (shared by all processes). +*/ +feed_pipe_fn feed_pipe; +void *feed_pipe_ctx;/* Number of threads to parallelize across */ int jobs;/* Path to initial working directory for subprocess */ const char *dir; + };/*
@@ -81,6 +100,9 @@ struct hook_cb_data { .path_to_stdin = NULL, \ .jobs = 1, \ .dir = NULL, \ + .str_stdin = STRING_LIST_INIT_DUP, \ + .feed_pipe = NULL, \ + .feed_pipe_ctx = NULL, \ .run_hookdir = configured_hookdir_opt() \ }
@@ -90,6 +112,9 @@ struct hook_cb_data { .path_to_stdin = NULL, \ .jobs = configured_hook_jobs(), \ .dir = NULL, \ + .str_stdin = STRING_LIST_INIT_DUP, \ + .feed_pipe = NULL, \ + .feed_pipe_ctx = NULL, \ .run_hookdir = configured_hookdir_opt() \ }
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
Re: [PATCH v7 15/17] hook: provide stdin by string_list or callback
Quoted 5 lines
In cases where a hook requires only a small amount of information via stdin, it should be simple for users to provide a
string_listalone. But in more complicated cases where the stdin is too large to hold in memory, let's provide a callback the users can populate line after line with instead.
[snip]
Quoted 13 lines
hook.h +1
-0@@ -2,6 +2,7 @@#include "list.h" #include "strbuf.h" #include "strvec.h" +#include "run-command.h" struct hook {
hook.h struct hook · line 23
Quoted 9 lines
hook.h +6
-0@@ -14,6 +15,12 @@ struct hook/* The literal command to run. */ struct strbuf command;int from_hookdir; + +/* +* Use this to keep state for your feed_pipe_fn if you are using +* run_hooks_opt.feed_pipe. Otherwise, do not touch it. +*/ +void *feed_pipe_cb_data;
When would we need per-hook state? I see in patch 14 that you give each running process little by little (in pp_buffer_stdin()), perhaps so that each hook can make progress at roughly the same pace, but I don't think we can expect all hooks to work the same, so I don't think it's worth complicating the design for all that.
Quoted 8 lines
hook.h +1
-0};/* @@ -57,12 +64,24 @@ struct run_hooks_opt/* Path to file which should be piped to stdin for each hook */ const char *path_to_stdin; +/* Pipe each string to stdin, separated by newlines */
hook.h struct run_hooks_opt · line 77
Quoted 10 lines
hook.h +10
-0+struct string_list str_stdin; +/* +* Callback and state pointer to ask for more content to pipe to stdin. +* Will be called repeatedly, for each hook. See +* hook.c:pipe_from_stdin() for an example. Keep per-hook state in +* hook.feed_pipe_cb_data (per process). Keep initialization context in +* feed_pipe_ctx (shared by all processes). +*/ +feed_pipe_fn feed_pipe; +void *feed_pipe_ctx;
Instead of 3 fields, I think 2 suffice - the function and the data (called "ctx" here). We can supply a function that treats the data as a string_list.
Re: [PATCH v7 15/17] hook: provide stdin by string_list or callback
Quoted 29 lines · Jonathan Tan
Quoted 6 lines
In cases where a hook requires only a small amount of information via stdin, it should be simple for users to provide a
string_listalone. But in more complicated cases where the stdin is too large to hold in memory, let's provide a callback the users can populate line after line with instead.[snip]
Quoted 20 lines
hook.h +5
-0@@ -2,6 +2,7 @@#include "list.h" #include "strbuf.h" #include "strvec.h" +#include "run-command.h" struct hook {@@ -14,6 +15,12 @@ struct hook/* The literal command to run. */ struct strbuf command;int from_hookdir; + +/* +* Use this to keep state for your feed_pipe_fn if you are using +* run_hooks_opt.feed_pipe. Otherwise, do not touch it.
Quoted 8 lines
hook.h struct hook · line 23
Quoted 2 lines
hook.h +2
-0+*/ +void *feed_pipe_cb_data;When would we need per-hook state? I see in patch 14 that you give each running process little by little (in
pp_buffer_stdin()), perhaps so that each hook can make progress at roughly the same pace, but I don't think we can expect all hooks to work the same, so I don't think it's worth complicating the design for all that.
I agree that this is a complicated way of doing it, and if you have a better design I'd be really excited to hear it.
It seemed like this was what was necessary for hooks like
https://lore.kernel.org/git/[email protected]
where the hook and the invoking process talk back and forth, or like
https://lore.kernel.org/git/[email protected]
which generates stdin on the fly for hooks which cannot be parallelized (and so won't run at the same pace).
The former example - proc-receive - does have a constraint that multiple hooks can't be specified, so we could theoretically keep the old implementation and just pick up the single hook's location from the new hook library. But the latter example still makes me think this much complexity is needed.
Quoted 15 lines
Quoted 15 lines
hook.h +7
-0};/* @@ -57,12 +64,24 @@ struct run_hooks_opt/* Path to file which should be piped to stdin for each hook */ const char *path_to_stdin; +/* Pipe each string to stdin, separated by newlines */ +struct string_list str_stdin; +/* +* Callback and state pointer to ask for more content to pipe to stdin. +* Will be called repeatedly, for each hook. See +* hook.c:pipe_from_stdin() for an example. Keep per-hook state in +* hook.feed_pipe_cb_data (per process). Keep initialization context in
Quoted 8 lines
hook.h struct run_hooks_opt · line 77
Quoted 4 lines
hook.h +4
-0+* feed_pipe_ctx (shared by all processes). +*/ +feed_pipe_fn feed_pipe; +void *feed_pipe_ctx;Instead of 3 fields, I think 2 suffice - the function and the data (called "ctx" here). We can supply a function that treats the data as a
string_list.
Nice catch, sure.
- Emily
Re: [PATCH v7 15/17] hook: provide stdin by string_list or callback
Quoted 16 lines
Quoted 5 lines
When would we need per-hook state? I see in patch 14 that you give each running process little by little (in
pp_buffer_stdin()), perhaps so that each hook can make progress at roughly the same pace, but I don't think we can expect all hooks to work the same, so I don't think it's worth complicating the design for all that.I agree that this is a complicated way of doing it, and if you have a better design I'd be really excited to hear it.
It seemed like this was what was necessary for hooks like
https://lore.kernel.org/git/[email protected]
where the hook and the invoking process talk back and forth, or like
https://lore.kernel.org/git/[email protected]
which generates stdin on the fly for hooks which cannot be parallelized (and so won't run at the same pace).
Quoted 5 lines
The former example - proc-receive - does have a constraint that multiple hooks can't be specified, so we could theoretically keep the old implementation and just pick up the single hook's location from the new hook library. But the latter example still makes me think this much complexity is needed.
Ah, I see. From your explanation, in these 2 cases, only one hook executes at a time (in the former case, because there is only one hook, and in the latter case, you said that the hooks cannot be parallelized). So it seems to me that the global state (in struct run_hooks_opt) would be sufficient to keep track of what's going on. (The feed_pipe_fn function can use pp_cb to keep track of the last executing pp_task_cb and then compare it against the new pp_task_cb, I think, to keep track of when a new hook has started.)
Even in the case of multiple hooks run in series (as opposed to a single hook), I would think that the reason they can't be run in parallel is because the nature of execution of a hook depends on what happened during the execution of the previous hook, which seems to me to be even more reason to centralize the state in struct run_hooks_opt.
Having said that, if my suggestion of not having per-hook state makes certain patches more complicated, then that might be reason enough to have per-hook state. In that case, you should write "per-hook state, though strictly not necessary, makes <case> simpler" (or something like that) in the commit message.
Re: [PATCH v7 15/17] hook: provide stdin by string_list or callback
Quoted 38 lines · Jonathan Tan
Quoted 17 lines
Quoted 6 lines
When would we need per-hook state? I see in patch 14 that you give each running process little by little (in
pp_buffer_stdin()), perhaps so that each hook can make progress at roughly the same pace, but I don't think we can expect all hooks to work the same, so I don't think it's worth complicating the design for all that.I agree that this is a complicated way of doing it, and if you have a better design I'd be really excited to hear it.
It seemed like this was what was necessary for hooks like
https://lore.kernel.org/git/[email protected]
where the hook and the invoking process talk back and forth, or like
https://lore.kernel.org/git/[email protected]
which generates stdin on the fly for hooks which cannot be parallelized (and so won't run at the same pace).Quoted 5 lines
The former example - proc-receive - does have a constraint that multiple hooks can't be specified, so we could theoretically keep the old implementation and just pick up the single hook's location from the new hook library. But the latter example still makes me think this much complexity is needed.
Ah, I see. From your explanation, in these 2 cases, only one hook executes at a time (in the former case, because there is only one hook, and in the latter case, you said that the hooks cannot be parallelized). So it seems to me that the global state (in struct
run_hooks_opt) would be sufficient to keep track of what's going on. (Thefeed_pipe_fnfunction can usepp_cbto keep track of the last executingpp_task_cband then compare it against the newpp_task_cb, I think, to keep track of when a new hook has started.)Even in the case of multiple hooks run in series (as opposed to a single hook), I would think that the reason they can't be run in parallel is because the nature of execution of a hook depends on what happened during the execution of the previous hook, which seems to me to be even more reason to centralize the state in struct
run_hooks_opt.
Quoted 5 lines
Having said that, if my suggestion of not having per-hook state makes certain patches more complicated, then that might be reason enough to have per-hook state. In that case, you should write "per-hook state, though strictly not necessary, makes <case> simpler" (or something like that) in the commit message.
Jonathan and I discussed this a little more offline and agreed to leave the implementation as is.
Jonathan had suggested "have one callback invocation apply to all hooks that are running now", either by having the callback iterate over the task queue or by having the run-command lib take the result from the callback and have *that* iterate over the task queue. The idea being, one pointer to one copy of source material is easier to handle than many.
I suggested that the callback's implementation of the second version of that, where the library takes care of the "and do it for each task in progress" part, would be pretty much identical to the callback's implementation as it is in this patch, except that as it is here the context pointer is per-task and as Jonathan suggests the context pointer is per-entire-hook-invocation - so there isn't much complexity difference between the two, from the user's perspective.
We also talked about cases where N=# of hooks > M=# of jobs, that is, where some hooks must wait for other hooks to finish executing before that could start. In this case, users' callback implementations would need to be able to start over from the beginning of the source material, and a long-running hook would block other short-running hooks from beginning (because the long-running hook would be confused by hearing the source material to its stdin again).
Hopefully this diagram illustrates better based on my understanding of Jonathan's suggestion:
A B <- "hey everyone, stdin 'foo'"
A B <- "hey everyone, stdin 'bar'"
A B <- "hey everyone, stdin 'baz'"
B
B
B
B
C <- "hey everyone, stdin 'foo'" C <- "hey everyone, stdin 'bar'" C <- "hey everyone, stdin 'baz'"
C
Anyway, since the complexity is probably about the same to the end user and using per-hook context means we don't have to wait like this, we agreed to stick with the implementation as is.
- Emily
Re: [PATCH v7 15/17] hook: provide stdin by string_list or callback
Quoted 18 lines
Jonathan and I discussed this a little more offline and agreed to leave the implementation as is.
Jonathan had suggested "have one callback invocation apply to all hooks that are running now", either by having the callback iterate over the task queue or by having the run-command lib take the result from the callback and have *that* iterate over the task queue. The idea being, one pointer to one copy of source material is easier to handle than many.
I suggested that the callback's implementation of the second version of that, where the library takes care of the "and do it for each task in progress" part, would be pretty much identical to the callback's implementation as it is in this patch, except that as it is here the context pointer is per-task and as Jonathan suggests the context pointer is per-entire-hook-invocation - so there isn't much complexity difference between the two, from the user's perspective.
Quoted 7 lines
We also talked about cases where N=# of hooks > M=# of jobs, that is, where some hooks must wait for other hooks to finish executing before that could start. In this case, users' callback implementations would need to be able to start over from the beginning of the source material, and a long-running hook would block other short-running hooks from beginning (because the long-running hook would be confused by hearing the source material to its stdin again).
Yes - this (number of hooks greater than number of jobs allowed to run in parallel) was the case in which my suggestion of not having hook-specific state would not work. The case we were talking about is when there's a large amount of dynamically-generated data to be transmitted to the hooks' stdins and I was thinking that it would be best anyway if the callback looped over all hooks as data was generated, but it would not be possible to only do a single pass if the number of hooks is greater than the number of jobs.
PATCH v7 16/17 run-command: allow capturing of collated output
In reply to Emily ShafferSome callers, for example server-side hooks which wish to relay hook output to clients across a transport, want to capture what would normally print to stderr and do something else with it. Allow that via a callback.
By calling the callback regardless of whether there's output available, we allow clients to send e.g. a keepalive if necessary.
Because we expose a strbuf, not a fd or FILE*, there's no need to create a temporary pipe or similar - we can just skip the print to stderr and instead hand it to the caller.
---
Notes:
Originally when writing this patch I attempted to use a pipe in memory - but managing its lifetime was actually pretty tricky, and I found I could achieve the same thing with less code by doing it this way. Critique welcome, including "no, you really need to do it with a pipe".
builtin/fetch.c | 2 +- builtin/submodule--helper.c | 2 +- hook.c | 1 + run-command.c | 33 +++++++++++++++++++++++++-------- run-command.h | 18 +++++++++++++++++- submodule.c | 2 +- t/helper/test-run-command.c | 25 ++++++++++++++++++++----- t/t0061-run-command.sh | 7 +++++++ 8 files changed, 73 insertions(+), 17 deletions(-)
builtin/fetch.c +1 -1
@@ -1647,7 +1647,7 @@ static int fetch_multiple(struct string_list *list, int max_children) result =run_processes_parallel_tr2 (max_children, &fetch_next_remote, &fetch_failed_to_start, -NULL , +NULL ,NULL , &fetch_finished, &state,"fetch" ,"parallel/fetch" );
builtin/submodule--helper.c +1 -1
@@ -2294,7 +2294,7 @@ static int update_submodules(struct submodule_update_clone *suc)int i;run_processes_parallel_tr2 (suc->max_jobs, update_clone_get_next_task, - update_clone_start_failure,NULL , + update_clone_start_failure,NULL ,NULL , update_clone_task_finished, suc,"submodule" ,"parallel/update" );
hook.c +1 -0
@@ -406,6 +406,7 @@ int run_hooks(const char *hookname, struct run_hooks_opt *options) pick_next_hook, notify_start_failure, options->feed_pipe, +NULL , notify_hook_finished, &cb_data, "hook",
run-command.c +25 -8
@@ -1549,6 +1549,7 @@ struct parallel_processes {get_next_task_fn get_next_task;start_failure_fn start_failure;feed_pipe_fn feed_pipe; +consume_sideband_fn consume_sideband;task_finished_fn task_finished;struct {
@@ -1614,6 +1615,7 @@ static void pp_init(struct parallel_processes *pp,get_next_task_fn get_next_task, start_failure_fn start_failure, feed_pipe_fn feed_pipe, + consume_sideband_fn consume_sideband, task_finished_fn task_finished, void *data) {
@@ -1634,6 +1636,7 @@ static void pp_init(struct parallel_processes *pp, pp->start_failure = start_failure ? start_failure : default_start_failure; pp->feed_pipe = feed_pipe ? feed_pipe : default_feed_pipe; pp->task_finished = task_finished ? task_finished : default_task_finished; + pp->consume_sideband = consume_sideband; pp->nr_processes =0 ; pp->output_owner =0 ;
@@ -1670,7 +1673,10 @@ static void pp_cleanup(struct parallel_processes *pp)* When get_next_task added messages to the buffer in its last * iteration, the buffered output is non empty. */ -strbuf_write (&pp->buffered_output, stderr); +if (pp->consume_sideband) + pp->consume_sideband (&pp->buffered_output, pp->data); +else +strbuf_write (&pp->buffered_output, stderr);strbuf_release (&pp->buffered_output);sigchain_pop_common ();
@@ -1786,9 +1792,13 @@ static void pp_buffer_stderr(struct parallel_processes *pp, int output_timeout)static void pp_output (struct parallel_processes *pp) {int i = pp->output_owner; +if (pp->children[i].state ==GIT_CP_WORKING && pp->children[i].err.len) { -strbuf_write (&pp->children[i].err, stderr); +if (pp->consume_sideband) + pp->consume_sideband (&pp->children[i].err, pp->data); +else +strbuf_write (&pp->children[i].err, stderr);strbuf_reset (&pp->children[i].err); } }
@@ -1827,11 +1837,15 @@ static int pp_collect_finished(struct parallel_processes *pp)strbuf_addbuf (&pp->buffered_output, &pp->children[i].err);strbuf_reset (&pp->children[i].err); }else { -strbuf_write (&pp->children[i].err, stderr); +/* Output errors, then all other finished child processes */ +if (pp->consume_sideband) { + pp->consume_sideband (&pp->children[i].err, pp->data); + pp->consume_sideband (&pp->buffered_output, pp->data); + }else { +strbuf_write (&pp->children[i].err, stderr); +strbuf_write (&pp->buffered_output, stderr); + }strbuf_reset (&pp->children[i].err); - -/* Output all other finished child processes */ -strbuf_write (&pp->buffered_output, stderr);strbuf_reset (&pp->buffered_output);/*
@@ -1855,6 +1869,7 @@ int run_processes_parallel(int n,get_next_task_fn get_next_task, start_failure_fn start_failure, feed_pipe_fn feed_pipe, + consume_sideband_fn consume_sideband, task_finished_fn task_finished, void *pp_cb) {
@@ -1865,7 +1880,7 @@ int run_processes_parallel(int n,sigchain_push (SIGPIPE ,SIG_IGN ); -pp_init (&pp, n, get_next_task, start_failure, feed_pipe, task_finished, pp_cb); +pp_init (&pp, n, get_next_task, start_failure, feed_pipe, consume_sideband, task_finished, pp_cb);while (1 ) { for (i = 0; i < spawn_cap && !pp.shutdown &&
@@ -1903,6 +1918,7 @@ int run_processes_parallel(int n,int run_processes_parallel_tr2 (int n,get_next_task_fn get_next_task,start_failure_fn start_failure,feed_pipe_fn feed_pipe, +consume_sideband_fn consume_sideband,task_finished_fn task_finished,void *pp_cb,const char *tr2_category,const char *tr2_label) {
@@ -1912,7 +1928,8 @@ int run_processes_parallel_tr2(int n, get_next_task_fn get_next_task, ((n < 1) ? online_cpus() : n)); result =run_processes_parallel (n, get_next_task, start_failure, - feed_pipe, task_finished, pp_cb); + feed_pipe, consume_sideband, + task_finished, pp_cb);trace2_region_leave (tr2_category, tr2_label,NULL );
run-command.h +17 -1
@@ -450,6 +450,20 @@ typedef int (*feed_pipe_fn)(struct strbuf *pipe,void *pp_cb, void *pp_task_cb); +/** +* If this callback is provided, instead of collating process output to stderr, +* they will be collated into a new pipe. consume_sideband_fn will be called +* repeatedly. When output is available on that pipe, it will be contained in +* 'output'. But it will be called with an empty 'output' too, to allow for +* keepalives or similar operations if necessary. +* +* pp_cb is the callback cookie as passed into run_processes_parallel. +* +* Since this callback is provided with the collated output, no task cookie is +* provided. +*/ +typedef void (*consume_sideband_fn )(struct strbuf *output,void *pp_cb); +/** * This callback is called on every child process that finished processing. *
@@ -485,10 +499,12 @@ int run_processes_parallel(int n, get_next_task_fn, start_failure_fn, feed_pipe_fn, + consume_sideband_fn, task_finished_fn, void *pp_cb);int run_processes_parallel_tr2 (int n,get_next_task_fn ,start_failure_fn , -feed_pipe_fn ,task_finished_fn ,void *pp_cb, +feed_pipe_fn ,consume_sideband_fn , +task_finished_fn ,void *pp_cb,const char *tr2_category,const char *tr2_label);#endif
submodule.c +1 -1
@@ -1638,7 +1638,7 @@ int fetch_populated_submodules(struct repository *r,run_processes_parallel_tr2 (max_parallel_jobs, get_next_submodule, fetch_start_failure, -NULL , +NULL ,NULL , fetch_finish, &spf,"submodule" ,"parallel/fetch" );
t/helper/test-run-command.c +20 -5
@@ -51,6 +51,16 @@ static int no_job(struct child_process *cp,return 0 ; } +static void test_consume_sideband (struct strbuf *output,void *cb) +{ +FILE *sideband; + + sideband =fopen ("./sideband" ,"a" ); + +strbuf_write (output, sideband); +fclose (sideband); +} + static int task_finished(int result, struct strbuf *err, void *pp_cb,
@@ -201,7 +211,7 @@ static int testsuite(int argc, const char **argv) suite.tests.nr, max_jobs); ret =run_processes_parallel (max_jobs, next_test, test_failed, - test_stdin, test_finished, &suite); + test_stdin,NULL , test_finished, &suite);if (suite.failed.nr >0 ) { ret =1 ;
@@ -429,23 +439,28 @@ int cmd__run_command(int argc, const char **argv)if (!strcmp (argv[1 ],"run-command-parallel" ))exit (run_processes_parallel (jobs, parallel_next, -NULL ,NULL ,NULL , &proc)); +NULL ,NULL ,NULL ,NULL , &proc));if (!strcmp (argv[1 ],"run-command-abort" ))exit (run_processes_parallel (jobs, parallel_next, -NULL ,NULL , task_finished, &proc)); +NULL ,NULL ,NULL , task_finished, &proc));if (!strcmp (argv[1 ],"run-command-no-jobs" ))exit (run_processes_parallel (jobs, no_job, -NULL ,NULL , task_finished, &proc)); +NULL ,NULL ,NULL , task_finished, &proc));if (!strcmp (argv[1 ],"run-command-stdin" )) { proc.in =-1 ; proc.no_stdin =0 ;exit (run_processes_parallel (jobs, parallel_next,NULL , - test_stdin,NULL , &proc)); + test_stdin,NULL ,NULL , &proc)); } +if (!strcmp (argv[1 ],"run-command-sideband" )) +exit (run_processes_parallel (jobs, parallel_next,NULL ,NULL , + test_consume_sideband,NULL , + &proc)); +fprintf (stderr,"check usage\n" );return 1 ; }
t/t0061-run-command.sh +7 -0
@@ -143,6 +143,13 @@ test_expect_success 'run_command runs in parallel with more tasks than jobs avai test_cmp expect actual ' +test_expect_success 'run_command can divert output' ' + test_when_finished rm sideband && + test-tool run-command run-command-sideband 3 sh -c "printf \"%s\n%s\n\" Hello World" 2>actual && + test_must_be_empty actual && + test_cmp expect sideband +' + cat >expect <<-EOF preloaded output of a child listening for stdin:
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
PATCH v7 17/17 hooks: allow callers to capture output
In reply to Emily ShafferSome server-side hooks will require capturing output to send over sideband instead of printing directly to stderr. Expose that capability.
---
Notes:
You can see this in practice in the conversions for some of the push hooks, like 'receive-pack'.
hook.c | 2 +- hook.h | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-)
hook.c +1 -1
@@ -406,7 +406,7 @@ int run_hooks(const char *hookname, struct run_hooks_opt *options) pick_next_hook, notify_start_failure, options->feed_pipe, -NULL , + options->consume_sideband, notify_hook_finished, &cb_data, "hook",
hook.h +10 -0
@@ -76,6 +76,14 @@ struct run_hooks_optfeed_pipe_fn feed_pipe;void *feed_pipe_ctx; +/* +* Populate this to capture output and prevent it from being printed to +* stderr. This will be passed directly through to +* run_command:run_parallel_processes(). See t/helper/test-run-command.c +* for an example. +*/ +consume_sideband_fn consume_sideband; +/* Number of threads to parallelize across */ int jobs;
@@ -103,6 +111,7 @@ struct hook_cb_data { .str_stdin = STRING_LIST_INIT_DUP, \ .feed_pipe = NULL, \ .feed_pipe_ctx = NULL, \ + .consume_sideband = NULL, \ .run_hookdir = configured_hookdir_opt() \ }
@@ -115,6 +124,7 @@ struct hook_cb_data { .str_stdin = STRING_LIST_INIT_DUP, \ .feed_pipe = NULL, \ .feed_pipe_ctx = NULL, \ + .consume_sideband = NULL, \ .run_hookdir = configured_hookdir_opt() \ }
Signature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
Re: [PATCH v7 00/17] propose config-based hooks (part I)
In reply to Emily ShafferQuoted 8 lines · Emily Shaffer
Since v6:
- Converted 'enum
hookdir_opt' toUPPER_SNAKE
- Coccinelle fix in the hook destructor
- Fixed a bug wherebuiltin/hook.cwasn't running the default git config setup and therefore missed hooks in core.hooksPath when it was set. (These hooks would still run except when invoked by 'git hook run' as the config was called by the processes which invoked the hook library.)
Thanks. Queued both series (it probably is easier to think of these as a single 34-patch series, as long as they both are in flight at the same time).
Re: [PATCH v7 00/17] propose config-based hooks (part I)
Quoted 12 lines · Junio C Hamano
Quoted 8 lines · Emily Shaffer
Since v6:
- Converted 'enum
hookdir_opt' toUPPER_SNAKE
- Coccinelle fix in the hook destructor
- Fixed a bug wherebuiltin/hook.cwasn't running the default git config setup and therefore missed hooks in core.hooksPath when it was set. (These hooks would still run except when invoked by 'git hook run' as the config was called by the processes which invoked the hook library.)
Quoted 4 lines
Thanks. Queued both series (it probably is easier to think of these as a single 34-patch series, as long as they both are in flight at the same time).
Do you want me to send them as a single thread for next version?
Re: [PATCH v7 00/17] propose config-based hooks (part I)
Quoted 13 lines · Emily Shaffer
Quoted 12 lines · Junio C Hamano
Quoted 8 lines · Emily Shaffer
Since v6:
- Converted 'enum
hookdir_opt' toUPPER_SNAKE
- Coccinelle fix in the hook destructor
- Fixed a bug wherebuiltin/hook.cwasn't running the default git config setup and therefore missed hooks in core.hooksPath when it was set. (These hooks would still run except when invoked by 'git hook run' as the config was called by the processes which invoked the hook library.)
Quoted 6 lines
Quoted 4 lines
Thanks. Queued both series (it probably is easier to think of these as a single 34-patch series, as long as they both are in flight at the same time).
Do you want me to send them as a single thread for next version?
Unless we deliberately focus on stabilizing the early 17 patches into a shape that they won't need updating while working on the later part of the series, I'd guess that your next resend would contain updated versions of these 17 patches, so the only effect that it has to pretend that the patches belong to two separate series is to invite mistakes while queuing on my part. So either
(1) a single thread of all patches, or (2) just the early part to really make sure everybody is happy with them, so that we can graduate it early even while the remainder may be going through revisions, would be more preferrable than the way they have been structured so far.
Thanks.
[PATCH v3 18/17] doc: make git-hook.txt point of truth
In reply to Emily ShafferBy showing the list of all hooks in 'git help hook' for users to refer to, 'git help hook' becomes a one-stop shop for hook authorship. Since some may still have muscle memory for 'git help githooks', though, reference the 'git hook' commands and otherwise don't remove content.
---
Sorry for the wonky subject. It seemed unnecessary to send the entirety of the topic again when there were no changes. (I was able to push this patch to my fork without -f, so indeed the rest is unchanged.)
I'd really prefer if 'git help githooks' opens the 'git-hook.txt' manpage, but I couldn't figure out how to do that. This seemed like the next best thing to me - users can still find information at the old manpage, but get a little nudge towards the new manpage in case they didn't notice it.
Extremely open to other notes about the direction of these docs; I'm not confident in anything except that it'd be annoying to have 'git help githooks' remain unchanged.
- Emily
Documentation/git-hook.txt | 5 + Documentation/githooks.txt | 701 +------------------------------------ 2 files changed, 13 insertions(+), 693 deletions(-)
Documentation/git-hook.txt +5 -0
@@ -112,6 +112,11 @@ CONFIGURATION ------------- include::config/hook.txt[] +HOOKS +----- + +include::native-hooks.txt[] + GIT --- Part of the linkgit:git[1] suite
Documentation/githooks.txt +8 -693
720 lines not on this page · raw message
Signature · 1 line
2.29.2.490.gc7ae633391
Re: [PATCH v3 18/17] doc: make git-hook.txt point of truth
On Mon, Dec 28, 2020 at 02:37:16PM -0800, Emily Shaffer wrote:
Argh. I am having awful Monday brain and this should have been in-reply-to the other thread. I guess that's a point in opposition of splitting big topics into multiple threads. :|
I'll resend it on the other topic. I'm sorry.
- Emily
Re: [PATCH v7 00/17] propose config-based hooks (part I)
In reply to Emily ShafferQuoted 11 lines · Emily Shaffer
Since v6:
- Converted 'enum
hookdir_opt' toUPPER_SNAKE
- Coccinelle fix in the hook destructor
- Fixed a bug wherebuiltin/hook.cwasn't running the default git config setup and therefore missed hooks in core.hooksPath when it was set. (These hooks would still run except when invoked by 'git hook run' as the config was called by the processes which invoked the hook library.)CI run: https://github.com/nasamuffin/git/actions/runs/436864964
Some updates on this series...
Since Jan 21 we've been running this series as picked from gitster/git:es/config-hooks on Googler machines, with a subset of users asked to try out putting their hooks into config instead of hookdir. So far we haven't heard any crashes or bugs like that, although I did hear a couple places where the user documentation is lacking. I feel encouraged by that, and I'm hoping to improve the documentation in the next week or so, pending $DAYJOB concerns.
We also addressed some of this series in our every-other-week review club (me, Jonathan Tan, Jonathan Nieder, and Josh Steadmon; although in this case I tried to be quiet :) ) and so I hope there will be some comments from my three teammates coming to list sometime next week.
Since I feel pretty comfortable that it doesn't seem to explode anywhere, I'm really keen to hear nitpicky reviews and try to push to get this into 'next'; maybe I can barter my eyes on someone else's neglected review? That sounds pretty mercenary but I think Junio is the one who suggested it a few weeks ago... ;)
- Emily
Re: [PATCH v7 00/17] propose config-based hooks (part I)
In reply to Emily ShafferQuoted 57 lines · Emily Shaffer
Since v6:
- Converted 'enum
hookdir_opt' toUPPER_SNAKE
- Coccinelle fix in the hook destructor
- Fixed a bug wherebuiltin/hook.cwasn't running the default git config setup and therefore missed hooks in core.hooksPath when it was set. (These hooks would still run except when invoked by 'git hook run' as the config was called by the processes which invoked the hook library.)CI run: https://github.com/nasamuffin/git/actions/runs/436864964
Thanks!
- EmilyEmily Shaffer (17):
doc: propose hooks managed by the config
hook: scaffolding for git-hook subcommand
hook: add list command
hook: include hookdir hook in list
hook: respect hook.runHookDir
hook: implement hookcmd.<name>.skip
parse-options: parse into strvec
hook: add 'run' subcommand
hook: replacefind_hook()withhook_exists()
hook: support passing stdin to hooks
run-command: allow stdin forrun_processes_parallel
hook: allow parallel hook execution
hook: allow specifying working directory for hooks
run-command: add stdin callback for parallelization
hook: provide stdin bystring_listor callback
run-command: allow capturing of collated output
hooks: allow callers to capture output.gitignore | 1 + Documentation/Makefile | 1 + Documentation/config/hook.txt | 19 + Documentation/git-hook.txt | 117 +++++ Documentation/technical/api-parse-options.txt | 5 + .../technical/config-based-hooks.txt | 355 +++++++++++++++ Makefile | 2 + builtin.h | 1 + builtin/bugreport.c | 4 +- builtin/fetch.c | 1 + builtin/hook.c | 176 ++++++++ builtin/submodule--helper.c | 2 +- command-list.txt | 1 + git.c | 1 + hook.c | 416 ++++++++++++++++++ hook.h | 154 +++++++ parse-options-cb.c | 16 + parse-options.h | 4 + run-command.c | 85 +++- run-command.h | 31 ++ submodule.c | 1 + t/helper/test-run-command.c | 46 +- t/t0061-run-command.sh | 37 ++ t/t1360-config-based-hooks.sh | 256 +++++++++++
Quoted 12 lines
24 files changed, 1717 insertions(+), 15 deletions(-) create mode 100644 Documentation/config/hook.txt create mode 100644 Documentation/git-hook.txt create mode 100644 Documentation/technical/config-based-hooks.txt create mode 100644 builtin/hook.c create mode 100644 hook.c create mode 100644 hook.h create mode 100755 t/t1360-config-based-hooks.shSignature · 1 line
2.28.0.rc0.142.g3c755180ce-goog
Sorry for the delayed reply. I am happy with this series as-is. Thanks for all your work on it!
Re: [PATCH v7 00/17] propose config-based hooks (part I)
Quoted 19 lines · Josh Steadmon
Quoted 10 lines
Emily Shaffer (17):
doc: propose hooks managed by the config
hook: scaffolding for git-hook subcommand
hook: add list command
hook: include hookdir hook in list
hook: respect hook.runHookDir
hook: implement hookcmd.<name>.skip
parse-options: parse into strvec
hook: add 'run' subcommand
hook: replacefind_hook()withhook_exists()Quoted 9 lines
hook: support passing stdin to hooks
run-command: allow stdin forrun_processes_parallel
hook: allow parallel hook execution
hook: allow specifying working directory for hooks
run-command: add stdin callback for parallelization
hook: provide stdin bystring_listor callback
run-command: allow capturing of collated output
hooks: allow callers to capture output
Quoted 4 lines
Sorry for the delayed reply. I am happy with this series as-is. Thanks for all your work on it!
Reviewed-by Josh Steadmon <[email protected]>
The topic branch has a lot more commits than these 17; I am wondering if the reviewed-by applies only to the bottom 17, or as the whole? I recall that the upper half was expecting at least some documentation updates.
Thanks.
Re: [PATCH v7 00/17] propose config-based hooks (part I)
Quoted 24 lines · Junio C Hamano
Quoted 22 lines · Josh Steadmon
Quoted 10 lines
Emily Shaffer (17):
doc: propose hooks managed by the config
hook: scaffolding for git-hook subcommand
hook: add list command
hook: include hookdir hook in list
hook: respect hook.runHookDir
hook: implement hookcmd.<name>.skip
parse-options: parse into strvec
hook: add 'run' subcommand
hook: replacefind_hook()withhook_exists()Quoted 9 lines
hook: support passing stdin to hooks
run-command: allow stdin forrun_processes_parallel
hook: allow parallel hook execution
hook: allow specifying working directory for hooks
run-command: add stdin callback for parallelization
hook: provide stdin bystring_listor callback
run-command: allow capturing of collated output
hooks: allow callers to capture outputSorry for the delayed reply. I am happy with this series as-is. Thanks for all your work on it!
Quoted 8 lines
Quoted 1 line
Reviewed-by Josh Steadmon <[email protected]>The topic branch has a lot more commits than these 17; I am wondering if the reviewed-by applies only to the bottom 17, or as the whole? I recall that the upper half was expecting at least some documentation updates.
Thanks.
Just to these 17, sorry for being unclear.
Re: [PATCH v7 00/17] propose config-based hooks (part I)
Quoted 8 lines · Josh Steadmon
Quoted 6 lines
The topic branch has a lot more commits than these 17; I am wondering if the reviewed-by applies only to the bottom 17, or as the whole? I recall that the upper half was expecting at least some documentation updates.
Thanks.
Just to these 17, sorry for being unclear.
Thanks for reading them through.
I am tempted to say we should merge these "mechanism" part down to 'next', hoping that the "rewrite existing ones using the new mechansim" part can follow soon.
Re: [PATCH v7 00/17] propose config-based hooks (part I)
Quoted 9 lines · Junio C Hamano
Quoted 7 lines · Josh Steadmon
Quoted 6 lines
The topic branch has a lot more commits than these 17; I am wondering if the reviewed-by applies only to the bottom 17, or as the whole? I recall that the upper half was expecting at least some documentation updates.
Thanks.
Quoted 7 lines
Quoted 1 line
Just to these 17, sorry for being unclear.
Thanks for reading them through.
I am tempted to say we should merge these "mechanism" part down to 'next', hoping that the "rewrite existing ones using the new mechansim" part can follow soon.
I said this on Feb 17th, but since then I think I saw you answer "I'll do that" in responses to JTan's reviews in the past few days (e.g. <[email protected]>). Would I regret if I merge the topic down to 'next' today?
Thanks.
Re: [PATCH v7 00/17] propose config-based hooks (part I)
Quoted 16 lines · Junio C Hamano
Quoted 13 lines · Junio C Hamano
Quoted 8 lines · Josh Steadmon
Quoted 6 lines
The topic branch has a lot more commits than these 17; I am wondering if the reviewed-by applies only to the bottom 17, or as the whole? I recall that the upper half was expecting at least some documentation updates.
Thanks.
Just to these 17, sorry for being unclear.
Thanks for reading them through.
Quoted 8 lines
Quoted 3 lines
I am tempted to say we should merge these "mechanism" part down to 'next', hoping that the "rewrite existing ones using the new mechansim" part can follow soon.
I said this on Feb 17th, but since then I think I saw you answer "I'll do that" in responses to JTan's reviews in the past few days (e.g. <[email protected]>). Would I regret if I merge the topic down to 'next' today?
Bah, I'm sorry I missed this - I had a broken mutt config and wasn't seeing replies, my own fault. Argh.
I have some pretty significant changes from JTan's reviews, so I'd prefer if you would wait since it would be tricky to turn them into a patch commit now. But if you'd rather merge it and see a patch instead, that is fine with me.
- Emily
Re: [PATCH v7 00/17] propose config-based hooks (part I)
Quoted 5 lines · Emily Shaffer
Quoted 4 lines
I said this on Feb 17th, but since then I think I saw you answer "I'll do that" in responses to JTan's reviews in the past few days (e.g. <[email protected]>). Would I regret if I merge the topic down to 'next' today?
Quoted 7 lines
Bah, I'm sorry I missed this - I had a broken mutt config and wasn't seeing replies, my own fault. Argh.
I have some pretty significant changes from JTan's reviews, so I'd prefer if you would wait since it would be tricky to turn them into a patch commit now. But if you'd rather merge it and see a patch instead, that is fine with me.
OK, I still have it outside 'next', I think.
v8 cover config-based hooks
In reply to Emily ShafferSince v7:
- Addressed Jonathan Tan's review of part I
- Addressed Junio's review of part I and II
- Combined parts I and II
I think the updates to patch 1 between the rest of the work I've been doing probably have covered Ævar's comments.
More details about per-patch changes found in the notes on each mail (I hope).
I know that Junio was talking about merging v7 after Josh Steadmon's review and I asked him not to - this reroll has those changes from Jonathan Tan's review that I was wanting to wait for.
Thanks!
- Emily
Emily Shaffer (37):
doc: propose hooks managed by the config
hook: scaffolding for git-hook subcommand
hook: add list command
hook: include hookdir hook in list
hook: teach hook.runHookDir
hook: implement hookcmd.<name>.skip
parse-options: parse into strvec
hook: add 'run' subcommand
hook: introduce hook_exists()
hook: support passing stdin to hooks
run-command: allow stdin for run_processes_parallel
hook: allow parallel hook execution
hook: allow specifying working directory for hooks
run-command: add stdin callback for parallelization
hook: provide stdin by string_list or callback
run-command: allow capturing of collated output
hooks: allow callers to capture output
commit: use config-based hooks
am: convert applypatch hooks to use config
merge: use config-based hooks for post-merge hook
gc: use hook library for pre-auto-gc hook
rebase: teach pre-rebase to use hook.h
read-cache: convert post-index-change hook to use config
receive-pack: convert push-to-checkout hook to hook.h
git-p4: use 'git hook' to run hooks
hooks: convert 'post-checkout' hook to hook library
hook: convert 'post-rewrite' hook to config
transport: convert pre-push hook to use config
reference-transaction: look for hooks in config
receive-pack: convert 'update' hook to hook.h
proc-receive: acquire hook list from hook.h
post-update: use hook.h library
receive-pack: convert receive hooks to hook.h
bugreport: use hook_exists instead of find_hook
git-send-email: use 'git hook run' for 'sendemail-validate'
run-command: stop thinking about hooks
docs: unify githooks and git-hook manpages
.gitignore | 1 + Documentation/Makefile | 1 + Documentation/config/hook.txt | 27 + Documentation/git-hook.txt | 161 ++++ Documentation/githooks.txt | 655 +--------------- Documentation/native-hooks.txt | 708 ++++++++++++++++++ Documentation/technical/api-parse-options.txt | 7 + .../technical/config-based-hooks.txt | 369 +++++++++ Makefile | 2 + builtin.h | 1 + builtin/am.c | 33 +- builtin/bugreport.c | 4 +- builtin/checkout.c | 19 +- builtin/clone.c | 8 +- builtin/commit.c | 11 +- builtin/fetch.c | 1 + builtin/gc.c | 5 +- builtin/hook.c | 176 +++++ builtin/merge.c | 15 +- builtin/rebase.c | 9 +- builtin/receive-pack.c | 329 ++++---- builtin/submodule--helper.c | 2 +- builtin/worktree.c | 31 +- command-list.txt | 1 + commit.c | 22 +- commit.h | 3 +- git-p4.py | 67 +- git-send-email.perl | 21 +- git.c | 1 + hook.c | 480 ++++++++++++ hook.h | 138 ++++ parse-options-cb.c | 16 + parse-options.h | 4 + read-cache.c | 13 +- refs.c | 43 +- reset.c | 16 +- run-command.c | 156 ++-- run-command.h | 55 +- sequencer.c | 90 +-- submodule.c | 1 + t/helper/test-run-command.c | 46 +- t/t0061-run-command.sh | 37 + t/t1360-config-based-hooks.sh | 303 ++++++++ t/t1416-ref-transaction-hooks.sh | 12 +- t/t5411/test-0015-too-many-hooks-error.sh | 47 ++ ...3-pre-commit-and-pre-merge-commit-hooks.sh | 17 +- t/t9001-send-email.sh | 11 +- transport.c | 59 +- 48 files changed, 3052 insertions(+), 1182 deletions(-) create mode 100644 Documentation/config/hook.txt create mode 100644 Documentation/git-hook.txt create mode 100644 Documentation/native-hooks.txt create mode 100644 Documentation/technical/config-based-hooks.txt create mode 100644 builtin/hook.c create mode 100644 hook.c create mode 100644 hook.h create mode 100755 t/t1360-config-based-hooks.sh create mode 100644 t/t5411/test-0015-too-many-hooks-error.sh
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
PATCH v8 1/37 doc: propose hooks managed by the config
Begin a design document for config-based hooks, managed via git-hook. Focus on an overview of the implementation and motivation for design decisions. Briefly discuss the alternatives considered before this point. Also, attempt to redefine terms to fit into a multihook world.
---
Notes:
Since v7, made some wording changes based on reviewer comments (mostly Junio's, I think).
- Emily
Documentation/Makefile | 1 + .../technical/config-based-hooks.txt | 369 ++++++++++++++++++ 2 files changed, 370 insertions(+) create mode 100644 Documentation/technical/config-based-hooks.txt
Documentation/Makefile +1 -0
@@ -82,6 +82,7 @@ SP_ARTICLES += $(API_DOCS) TECH_DOCS += MyFirstContribution TECH_DOCS += MyFirstObjectWalk TECH_DOCS += SubmittingPatches +TECH_DOCS += technical/config-based-hooks TECH_DOCS += technical/hash-function-transition TECH_DOCS += technical/http-protocol TECH_DOCS += technical/index-format
Documentation/technical/config-based-hooks.txt new file +369 -0
369 lines not on this page · raw message
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
PATCH v8 2/37 hook: scaffolding for git-hook subcommand
In reply to Emily ShafferIntroduce infrastructure for a new subcommand, git-hook, which will be used to ease config-based hook management. This command will handle parsing configs to compose a list of hooks to run for a given event, as well as adding or modifying hook configs in an interactive fashion.
---
No change since v7.
.gitignore | 1 + Documentation/git-hook.txt | 20 ++++++++++++++++++++ Makefile | 1 + builtin.h | 1 + builtin/hook.c | 21 +++++++++++++++++++++ command-list.txt | 1 + git.c | 1 + t/t1360-config-based-hooks.sh | 11 +++++++++++ 8 files changed, 57 insertions(+) create mode 100644 Documentation/git-hook.txt create mode 100644 builtin/hook.c create mode 100755 t/t1360-config-based-hooks.sh
.gitignore +1 -0
@@ -76,6 +76,7 @@ /git-grep /git-hash-object /git-help +/git-hook /git-http-backend /git-http-fetch /git-http-push
Documentation/git-hook.txt new file +20 -0
@@ -0,0 +1,20 @@ +git-hook(1) +=========== + +NAME +---- +git-hook - Manage configured hooks + +SYNOPSIS +-------- +[verse] +'git hook' + +DESCRIPTION +----------- +A placeholder command. Later, you will be able to list, add, and modify hooks +with this command. + +GIT +--- +Part of the linkgit:git[1] suite
Makefile +1 -0
@@ -1087,6 +1087,7 @@ BUILTIN_OBJS += builtin/get-tar-commit-id.o BUILTIN_OBJS += builtin/grep.o BUILTIN_OBJS += builtin/hash-object.o BUILTIN_OBJS += builtin/help.o +BUILTIN_OBJS += builtin/hook.o BUILTIN_OBJS += builtin/index-pack.o BUILTIN_OBJS += builtin/init-db.o BUILTIN_OBJS += builtin/interpret-trailers.o
builtin.h +1 -0
@@ -163,6 +163,7 @@ int cmd_get_tar_commit_id(int argc, const char **argv, const char *prefix);int cmd_grep (int argc,const char **argv,const char *prefix);int cmd_hash_object (int argc,const char **argv,const char *prefix);int cmd_help (int argc,const char **argv,const char *prefix); +int cmd_hook (int argc,const char **argv,const char *prefix);int cmd_index_pack (int argc,const char **argv,const char *prefix);int cmd_init_db (int argc,const char **argv,const char *prefix);int cmd_interpret_trailers (int argc,const char **argv,const char *prefix);
builtin/hook.c new file +21 -0
@@ -0,0 +1,21 @@ +#include "cache.h" + +#include "builtin.h" +#include "parse-options.h" + +static const char *const builtin_hook_usage[] = { +N_ ("git hook" ), +NULL +}; + +int cmd_hook (int argc,const char **argv,const char *prefix) +{ +struct option builtin_hook_options[] = { +OPT_END (), + }; + + argc =parse_options (argc, argv, prefix, builtin_hook_options, + builtin_hook_usage,0 ); + +return 0 ; +}
command-list.txt +1 -0
@@ -103,6 +103,7 @@ git-grep mainporcelain info git-gui mainporcelain git-hash-object plumbingmanipulators git-help ancillaryinterrogators complete +git-hook mainporcelain git-http-backend synchingrepositories git-http-fetch synchelpers git-http-push synchelpers
git.c +1 -0
@@ -528,6 +528,7 @@ static struct cmd_struct commands[] = { { "grep", cmd_grep, RUN_SETUP_GENTLY }, { "hash-object", cmd_hash_object }, { "help", cmd_help }, + { "hook", cmd_hook, RUN_SETUP_GENTLY }, { "index-pack", cmd_index_pack, RUN_SETUP_GENTLY | NO_PARSEOPT }, { "init", cmd_init_db }, { "init-db", cmd_init_db },
t/t1360-config-based-hooks.sh new file +11 -0
@@ -0,0 +1,11 @@ +#!/bin/bash + +test_description='config-managed multihooks, including git-hook command' + +. ./test-lib.sh + +test_expect_success 'git hook command does not crash' ' + git hook +' + +test_done
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
PATCH v8 3/37 hook: add list command
In reply to Emily ShafferTeach 'git hook list <hookname>', which checks the known configs in order to create an ordered list of hooks to run on a given hook event.
Multiple commands can be specified for a given hook by providing multiple "hook.<hookname>.command = <path-to-hook>" lines. Hooks will be run in config order. If more properties need to be set on a given hook in the future, commands can also be specified by providing "hook.<hookname>.command = <hookcmd-name>", as well as a "[hookcmd <hookcmd-name>]" subsection; this subsection should contain a "hookcmd.<hookcmd-name>.command = <path-to-hook>" line.
For example:
$ git config --list | grep ^hook hook.pre-commit.command=baz hook.pre-commit.command=~/bar.sh hookcmd.baz.command=~/baz/from/hookcmd.sh
$ git hook list pre-commit global: ~/baz/from/hookcmd.sh local: ~/bar.sh
---
Notes:
Since v7, fixed some nits from Jonathan Tan, one of which revealed a bug in how I was adding hooks to the list.
Documentation/config/hook.txt | 9 +++ Documentation/git-hook.txt | 59 ++++++++++++++++- Makefile | 1 + builtin/hook.c | 56 ++++++++++++++-- hook.c | 120 ++++++++++++++++++++++++++++++++++ hook.h | 25 +++++++ t/t1360-config-based-hooks.sh | 81 ++++++++++++++++++++++- 7 files changed, 341 insertions(+), 10 deletions(-) create mode 100644 Documentation/config/hook.txt create mode 100644 hook.c create mode 100644 hook.h
Documentation/config/hook.txt new file +9 -0
@@ -0,0 +1,9 @@ +hook.<command>.command:: + A command to execute during the <command> hook event. This can be an + executable on your device, a oneliner for your shell, or the name of a + hookcmd. See linkgit:git-hook[1]. + +hookcmd.<name>.command:: + A command to execute during a hook for which <name> has been specified + as a command. This can be an executable on your device or a oneliner for + your shell. See linkgit:git-hook[1].
Documentation/git-hook.txt +56 -3
68 lines not on this page · raw message
Makefile +1 -0
@@ -891,6 +891,7 @@ LIB_OBJS += hash-lookup.o LIB_OBJS += hashmap.o LIB_OBJS += help.o LIB_OBJS += hex.o +LIB_OBJS += hook.o LIB_OBJS += ident.o LIB_OBJS += json-writer.o LIB_OBJS += kwset.o
builtin/hook.c +51 -5
72 lines not on this page · raw message
hook.c new file +120 -0
120 lines not on this page · raw message
hook.h new file +25 -0
@@ -0,0 +1,25 @@ +#include "config.h" +#include "list.h" +#include "strbuf.h" + +struct hook { +struct list_head list; +/* +* Config file which holds the hook.*.command definition. +* (This has nothing to do with the hookcmd.<name>.* configs.) +*/ +enum config_scope origin; +/* The literal command to run. */ +struct strbuf command; +}; + +/* +* Provides a linked list of 'struct hook' detailing commands which should run +* in response to the 'hookname' event, in execution order. +*/ +struct list_head *hook_list (const struct strbuf *hookname); + +/* Free memory associated with a 'struct hook' */ +void free_hook (struct hook *ptr); +/* Empties the list at 'head', calling 'free_hook()' on each entry */ +void clear_hook_list (struct list_head *head);
t/t1360-config-based-hooks.sh +79 -2
87 lines not on this page · raw message
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
Re: [PATCH v8 03/37] hook: add list command
Quoted 20 lines · Emily Shaffer
Documentation/config/hook.txt new file +9
-0@@ -0,0 +1,9 @@ +hook.<command>.command:: + A command to execute during the <command> hook event. This can be an + executable on your device, a oneliner for your shell, or the name of a + hookcmd. See linkgit:git-hook[1]. + +hookcmd.<name>.command:: + A command to execute during a hook for which <name> has been specified + as a command. This can be an executable on your device or a oneliner for + your shell. See linkgit:git-hook[1].Documentation/git-hook.txt +0
-0
Documentation/git-hook.txt git-hook - Manage configured hooks · line 11
Quoted 5 lines
Documentation/git-hook.txt +1
-1@@ -8,12 +8,65 @@ git-hook - Manage configured hooks SYNOPSIS -------- [verse] -'git hook' +'git hook' list <hook-name>
Having just read this far (maybe this pattern is shared in the rest of the series): Let's just squash this and the 2nd patch together.
Sometimes it's worth doing the scaffolding first, but adding a new built-in is so trivial that I don't think it's worth it, and it just results in back & forth churn like the above...
Documentation/git-hook.txt git-hook - Manage configured hooks · line 16
Quoted 4 lines
Documentation/git-hook.txt +0
-2DESCRIPTION ----------- -A placeholder command. Later, you will be able to list, add, and modify hooks -with this command.
...and this...
Quoted 75 lines
Documentation/git-hook.txt +55
-0+You can list configured hooks with this command. Later, you will be able to run, +add, and modify hooks with this command. + +This command parses the default configuration files for sections `hook` and +`hookcmd`. `hook` is used to describe the commands which will be run during a +particular hook event; commands are run in the order Git encounters them during +the configuration parse (see linkgit:git-config[1]). `hookcmd` is used to +describe attributes of a specific command. If additional attributes don't need +to be specified, a command to run can be specified directly in the `hook` +section; if a `hookcmd` by that name isn't found, Git will attempt to run the +provided value directly. For example: + +Global config +---- + [hook "post-commit"] + command = "linter" + command = "~/typocheck.sh" + + [hookcmd "linter"] + command = "/bin/linter --c" +---- + +Local config +---- + [hook "prepare-commit-msg"] + command = "linter" + [hook "post-commit"] + command = "python ~/run-test-suite.py" +---- + +With these configs, you'd then see: + +---- +$ git hook list "post-commit" +global: /bin/linter --c +global: ~/typocheck.sh +local: python ~/run-test-suite.py + +$ git hook list "prepare-commit-msg" +local: /bin/linter --c +---- + +COMMANDS +-------- + +list `<hook-name>`:: + +List the hooks which have been configured for `<hook-name>`. Hooks appear +in the order they should be run, and print the config scope where the relevant +`hook.<hook-name>.command` was specified, not the `hookcmd` (if applicable). +This output is human-readable and the format is subject to change over time. + +CONFIGURATION +------------- +include::config/hook.txt[] GIT ---Makefile +1
-0@@ -891,6 +891,7 @@ LIB_OBJS += hash-lookup.o LIB_OBJS += hashmap.o LIB_OBJS += help.o LIB_OBJS += hex.o +LIB_OBJS += hook.o LIB_OBJS += ident.o LIB_OBJS += json-writer.o LIB_OBJS += kwset.obuiltin/hook.c +0
-0
builtin/hook.c line 2
Quoted 2 lines
builtin/hook.c +0
-1@@ -1,21 +1,67 @@#include "cache.h" -
Stray back & forth whitespace churn?
Quoted 4 lines
builtin/hook.c +2
-0#include "builtin.h" +#include "config.h" +#include "hook.h" #include "parse-options.h"
builtin/hook.c line 13
Quoted 10 lines
builtin/hook.c +3
-2+#include "strbuf.h" static const char *const builtin_hook_usage[] = { -N_ ("git hook" ), +N_ ("git hook list <hookname>" ),NULL }; -int cmd_hook (int argc,const char **argv,const char *prefix) +static int list (int argc,const char **argv,const char *prefix)
...and here the cmd_hook() function being replaced (not really, just moved below, but you get my drift...)
Quoted 12 lines
builtin/hook.c +5
-2{ -struct option builtin_hook_options[] = { +struct list_head *head, *pos; +struct strbuf hookname =STRBUF_INIT ; + +struct option list_options[] = {OPT_END (), }; - argc =parse_options (argc, argv, prefix, builtin_hook_options, + argc =parse_options (argc, argv, prefix, list_options, builtin_hook_usage,0 );
builtin/hook.c line 33
Quoted 10 lines
builtin/hook.c +9
-0+if (argc <1 ) { +usage_msg_opt (_ ("You must specify a hook event name to list." ), + builtin_hook_usage, list_options); + } + +strbuf_addstr (&hookname, argv[0 ]); + + head =hook_list (&hookname); +
More on strbuf usage later in another soon-to-be-sent E-Mail.
Quoted 42 lines
builtin/hook.c +32
-0+if (list_empty (head)) { +printf (_ ("no commands configured for hook '%s'\n" ), + hookname.buf); +strbuf_release (&hookname); +return 0 ; + } + +list_for_each (pos, head) { + struct hook *item = list_entry(pos, struct hook, list); +if (item) +printf ("%s: %s\n" , +config_scope_name (item->origin), + item->command.buf); + } + +clear_hook_list (head); +strbuf_release (&hookname); +return 0 ; } + +int cmd_hook (int argc,const char **argv,const char *prefix) +{ +struct option builtin_hook_options[] = { +OPT_END (), + }; +if (argc <2 ) +usage_with_options (builtin_hook_usage, builtin_hook_options); + +if (!strcmp (argv[1 ],"list" )) +return list (argc -1 , argv +1 , prefix); + +usage_with_options (builtin_hook_usage, builtin_hook_options); +}hook.c new file +2
-0@@ -0,0 +1,120 @@ +#include "cache.h" +
hook.c line 12
Quoted 10 lines
hook.c new file +10
-0+#include "hook.h" +#include "config.h" + +void free_hook (struct hook *ptr) +{ +if (ptr) { +strbuf_release (&ptr->command); +free (ptr); + } +}
Neither strbuf_release() nor free() need or should have a "if (ptr)" guard.
Quoted 90 lines
hook.c new file +90
-0+ +static void append_or_move_hook (struct list_head *head,const char *command) +{ +struct list_head *pos =NULL , *tmp =NULL ; +struct hook *to_add =NULL ; + +/* +* remove the prior entry with this command; we'll replace it at the +* end. +*/ +list_for_each_safe (pos, tmp, head) { + struct hook *it = list_entry(pos, struct hook, list); +if (!strcmp (it->command.buf, command)) { +list_del (pos); +/* we'll simply move the hook to the end */ + to_add = it; +break ; + } + } + +if (!to_add) { +/* adding a new hook, not moving an old one */ + to_add =xmalloc (sizeof (*to_add)); +strbuf_init (&to_add->command,0 ); +strbuf_addstr (&to_add->command, command); + } + +/* re-set the scope so we show where an override was specified */ + to_add->origin =current_config_scope (); + +list_add_tail (&to_add->list, head); +} + +static void remove_hook (struct list_head *to_remove) +{ + struct hook *hook_to_remove = list_entry(to_remove, struct hook, list); +list_del (to_remove); +free_hook (hook_to_remove); +} + +void clear_hook_list (struct list_head *head) +{ +struct list_head *pos, *tmp; +list_for_each_safe (pos, tmp, head) +remove_hook (pos); +} + +struct hook_config_cb +{ +struct strbuf *hookname; +struct list_head *list; +}; + +static int hook_config_lookup (const char *key,const char *value,void *cb_data) +{ +struct hook_config_cb *data = cb_data; +const char *hook_key = data->hookname->buf; +struct list_head *head = data->list; + +if (!strcmp (key, hook_key)) { +const char *command = value; +struct strbuf hookcmd_name =STRBUF_INIT ; + +/* +* Check if a hookcmd with that name exists. If it doesn't, +* 'git_config_get_value()' is documented not to touch &command, +* so we don't need to do anything. +*/ +strbuf_addf (&hookcmd_name,"hookcmd.%s.command" , command); +git_config_get_value (hookcmd_name.buf, &command); + +if (!command) { +strbuf_release (&hookcmd_name); +BUG ("git_config_get_value overwrote a string it shouldn't have" ); + } + +/* +* TODO: implement an option-getting callback, e.g. +* get configs by pattern hookcmd.$value.* +* for each key+value, do_callback(key, value, cb_data) +*/ + +append_or_move_hook (head, command); + +strbuf_release (&hookcmd_name); + } + +return 0 ; +} +
hook.c line 112
Quoted 10 lines
hook.c new file +10
-0+struct list_head *hook_list (const struct strbuf * hookname) +{ +struct strbuf hook_key =STRBUF_INIT ; +struct list_head *hook_head =xmalloc (sizeof (struct list_head )); +struct hook_config_cb cb_data = { &hook_key, hook_head }; + +INIT_LIST_HEAD (hook_head); + +if (!hookname) +return NULL ;
...if a strbuf being passed in is NULL?
hook.c
Quoted 9 lines
[...]
hook.c new file +8
-0+ROOT = +if test_have_prereq MINGW +then +# In Git for Windows, Unix-like paths work only in shell scripts; + # `git.exe`, however, will prefix them with the pseudo root directory + # (of the Unix shell). Let's accommodate for that. +ROOT ="$(cd / && pwd)" +fi
I didn't read up on previous rounds, but if we're squashing this into 02 having a seperate commit summarizing this little hack would be most welcome, or have it in this commit message.
Isn't this sort of thing generally usable, maybe we can add it under a longer variable name to test-lib.sh?
Re: [PATCH v8 03/37] hook: add list command
Quoted 33 lines · Ævar Arnfjörð Bjarmason
Quoted 20 lines · Emily Shaffer
Documentation/config/hook.txt new file +9
-0@@ -0,0 +1,9 @@ +hook.<command>.command:: + A command to execute during the <command> hook event. This can be an + executable on your device, a oneliner for your shell, or the name of a + hookcmd. See linkgit:git-hook[1]. + +hookcmd.<name>.command:: + A command to execute during a hook for which <name> has been specified + as a command. This can be an executable on your device or a oneliner for + your shell. See linkgit:git-hook[1].Documentation/git-hook.txt +0
-0Documentation/git-hook.txt git-hook - Manage configured hooks · line 11
Quoted 5 lines
Documentation/git-hook.txt +1
-1@@ -8,12 +8,65 @@ git-hook - Manage configured hooks SYNOPSIS -------- [verse] -'git hook' +'git hook' list <hook-name>Having just read this far (maybe this pattern is shared in the rest of the series): Let's just squash this and the 2nd patch together.
Quoted 3 lines
Sometimes it's worth doing the scaffolding first, but adding a new built-in is so trivial that I don't think it's worth it, and it just results in back & forth churn like the above...
Yeah, I think you are right here :)
Quoted 9 lines
Documentation/git-hook.txt git-hook - Manage configured hooks · line 18
Quoted 7 lines
Documentation/git-hook.txt +7
-0+void free_hook(struct hook *ptr) +{ + if (ptr) { + strbuf_release(&ptr->command); + free(ptr); + } +}Neither
strbuf_release()norfree()need or should have a "if (ptr)" guard.
I'll take free() out of the if guard, but I think 'strbuf_release(&<null>->command)' will go poorly - dereferencing the NULL to even invoke strbuf_release will not be a happy time, and strbuf_release internally is not NULL-resistant.
Quoted 4 lines
Quoted 4 lines
Documentation/git-hook.txt +4
-0+struct list_head* hook_list(const struct strbuf* hookname) +{ + struct strbuf hook_key = STRBUF_INIT; + struct list_head *hook_head = xmalloc(sizeof(struct list_head));
Quoted 8 lines
Documentation/git-hook.txt git-hook - Manage configured hooks · line 28
Quoted 6 lines
Documentation/git-hook.txt +6
-0+ struct hook_config_cb cb_data = { &hook_key, hook_head }; + + INIT_LIST_HEAD(hook_head); + + if (!hookname) + return NULL;...if a strbuf being passed in is NULL?
Yeah, I think this is misplaced. But since it sounds like generally folks don't like having the strbuf at the input here, I will address the error checking then also.
Quoted 14 lines
Documentation/git-hook.txt
Quoted 10 lines
[...]
Documentation/git-hook.txt +8
-0+ROOT= +if test_have_prereq MINGW +then + # In Git for Windows, Unix-like paths work only in shell scripts; + # `git.exe`, however, will prefix them with the pseudo root directory + # (of the Unix shell). Let's accommodate for that. + ROOT="$(cd / && pwd)" +fiI didn't read up on previous rounds, but if we're squashing this into 02 having a seperate commit summarizing this little hack would be most welcome, or have it in this commit message.
Sure. I squashed it in from a commit dscho sent, so I can preserve that commit in tree instead.
Quoted 3 lines
Isn't this sort of thing generally usable, maybe we can add it under a longer variable name to
test-lib.sh?
I wonder. git grep cd \/ && shows me that this hack also happens in t1509-root-work-tree.sh. I think most tests must use relative paths, so this must not be in broad use? But since it's not used elsewhere I feel ambivalent about adding a helper to test-lib.sh. I can if you feel strongly :)
- Emily
Re: [PATCH v8 03/37] hook: add list command
Quoted 50 lines · Emily Shaffer
Quoted 33 lines · �var Arnfj�r� Bjarmason
Quoted 20 lines · Emily Shaffer
Documentation/config/hook.txt new file +9
-0@@ -0,0 +1,9 @@ +hook.<command>.command:: + A command to execute during the <command> hook event. This can be an + executable on your device, a oneliner for your shell, or the name of a + hookcmd. See linkgit:git-hook[1]. + +hookcmd.<name>.command:: + A command to execute during a hook for which <name> has been specified + as a command. This can be an executable on your device or a oneliner for + your shell. See linkgit:git-hook[1].Documentation/git-hook.txt +0
-0Documentation/git-hook.txt git-hook - Manage configured hooks · line 11
Quoted 5 lines
Documentation/git-hook.txt +1
-1@@ -8,12 +8,65 @@ git-hook - Manage configured hooks SYNOPSIS -------- [verse] -'git hook' +'git hook' list <hook-name>Having just read this far (maybe this pattern is shared in the rest of the series): Let's just squash this and the 2nd patch together.
Quoted 3 lines
Sometimes it's worth doing the scaffolding first, but adding a new built-in is so trivial that I don't think it's worth it, and it just results in back & forth churn like the above...
Yeah, I think you are right here :)
Quoted 9 lines
Documentation/git-hook.txt git-hook - Manage configured hooks · line 18
Quoted 7 lines
Documentation/git-hook.txt +7
-0+void free_hook(struct hook *ptr) +{ + if (ptr) { + strbuf_release(&ptr->command); + free(ptr); + } +}Neither
strbuf_release()norfree()need or should have a "if (ptr)" guard.
Quoted 4 lines
I'll take
free()out of the if guard, but I think 'strbuf_release(&<null>->command)' will go poorly - dereferencing the NULL to even invokestrbuf_releasewill not be a happy time, andstrbuf_releaseinternally is not NULL-resistant.
Sorry I meant something like:
if (ptr) strbuf_release(&ptr->command);
free(ptr);But maybe even more idiomatic would be:
if (!ptr)
return;
strbuf_release(&ptr->command);
free(ptr);Or some other variant of checking teh container struct early. Anyway, this doesn't really matter, per a below comment I had more meaningful feedback in [1]. Most of my other traffic on this topic (including this) was some stream-of-consciousness notes as I went along.
Quoted 36 lines
Quoted 4 lines
Quoted 4 lines
Documentation/git-hook.txt +4
-0+struct list_head* hook_list(const struct strbuf* hookname) +{ + struct strbuf hook_key = STRBUF_INIT; + struct list_head *hook_head = xmalloc(sizeof(struct list_head));Quoted 8 lines
Documentation/git-hook.txt git-hook - Manage configured hooks · line 28
Quoted 6 lines
Documentation/git-hook.txt +6
-0+ struct hook_config_cb cb_data = { &hook_key, hook_head }; + + INIT_LIST_HEAD(hook_head); + + if (!hookname) + return NULL;...if a strbuf being passed in is NULL?
Yeah, I think this is misplaced. But since it sounds like generally folks don't like having the strbuf at the input here, I will address the error checking then also.
Quoted 14 lines
Documentation/git-hook.txt
Quoted 10 lines
[...]
Documentation/git-hook.txt +8
-0+ROOT= +if test_have_prereq MINGW +then + # In Git for Windows, Unix-like paths work only in shell scripts; + # `git.exe`, however, will prefix them with the pseudo root directory + # (of the Unix shell). Let's accommodate for that. + ROOT="$(cd / && pwd)" +fiI didn't read up on previous rounds, but if we're squashing this into 02 having a seperate commit summarizing this little hack would be most welcome, or have it in this commit message.
Sure. I squashed it in from a commit dscho sent, so I can preserve that commit in tree instead.
Quoted 1 line
Quoted 8 lines
Quoted 2 lines
Isn't this sort of thing generally usable, maybe we can add it under a longer variable name to
test-lib.sh?I wonder.
git grep cd \/ &&shows me that this hack also happens int1509-root-work-tree.sh. I think most tests must use relative paths, so this must not be in broad use? But since it's not used elsewhere I feel ambivalent about adding a helper totest-lib.sh. I can if you feel strongly :)
After I sent this I saw that pretty much the same thing is happening in t1300-config.sh for the --show-origin option.
! test_have_prereq MINGW ||
HOME="$(pwd)" # convert to Windows pathI don't feel strongly about this at all, but per the outstanding feedback I had in[1] I wondered whether this whole thing wouln't be better as some variant of "git config --show-origin",
PATCH v8 4/37 hook: include hookdir hook in list
In reply to Emily ShafferHistorically, hooks are declared by placing an executable into $GIT_DIR/hooks/$HOOKNAME (or $HOOKDIR/$HOOKNAME). Although hooks taken from the config are more featureful than hooks placed in the $HOOKDIR, those hooks should not stop working for users who already have them. Let's list them to the user, but instead of displaying a config scope (e.g. "global: blah") we can prefix them with "hookdir:".
---
Notes:
Since v7, fix some nits from Jonathan Tan. The largest is to move reference to "hookdir annotation" from this commit to the next one which introduces the hook.runHookDir option.
builtin/hook.c | 11 +++++++++-- hook.c | 17 +++++++++++++++++ hook.h | 1 + t/t1360-config-based-hooks.sh | 19 +++++++++++++++++++ 4 files changed, 46 insertions(+), 2 deletions(-)
builtin/hook.c +9 -2
@@ -40,10 +40,15 @@ static int list(int argc, const char **argv, const char *prefix)list_for_each (pos, head) { struct hook *item = list_entry(pos, struct hook, list); -if (item) + item = list_entry(pos, struct hook, list); +if (item) { +/* Don't translate 'hookdir' - it matches the config */ printf ("%s: %s\n" , -config_scope_name (item->origin), + (item->from_hookdir + ?"hookdir" + :config_scope_name (item->origin)), item->command.buf); + } }clear_hook_list (head);
@@ -60,6 +65,8 @@ int cmd_hook(int argc, const char **argv, const char *prefix)if (argc <2 )usage_with_options (builtin_hook_usage, builtin_hook_options); +git_config (git_default_config,NULL ); +if (!strcmp (argv[1 ],"list" ))return list (argc -1 , argv +1 , prefix);
hook.c +17 -0
@@ -2,6 +2,7 @@#include "hook.h" #include "config.h" +#include "run-command.h" void free_hook (struct hook *ptr) {
@@ -35,6 +36,7 @@ static void append_or_move_hook(struct list_head *head, const char *command) to_add =xmalloc (sizeof (*to_add));strbuf_init (&to_add->command,0 );strbuf_addstr (&to_add->command, command); + to_add->from_hookdir =0 ; }/* re-set the scope so we show where an override was specified */
@@ -115,6 +117,21 @@ struct list_head* hook_list(const struct strbuf* hookname)git_config (hook_config_lookup, &cb_data); +if (have_git_dir ()) { +const char *legacy_hook_path =find_hook (hookname->buf); + +/* Unconditionally add legacy hook, but annotate it. */ +if (legacy_hook_path) { +struct hook *legacy_hook; + +append_or_move_hook (hook_head, +absolute_path (legacy_hook_path)); + legacy_hook = list_entry(hook_head->prev, struct hook, + list); + legacy_hook->from_hookdir =1 ; + } + } +strbuf_release (&hook_key);return hook_head; }
hook.h +1 -0
@@ -11,6 +11,7 @@ struct hook {enum config_scope origin;/* The literal command to run. */ struct strbuf command; + unsigned from_hookdir : 1; };/*
t/t1360-config-based-hooks.sh +19 -0
@@ -23,6 +23,14 @@ setup_hookcmd () { test_config_global hookcmd.abc.command "/path/abc" --add } +setup_hookdir () { + mkdir .git/hooks + write_script .git/hooks/pre-commit <<-EOF + echo \"Legacy Hook\" + EOF + test_when_finished rm -rf .git/hooks +} + test_expect_success 'git hook rejects commands without a mode' ' test_must_fail git hook pre-commit '
@@ -85,4 +93,15 @@ test_expect_success 'git hook list reorders on duplicate commands' ' test_cmp expected actual ' +test_expect_success 'git hook list shows hooks from the hookdir' ' + setup_hookdir && + + cat >expected <<-EOF && + hookdir: $(pwd)/.git/hooks/pre-commit + EOF + + git hook list pre-commit >actual && + test_cmp expected actual +' + test_done
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
Re: [PATCH v8 04/37] hook: include hookdir hook in list
Quoted 26 lines · Emily Shaffer
Historically, hooks are declared by placing an executable into $GIT_DIR/hooks/$HOOKNAME (or $HOOKDIR/$HOOKNAME). Although hooks taken from the config are more featureful than hooks placed in the $HOOKDIR, those hooks should not stop working for users who already have them. Let's list them to the user, but instead of displaying a config scope (e.g. "global: blah") we can prefix them with "hookdir:".
Signed-off-by Emily Shaffer <[email protected]>---
Notes:
Since v7, fix some nits from Jonathan Tan. The largest is to move reference to "hookdir annotation" from this commit to the next one which introduces the hook.runHookDir option.builtin/hook.c | 11 +++++++++-- hook.c | 17 +++++++++++++++++ hook.h | 1 + t/t1360-config-based-hooks.sh | 19 +++++++++++++++++++ 4 files changed, 46 insertions(+), 2 deletions(-)builtin/hook.c +0
-0
builtin/hook.c static int list(int argc, const char **argv, const char *prefix) · line 45
Quoted 7 lines
builtin/hook.c +3
-1@@ -40,10 +40,15 @@ static int list(int argc, const char **argv, const char *prefix)list_for_each (pos, head) { struct hook *item = list_entry(pos, struct hook, list); -if (item) + item = list_entry(pos, struct hook, list); +if (item) { +/* Don't translate 'hookdir' - it matches the config */
Let's prefix comments for translators with /* TRANSLATORS: .., see the coding style doc. That's what they'll see, and this is useful to them.
Better yet have a note here about the first argument being 'system', 'local' etc., which I had to source spelunge for, and translators won't have any idea about unless the magic parameter is documented.
builtin/hook.c static int list(int argc, const char **argv, const char *prefix) · line 49
Quoted 4 lines
builtin/hook.c +4
-0+setup_hookdir () { + mkdir .git/hooks + write_script .git/hooks/pre-commit <<-EOF + echo \"Legacy Hook\"
Nit, "'s not needed, but it also seems nothing uses this, so if it's just a pass-through script either "exit 0", or actually check if it's run or something?
builtin/hook.c
Quoted 11 lines
[...]
builtin/hook.c +10
-0+test_expect_success 'git hook list shows hooks from the hookdir' ' + setup_hookdir && + + cat >expected <<-EOF && + hookdir: $(pwd)/.git/hooks/pre-commit +EOF + + git hook list pre-commit >actual && + test_cmp expected actual +'
Ah, so it's just checking if it exists...
Re: [PATCH v8 04/37] hook: include hookdir hook in list
Quoted 41 lines · Ævar Arnfjörð Bjarmason
Quoted 26 lines · Emily Shaffer
Historically, hooks are declared by placing an executable into $GIT_DIR/hooks/$HOOKNAME (or $HOOKDIR/$HOOKNAME). Although hooks taken from the config are more featureful than hooks placed in the $HOOKDIR, those hooks should not stop working for users who already have them. Let's list them to the user, but instead of displaying a config scope (e.g. "global: blah") we can prefix them with "hookdir:".
Signed-off-by Emily Shaffer <[email protected]>---
Notes:
Since v7, fix some nits from Jonathan Tan. The largest is to move reference to "hookdir annotation" from this commit to the next one which introduces the hook.runHookDir option.builtin/hook.c | 11 +++++++++-- hook.c | 17 +++++++++++++++++ hook.h | 1 + t/t1360-config-based-hooks.sh | 19 +++++++++++++++++++ 4 files changed, 46 insertions(+), 2 deletions(-)builtin/hook.c +0
-0builtin/hook.c static int list(int argc, const char **argv, const char *prefix) · line 45
Quoted 7 lines
builtin/hook.c +3
-1@@ -40,10 +40,15 @@ static int list(int argc, const char **argv, const char *prefix)list_for_each (pos, head) { struct hook *item = list_entry(pos, struct hook, list); -if (item) + item = list_entry(pos, struct hook, list); +if (item) { +/* Don't translate 'hookdir' - it matches the config */ Let's prefix comments for translators with /* TRANSLATORS: .., see the coding style doc. That's what they'll see, and this is useful to them.
Quoted 3 lines
Better yet have a note here about the first argument being 'system', 'local' etc., which I had to source spelunge for, and translators won't have any idea about unless the magic parameter is documented.
It's not a comment for translators. It's a comment for someone helpful who comes later and says "oh, none of this is marked for translation, I'd better fix that."
Quoted 9 lines
builtin/hook.c static int list(int argc, const char **argv, const char *prefix) · line 49
Quoted 5 lines
builtin/hook.c +4
-0+setup_hookdir () { + mkdir .git/hooks + write_script .git/hooks/pre-commit <<-EOF + echo \"Legacy Hook\"Nit, "'s not needed, but it also seems nothing uses this, so if it's just a pass-through script either "exit 0", or actually check if it's run or something?
The output is checked in the run tests later on. I can remove it for this commit if you want.
Quoted 14 lines
builtin/hook.c
Quoted 12 lines
[...]
builtin/hook.c +10
-0+test_expect_success 'git hook list shows hooks from the hookdir' ' + setup_hookdir && + + cat >expected <<-EOF && + hookdir: $(pwd)/.git/hooks/pre-commit +EOF + + git hook list pre-commit >actual && + test_cmp expected actual +'Ah, so it's just checking if it exists...
Re: [PATCH v8 04/37] hook: include hookdir hook in list
Quoted 12 lines · Emily Shaffer
Quoted 12 lines
builtin/hook.c static int list(int argc, const char **argv, const char *prefix) · line 45
Quoted 8 lines
builtin/hook.c +3
-1@@ -40,10 +40,15 @@ static int list(int argc, const char **argv, const char *prefix)list_for_each (pos, head) { struct hook *item = list_entry(pos, struct hook, list); -if (item) + item = list_entry(pos, struct hook, list); +if (item) { +/* Don't translate 'hookdir' - it matches the config */ Let's prefix comments for translators with /* TRANSLATORS: .., see the coding style doc. That's what they'll see, and this is useful to them.
Quoted 7 lines
Quoted 3 lines
Better yet have a note here about the first argument being 'system', 'local' etc., which I had to source spelunge for, and translators won't have any idea about unless the magic parameter is documented.
It's not a comment for translators. It's a comment for someone helpful who comes later and says "oh, none of this is marked for translation, I'd better fix that."
Then, it is not limited to "hookdir", is it? Resurrecting the elided part back here:
Not just we do not want "hookdir" placed inside _(),
printf("%s: %s\n",
+ (item->from_hookdir
+ ? "hookdir"
+ : config_scope_name(item->origin)),
item->command.buf);
we do not want the "%s: %s\n" to be placed inside _() and get munged into "%2$s: %1$s\n" for languages that want the order swapped, for example.
So perhaps the comment should be about the entire output, i.e. "don't translate the output from this helper, as it is meant to be machine parseable", or something?
Re: [PATCH v8 04/37] hook: include hookdir hook in list
Quoted 8 lines · Junio C Hamano
Not just we do not want "hookdir" placed inside
_(),printf("%s: %s\n",builtin/hook.c +3
-0+ (item->from_hookdir + ?"hookdir" + :config_scope_name (item->origin)), item->command.buf);
Quoted 7 lines
we do not want the "%s: %s\n" to be placed inside
_()and get munged into "%2$s: %1$s\n" for languages that want the order swapped, for example.So perhaps the comment should be about the entire output, i.e. "don't translate the output from this helper, as it is meant to be machine parseable", or something?
Having the word "translate" in the comment automatically implies localization, which confuses the issue. It would be clearer to avoid that word altogether. Perhaps something along the lines of:
/* machine-parseable output; do not apply _() localization */
Re: [PATCH v8 04/37] hook: include hookdir hook in list
Quoted 18 lines · Eric Sunshine
Quoted 8 lines · Junio C Hamano
Not just we do not want "hookdir" placed inside
_(),printf("%s: %s\n",builtin/hook.c +3
-0+ (item->from_hookdir + ?"hookdir" + :config_scope_name (item->origin)), item->command.buf);Quoted 7 lines
we do not want the "%s: %s\n" to be placed inside
_()and get munged into "%2$s: %1$s\n" for languages that want the order swapped, for example.So perhaps the comment should be about the entire output, i.e. "don't translate the output from this helper, as it is meant to be machine parseable", or something?
Quoted 5 lines
Having the word "translate" in the comment automatically implies localization, which confuses the issue. It would be clearer to avoid that word altogether. Perhaps something along the lines of:
/* machine-parseable output; do not apply _() localization */
After I read Ævar's comments on the next patch in this series, I decided to rework the comments and translation markers for this whole section.
if (item) {
if (item->from_hookdir) {
/*
* TRANSLATORS: do not translate 'hookdir' as
* it matches the config setting.
*/
switch (should_run_hookdir) {
case HOOKDIR_NO:
printf(_("hookdir: %s (will not run)\n"),
item->command.buf);
break;
case HOOKDIR_ERROR:
printf(_("hookdir: %s (will error and not run)\n"),
item->command.buf);
break;
case HOOKDIR_INTERACTIVE:
printf(_("hookdir: %s (will prompt)\n"),
item->command.buf);
break;
case HOOKDIR_WARN:
printf(_("hookdir: %s (will warn but run)\n"),
item->command.buf);
break;
case HOOKDIR_YES:
/*
* The default behavior should agree with
* hook.c:configured_hookdir_opt(). HOOKDIR_UNKNOWN should just
* do the default behavior.
*/
case HOOKDIR_UNKNOWN:
default:
printf(_("hookdir: %s\n"),
item->command.buf);
break;
}
} else {
/*
* TRANSLATORS: "<config scope>: <path>". Both fields
* should be left untranslated; config scope matches the
* output of 'git config --show-scope'. Marked for
* translation to provide better RTL support later.
*/
printf(_("%s: %s\n"),
config_scope_name(item->origin),
item->command.buf);
}}
PATCH v8 5/37 hook: teach hook.runHookDir
In reply to Emily ShafferFor now, just give a hint about how these hooks will be run in 'git hook list'. Later on, though, we will pay attention to this enum when running the hooks.
---
Notes:
Since v7, tidied up the behavior of the HOOK_UNKNOWN flag and added a test to enforce it - now it matches the design doc much better.
Also, thanks Jonathan Tan for pointing out that the commit message made no sense and was targeted for a different change. Rewrote the commit message now.
Plus, added HOOK_ERROR flag per Junio and Jonathan Nieder.
- Emily
Documentation/config/hook.txt | 5 +++ builtin/hook.c | 69 +++++++++++++++++++++++++++++++--- hook.c | 24 ++++++++++++ hook.h | 16 ++++++++ t/t1360-config-based-hooks.sh | 71 +++++++++++++++++++++++++++++++++++ 5 files changed, 180 insertions(+), 5 deletions(-)
Documentation/config/hook.txt +5 -0
@@ -7,3 +7,8 @@ hookcmd.<name>.command:: A command to execute during a hook for which <name> has been specified as a command. This can be an executable on your device or a oneliner for your shell. See linkgit:git-hook[1]. + +hook.runHookDir:: + Controls how hooks contained in your hookdir are executed. Can be any of + "yes", "warn", "interactive", or "no". Defaults to "yes". See + linkgit:git-hook[1] and linkgit:git-config[1] "core.hooksPath").
builtin/hook.c +64 -5
110 lines not on this page · raw message
hook.c +24 -0
@@ -102,6 +102,30 @@ static int hook_config_lookup(const char *key, const char *value, void *cb_data)return 0 ; } +enum hookdir_opt configured_hookdir_opt (void ) +{ +const char *key; +if (git_config_get_value ("hook.runhookdir" , &key)) +return HOOKDIR_YES ;/* by default, just run it. */ + +if (!strcmp (key,"no" )) +return HOOKDIR_NO ; + +if (!strcmp (key,"error" )) +return HOOKDIR_ERROR ; + +if (!strcmp (key,"yes" )) +return HOOKDIR_YES ; + +if (!strcmp (key,"warn" )) +return HOOKDIR_WARN ; + +if (!strcmp (key,"interactive" )) +return HOOKDIR_INTERACTIVE ; + +return HOOKDIR_UNKNOWN ; +} +struct list_head *hook_list (const struct strbuf * hookname) {struct strbuf hook_key =STRBUF_INIT ;
hook.h +16 -0
@@ -20,6 +20,22 @@ struct hook {*/ struct list_head *hook_list (const struct strbuf *hookname); +enum hookdir_opt +{ +HOOKDIR_NO , +HOOKDIR_ERROR , +HOOKDIR_WARN , +HOOKDIR_INTERACTIVE , +HOOKDIR_YES , +HOOKDIR_UNKNOWN , +}; + +/* +* Provides the hookdir_opt specified in the config without consulting any +* command line arguments. +*/ +enum hookdir_opt configured_hookdir_opt (void ); +/* Free memory associated with a 'struct hook' */ void free_hook (struct hook *ptr);/* Empties the list at 'head', calling 'free_hook()' on each entry */
t/t1360-config-based-hooks.sh +71 -0
75 lines not on this page · raw message
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
Re: [PATCH v8 05/37] hook: teach hook.runHookDir
builtin/hook.c static int list(int argc, const char **argv, const char *prefix) · line 45
Quoted 2 lines · Emily Shaffer
builtin/hook.c +2
-0+switch (should_run_hookdir) { +case HOOKDIR_NO :
Style: case shouldn't be indented
Quoted 10 lines
builtin/hook.c +10
-0+strbuf_addstr (&hookdir_annotation,_ (" (will not run)" )); +break ; +case HOOKDIR_ERROR : +strbuf_addstr (&hookdir_annotation,_ (" (will error and not run)" )); +break ; +case HOOKDIR_INTERACTIVE : +strbuf_addstr (&hookdir_annotation,_ (" (will prompt)" )); +break ; +case HOOKDIR_WARN : +strbuf_addstr (&hookdir_annotation,_ (" (will warn but run)" ));
builtin/hook.c static int list(int argc, const char **argv, const char *prefix) · line 65
Quoted 10 lines
builtin/hook.c +10
-0+break ; +case HOOKDIR_YES : +/* +* The default behavior should agree with +* hook.c:configured_hookdir_opt(). HOOKDIR_UNKNOWN should just +* do the default behavior. +*/ +case HOOKDIR_UNKNOWN : +default : +break ;
We should avoid this sort of translation lego.
builtin/hook.c static int list(int argc, const char **argv, const char *prefix) · line 73
Quoted 9 lines
builtin/hook.c +3
-1+ } +list_for_each (pos, head) { struct hook *item = list_entry(pos, struct hook, list); item = list_entry(pos, struct hook, list);if (item) {/* Don't translate 'hookdir' - it matches the config */ - printf("%s: %s\n", + printf("%s: %s%s\n",
native speakers in some languages to read the sentance backwards. Because if you concatenate strings like this you force.
(We don't currently have a RTL language in po/, still, but let's not create churn for if/when we do if we can help it)>
I have a patch on top to fix this, will send it as some general reply of proposed fixup.s
t/t1360-config-based-hooks.sh test_expect_success 'git hook list shows hooks from the hookdir' ' · line 118
Quoted 4 lines
t/t1360-config-based-hooks.sh +3
-0(item->from_hookdir + git hook list pre-commit >actual && + # the hookdir annotation is translated + test_i18ncmp expected actual
This (and the rest of test_i18ncmp in this series) can and should just be "test_cmp" or "test_i18ncmp", the poison mode is dead. See my recent patches to search/replace test_i18ncmp.
The reason the function isn't gone entirely was to help a series like yours in "seen", but if we're re-rolling...
Re: [PATCH v8 05/37] hook: teach hook.runHookDir
Quoted 8 lines · Ævar Arnfjörð Bjarmason
builtin/hook.c static int list(int argc, const char **argv, const char *prefix) · line 45
Quoted 2 lines · Emily Shaffer
builtin/hook.c +2
-0+switch (should_run_hookdir) { +case HOOKDIR_NO :Style: case shouldn't be indented
Done, thanks.
Quoted 37 lines
Quoted 11 lines
builtin/hook.c +10
-0+strbuf_addstr (&hookdir_annotation,_ (" (will not run)" )); +break ; +case HOOKDIR_ERROR : +strbuf_addstr (&hookdir_annotation,_ (" (will error and not run)" )); +break ; +case HOOKDIR_INTERACTIVE : +strbuf_addstr (&hookdir_annotation,_ (" (will prompt)" )); +break ; +case HOOKDIR_WARN : +strbuf_addstr (&hookdir_annotation,_ (" (will warn but run)" ));builtin/hook.c static int list(int argc, const char **argv, const char *prefix) · line 65
Quoted 10 lines
builtin/hook.c +10
-0+break ; +case HOOKDIR_YES : +/* +* The default behavior should agree with +* hook.c:configured_hookdir_opt(). HOOKDIR_UNKNOWN should just +* do the default behavior. +*/ +case HOOKDIR_UNKNOWN : +default : +break ;We should avoid this sort of translation lego.
builtin/hook.c static int list(int argc, const char **argv, const char *prefix) · line 73
Quoted 9 lines
builtin/hook.c +3
-1+ } +list_for_each (pos, head) { struct hook *item = list_entry(pos, struct hook, list); item = list_entry(pos, struct hook, list);if (item) {/* Don't translate 'hookdir' - it matches the config */ - printf("%s: %s\n", + printf("%s: %s%s\n",native speakers in some languages to read the sentance backwards. Because if you concatenate strings like this you force.
Quoted 2 lines
(We don't currently have a RTL language in po/, still, but let's not create churn for if/when we do if we can help it)>
Yeah, you are absolutely right. I'll take a look at your suggestion, thanks.
Quoted 4 lines
I have a patch on top to fix this, will send it as some general reply of proposed fixup.s
FWIW, I found this format of suggestions really hard to navigate. I had to go find your fixup (and I think you sent two different ones) and then had to scroll around and find what you're referring to from here. If I were to apply the fixup directly, I'd have to split it up and find the appropriate commits to associate each part of the diff with. I know generating scissors patches for each review is a pain, but I'd even prefer a prose "How about printing in each part of the case instead, so each string can be translated in full" or a non-formatted example inline to the catchall fixups patch you sent.
Quoted 10 lines
t/t1360-config-based-hooks.sh test_expect_success 'git hook list shows hooks from the hookdir' ' · line 118
Quoted 5 lines
t/t1360-config-based-hooks.sh +3
-0(item->from_hookdir + git hook list pre-commit >actual && + # the hookdir annotation is translated + test_i18ncmp expected actualThis (and the rest of
test_i18ncmpin this series) can and should just be "test_cmp" or "test_i18ncmp", the poison mode is dead. See my recent patches to search/replacetest_i18ncmp.
Quoted 2 lines
The reason the function isn't gone entirely was to help a series like yours in "seen", but if we're re-rolling...
Oh cool, thanks, will do.
Re: [PATCH v8 05/37] hook: teach hook.runHookDir
Quoted 60 lines · Emily Shaffer
Quoted 8 lines · �var Arnfj�r� Bjarmason
builtin/hook.c static int list(int argc, const char **argv, const char *prefix) · line 45
Quoted 2 lines · Emily Shaffer
builtin/hook.c +2
-0+switch (should_run_hookdir) { +case HOOKDIR_NO :Style: case shouldn't be indented
Done, thanks.
Quoted 37 lines
Quoted 11 lines
builtin/hook.c +10
-0+strbuf_addstr (&hookdir_annotation,_ (" (will not run)" )); +break ; +case HOOKDIR_ERROR : +strbuf_addstr (&hookdir_annotation,_ (" (will error and not run)" )); +break ; +case HOOKDIR_INTERACTIVE : +strbuf_addstr (&hookdir_annotation,_ (" (will prompt)" )); +break ; +case HOOKDIR_WARN : +strbuf_addstr (&hookdir_annotation,_ (" (will warn but run)" ));builtin/hook.c static int list(int argc, const char **argv, const char *prefix) · line 65
Quoted 10 lines
builtin/hook.c +10
-0+break ; +case HOOKDIR_YES : +/* +* The default behavior should agree with +* hook.c:configured_hookdir_opt(). HOOKDIR_UNKNOWN should just +* do the default behavior. +*/ +case HOOKDIR_UNKNOWN : +default : +break ;We should avoid this sort of translation lego.
builtin/hook.c static int list(int argc, const char **argv, const char *prefix) · line 73
Quoted 9 lines
builtin/hook.c +3
-1+ } +list_for_each (pos, head) { struct hook *item = list_entry(pos, struct hook, list); item = list_entry(pos, struct hook, list);if (item) {/* Don't translate 'hookdir' - it matches the config */ - printf("%s: %s\n", + printf("%s: %s%s\n",native speakers in some languages to read the sentance backwards. Because if you concatenate strings like this you force.
Quoted 2 lines
(We don't currently have a RTL language in po/, still, but let's not create churn for if/when we do if we can help it)>
Yeah, you are absolutely right. I'll take a look at your suggestion, thanks.
Quoted 4 lines
I have a patch on top to fix this, will send it as some general reply of proposed fixup.s
Quoted 9 lines
FWIW, I found this format of suggestions really hard to navigate. I had to go find your fixup (and I think you sent two different ones) and then had to scroll around and find what you're referring to from here. If I were to apply the fixup directly, I'd have to split it up and find the appropriate commits to associate each part of the diff with. I know generating scissors patches for each review is a pain, but I'd even prefer a prose "How about printing in each part of the case instead, so each string can be translated in full" or a non-formatted example inline to the catchall fixups patch you sent.
Sorry about that. FWIW I did try, but found myself jumping a bit too much between different commits (as noted in some "maybe squash?" comments elsewhere), and eventually just gave up and produced some RFC-just-for-discussion patch on top of the whole thing and mostly looked at the entire diff forthe series.
Quoted 14 lines
Quoted 10 lines
t/t1360-config-based-hooks.sh test_expect_success 'git hook list shows hooks from the hookdir' ' · line 118
Quoted 5 lines
t/t1360-config-based-hooks.sh +3
-0(item->from_hookdir + git hook list pre-commit >actual && + # the hookdir annotation is translated + test_i18ncmp expected actualThis (and the rest of
test_i18ncmpin this series) can and should just be "test_cmp" or "test_i18ncmp", the poison mode is dead. See my recent patches to search/replacetest_i18ncmp.Quoted 2 lines
The reason the function isn't gone entirely was to help a series like yours in "seen", but if we're re-rolling...
Oh cool, thanks, will do.
PATCH v8 6/37 hook: implement hookcmd.<name>.skip
In reply to Emily ShafferIf a user wants a specific repo to skip execution of a hook which is set at a global or system level, they will be able to do so by specifying 'skip' in their repo config:
~/.gitconfig
[hook.pre-commit]
command = skippable-oneliner
command = skippable-hookcmd
[hookcmd.skippable-hookcmd]
command = foo.sh
$GIT_DIR/.git/config
[hookcmd.skippable-oneliner]
skip = true
[hookcmd.skippable-hookcmd]
skip = true
Later it may make sense to add an option like "hookcmd.<name>.<hook-event>-skip" - but for simplicity, let's start with a universal skip setting like this.
---
Documentation/config/hook.txt | 8 ++++++++ Documentation/git-hook.txt | 33 +++++++++++++++++++++++++++++++++ hook.c | 35 ++++++++++++++++++++++++++--------- t/t1360-config-based-hooks.sh | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 9 deletions(-)
Documentation/config/hook.txt +8 -0
@@ -8,6 +8,14 @@ hookcmd.<name>.command:: as a command. This can be an executable on your device or a oneliner for your shell. See linkgit:git-hook[1]. +hookcmd.<name>.skip:: + Specify this boolean to remove a command from earlier in the execution + order. Useful if you want to make a single repo an exception to hook + configured at the system or global scope. If there is no hookcmd + specified for the command you want to skip, you can use the value of + `hook.<command>.command` as <name> as a shortcut. The "skip" setting + must be specified after the "hook.<command>.command" to have an effect. + hook.runHookDir:: Controls how hooks contained in your hookdir are executed. Can be any of "yes", "warn", "interactive", or "no". Defaults to "yes". See
Documentation/git-hook.txt +33 -0
@@ -54,6 +54,39 @@ $ git hook list "prepare-commit-msg" local: /bin/linter --c ---- +If there is a command you wish to run in most cases but have one or two +exceptional repos where it should be skipped, you can use specify +`hookcmd.<name>.skip`, for example: + +System config +---- + [hook "pre-commit"] + command = check-for-secrets + + [hookcmd "check-for-secrets"] + command = /bin/secret-checker --aggressive +---- + +Local config +---- + [hookcmd "check-for-secrets"] + skip = true + # This works for inlined hook commands, too: + [hookcmd "~/typocheck.sh"] + skip = true +---- + +After these configs are added, the hook list becomes: + +---- +$ git hook list "post-commit" +global: /bin/linter --c +local: python ~/run-test-suite.py + +$ git hook list "pre-commit" +no commands configured for hook 'pre-commit' +---- + COMMANDS --------
hook.c +26 -9
@@ -12,24 +12,25 @@ void free_hook(struct hook *ptr) } } -static void append_or_move_hook (struct list_head *head,const char *command) +static struct hook *find_hook_by_command (struct list_head *head,const char *command) {struct list_head *pos =NULL , *tmp =NULL ; -struct hook *to_add =NULL ; +struct hook *found =NULL ; -/* -* remove the prior entry with this command; we'll replace it at the -* end. -*/ list_for_each_safe (pos, tmp, head) { struct hook *it = list_entry(pos, struct hook, list);if (!strcmp (it->command.buf, command)) {list_del (pos); -/* we'll simply move the hook to the end */ - to_add = it; + found = it;break ; } } +return found; +} + +static void append_or_move_hook (struct list_head *head,const char *command) +{ +struct hook *to_add =find_hook_by_command (head, command);if (!to_add) {/* adding a new hook, not moving an old one */
@@ -74,12 +75,22 @@ static int hook_config_lookup(const char *key, const char *value, void *cb_data)if (!strcmp (key, hook_key)) {const char *command = value;struct strbuf hookcmd_name =STRBUF_INIT ; +int skip =0 ; + +/* +* Check if we're removing that hook instead. Hookcmds are +* removed by name, and inlined hooks are removed by command +* content. +*/ +strbuf_addf (&hookcmd_name,"hookcmd.%s.skip" , command); +git_config_get_bool (hookcmd_name.buf, &skip);/* * Check if a hookcmd with that name exists. If it doesn't, * 'git_config_get_value()' is documented not to touch &command, * so we don't need to do anything. */ +strbuf_reset (&hookcmd_name);strbuf_addf (&hookcmd_name,"hookcmd.%s.command" , command);git_config_get_value (hookcmd_name.buf, &command);
@@ -94,7 +105,13 @@ static int hook_config_lookup(const char *key, const char *value, void *cb_data)* for each key+value, do_callback(key, value, cb_data) */ -append_or_move_hook (head, command); +if (skip) { +struct hook *to_remove =find_hook_by_command (head, command); +if (to_remove) +remove_hook (&(to_remove->list)); + }else { +append_or_move_hook (head, command); + }strbuf_release (&hookcmd_name); }
t/t1360-config-based-hooks.sh +35 -0
@@ -146,6 +146,41 @@ test_expect_success 'hook.runHookDir = warn is respected by list' ' test_i18ncmp expected actual ' +test_expect_success 'git hook list removes skipped hookcmd' ' + setup_hookcmd && + test_config hookcmd.abc.skip "true" --add && + + cat >expected <<-EOF && + no commands configured for hook '\''pre-commit'\'' + EOF + + git hook list pre-commit >actual && + test_i18ncmp expected actual +' + +test_expect_success 'git hook list ignores skip referring to unused hookcmd' ' + test_config hookcmd.abc.command "/path/abc" --add && + test_config hookcmd.abc.skip "true" --add && + + cat >expected <<-EOF && + no commands configured for hook '\''pre-commit'\'' + EOF + + git hook list pre-commit >actual && + test_i18ncmp expected actual +' + +test_expect_success 'git hook list removes skipped inlined hook' ' + setup_hooks && + test_config hookcmd."$ROOT/path/ghi".skip "true" --add && + + cat >expected <<-EOF && + global: $ROOT/path/def + EOF + + git hook list pre-commit >actual && + test_cmp expected actual +' test_expect_success 'hook.runHookDir = interactive is respected by list' ' setup_hookdir &&
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
Re: [PATCH v8 06/37] hook: implement hookcmd.<name>.skip
Quoted 4 lines · Emily Shaffer
t/t1360-config-based-hooks.sh +4
-0+ cat >expected <<-EOF && + no commands configured for hook '\''pre-commit'\'' + EOF +
t/t1360-config-based-hooks.sh test_expect_success 'hook.runHookDir = warn is respected by list' ' · line 166
Quoted 10 lines
t/t1360-config-based-hooks.sh +10
-0+ git hook list pre-commit >actual && + test_i18ncmp expected actual +' + +test_expect_success 'git hook list ignores skip referring to unused hookcmd' ' + test_config hookcmd.abc.command "/path/abc" --add && + test_config hookcmd.abc.skip "true" --add && + + cat >expected <<-EOF && + no commands configured for hook '\''pre-commit'\''
ditto on the "echo" comment in a previous mail, looks like we can avoid both of these entirely.
PATCH v8 7/37 parse-options: parse into strvec
In reply to Emily Shafferparse-options already knows how to read into a string_list, and it knows how to read into an strvec as a passthrough (that is, including the argument as well as its value). string_list and strvec serve similar purposes but are somewhat painful to convert between; so, let's teach parse-options to read values of string arguments directly into an strvec without preserving the argument name.
This is useful if collecting generic arguments to pass through to another command, for example, 'git hook run --arg "--quiet" --arg "--format=pretty" some-hook'. The resulting strvec would contain { "--quiet", "--format=pretty" }.
The implementation is based on that of OPT_STRING_LIST.
---
Notes:
Since v7, updated the reference doc to make the intended usage for OPT_STRVEC more clear.
Since v4, fixed one or two more places where I missed the argv_array->strvec rename.
Documentation/technical/api-parse-options.txt | 7 +++++++ parse-options-cb.c | 16 ++++++++++++++++ parse-options.h | 4 ++++ 3 files changed, 27 insertions(+)
Documentation/technical/api-parse-options.txt +7 -0
@@ -173,6 +173,13 @@ There are some macros to easily define options: The string argument is stored as an element in `string_list`. Use of `--no-option` will clear the list of preceding values. +`OPT_STRVEC(short, long, &struct strvec, arg_str, description)`:: + Introduce an option with a string argument, meant to be specified + multiple times. + The string argument is stored as an element in `strvec`, and later + arguments are added to the same `strvec`. + Use of `--no-option` will clear the list of preceding values. + `OPT_INTEGER(short, long, &int_var, description)`:: Introduce an option with integer argument. The integer is put into `int_var`.
parse-options-cb.c +16 -0
@@ -207,6 +207,22 @@ int parse_opt_string_list(const struct option *opt, const char *arg, int unset)return 0 ; } +int parse_opt_strvec (const struct option *opt,const char *arg,int unset) +{ +struct strvec *v = opt->value; + +if (unset) { +strvec_clear (v); +return 0 ; + } + +if (!arg) +return -1 ; + +strvec_push (v, arg); +return 0 ; +} +int parse_opt_noop_cb (const struct option *opt,const char *arg,int unset) {return 0 ;
parse-options.h +4 -0
@@ -177,6 +177,9 @@ struct option {#define OPT_STRING_LIST (s, l, v, a, h) \ { OPTION_CALLBACK, (s), (l), (v), (a), \ (h), 0, &parse_opt_string_list } +#define OPT_STRVEC (s, l, v, a, h) \ + { OPTION_CALLBACK, (s), (l), (v), (a), \ + (h), 0, &parse_opt_strvec }#define OPT_UYN (s, l, v, h) { OPTION_CALLBACK, (s), (l), (v), NULL, \ (h), PARSE_OPT_NOARG, &parse_opt_tertiary }#define OPT_EXPIRY_DATE (s, l, v, h) \
@@ -296,6 +299,7 @@ int parse_opt_commits(const struct option *, const char *, int);int parse_opt_commit (const struct option *,const char *,int );int parse_opt_tertiary (const struct option *,const char *,int );int parse_opt_string_list (const struct option *,const char *,int ); +int parse_opt_strvec (const struct option *,const char *,int );int parse_opt_noop_cb (const struct option *,const char *,int ); enum parse_opt_result parse_opt_unknown_cb(struct parse_opt_ctx_t *ctx, const struct option *,
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
Re: [PATCH v8 07/37] parse-options: parse into strvec
Quoted 60 lines · Emily Shaffer
Documentation/technical/api-parse-options.txt +7
-0@@ -173,6 +173,13 @@ There are some macros to easily define options: The string argument is stored as an element in `string_list`. Use of `--no-option` will clear the list of preceding values. +`OPT_STRVEC(short, long, &struct strvec, arg_str, description)`:: + Introduce an option with a string argument, meant to be specified + multiple times. + The string argument is stored as an element in `strvec`, and later + arguments are added to the same `strvec`. + Use of `--no-option` will clear the list of preceding values. + `OPT_INTEGER(short, long, &int_var, description)`:: Introduce an option with integer argument. The integer is put into `int_var`.parse-options-cb.c +16
-0@@ -207,6 +207,22 @@ int parse_opt_string_list(const struct option *opt, const char *arg, int unset)return 0 ; } +int parse_opt_strvec (const struct option *opt,const char *arg,int unset) +{ +struct strvec *v = opt->value; + +if (unset) { +strvec_clear (v); +return 0 ; + } + +if (!arg) +return -1 ; + +strvec_push (v, arg); +return 0 ; +} +int parse_opt_noop_cb (const struct option *opt,const char *arg,int unset) {return 0 ;parse-options.h +3
-0@@ -177,6 +177,9 @@ struct option {#define OPT_STRING_LIST (s, l, v, a, h) \ { OPTION_CALLBACK, (s), (l), (v), (a), \ (h), 0, &parse_opt_string_list } +#define OPT_STRVEC (s, l, v, a, h) \ + { OPTION_CALLBACK, (s), (l), (v), (a), \ + (h), 0, &parse_opt_strvec }#define OPT_UYN (s, l, v, h) { OPTION_CALLBACK, (s), (l), (v), NULL, \ (h), PARSE_OPT_NOARG, &parse_opt_tertiary }#define OPT_EXPIRY_DATE (s, l, v, h) \
parse-options.h int parse_opt_commits(const struct option *, const char *, int); · line 305
Quoted 7 lines
parse-options.h +1
-0@@ -296,6 +299,7 @@ int parse_opt_commits(const struct option *, const char *, int);int parse_opt_commit (const struct option *,const char *,int );int parse_opt_tertiary (const struct option *,const char *,int );int parse_opt_string_list (const struct option *,const char *,int ); +int parse_opt_strvec (const struct option *,const char *,int );int parse_opt_noop_cb (const struct option *,const char *,int ); enum parse_opt_result parse_opt_unknown_cb(struct parse_opt_ctx_t *ctx, const struct option *,
Nice, seems very useful.
But let's add a test in test-parse-options.c like we have for string_list?
Re: [PATCH v8 07/37] parse-options: parse into strvec
On Fri, Mar 12, 2021 at 09:50:58AM +0100, Ævar Arnfjörð Bjarmason wrote:
[snip]
Quoted 4 lines
Nice, seems very useful.
But let's add a test in
test-parse-options.clike we have forstring_list?
Sure, done. Thanks.
PATCH v8 8/37 hook: add 'run' subcommand
In reply to Emily ShafferIn order to enable hooks to be run as an external process, by a standalone Git command, or by tools which wrap Git, provide an external means to run all configured hook commands for a given hook event.
For now, the hook commands will run in config order, in series. As alternate ordering or parallelism is supported in the future, we should add knobs to use those to the command line as well.
As with the legacy hook implementation, all stdout generated by hook commands is redirected to stderr. Piping from stdin is not yet supported.
Legacy hooks (those present in $GITDIR/hooks) are run at the end of the execution list. They can be disabled, or made to print warnings, or to prompt before running, with the 'hook.runHookDir' config.
Users may wish to provide hook commands like 'git config hook.pre-commit.command "~/linter.sh --pre-commit"'. To enable this, config-defined hooks are run in a shell. (Since hooks in $GITDIR/hooks can't be specified with included arguments or paths which need expansion like this, they are run without a shell instead.)
---
Notes:
Since v7, added support for "error" hook.runHookDir setting.
Since v4, updated the docs, and did less local application of single quotes. In order for hookdir hooks to run successfully with a space in the path, though, they must not be run with 'sh -c'. So we can treat the hookdir hooks specially, and warn users via doc about special considerations for configured hooks with spaces in their path.
Documentation/git-hook.txt | 31 +++++++- builtin/hook.c | 42 ++++++++++- hook.c | 128 ++++++++++++++++++++++++++++++++++ hook.h | 26 +++++++ t/t1360-config-based-hooks.sh | 72 ++++++++++++++++++- 5 files changed, 292 insertions(+), 7 deletions(-)
Documentation/git-hook.txt +29 -2
@@ -9,11 +9,12 @@ SYNOPSIS -------- [verse] 'git hook' list <hook-name> +'git hook' run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] <hook-name> DESCRIPTION ----------- -You can list configured hooks with this command. Later, you will be able to run, -add, and modify hooks with this command. +You can list and run configured hooks with this command. Later, you will be able +to add and modify hooks with this command. This command parses the default configuration files for sections `hook` and `hookcmd`. `hook` is used to describe the commands which will be run during a
@@ -97,6 +98,32 @@ in the order they should be run, and print the config scope where the relevant `hook.<hook-name>.command` was specified, not the `hookcmd` (if applicable). This output is human-readable and the format is subject to change over time. +run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] `<hook-name>`:: + +Runs hooks configured for `<hook-name>`, in the same order displayed by `git +hook list`. Hooks configured this way may be run prepended with `sh -c`, so +paths containing special characters or spaces should be wrapped in single +quotes: `command = '/my/path with spaces/script.sh' some args`. + +OPTIONS +------- +--run-hookdir:: + Overrides the hook.runHookDir config. Must be 'yes', 'warn', + 'interactive', or 'no'. Specifies how to handle hooks located in the Git + hook directory (core.hooksPath). + +-a:: +--arg:: + Only valid for `run`. ++ +Specify arguments to pass to every hook that is run. + +-e:: +--env:: + Only valid for `run`. ++ +Specify environment variables to set for every hook that is run. + CONFIGURATION ------------- include::config/hook.txt[]
builtin/hook.c +40 -2
71 lines not on this page · raw message
hook.c +128 -0
143 lines not on this page · raw message
hook.h +26 -0
@@ -1,6 +1,7 @@#include "config.h" #include "list.h" #include "strbuf.h" +#include "strvec.h" struct hook {struct list_head list;
@@ -36,6 +37,31 @@ enum hookdir_opt*/ enum hookdir_opt configured_hookdir_opt (void ); +struct run_hooks_opt +{ +/* Environment vars to be set for each hook */ +struct strvec env; + +/* Args to be passed to each hook */ +struct strvec args; + +/* +* How should the hookdir be handled? +* Leave the RUN_HOOKS_OPT_INIT default in most cases; this only needs +* to be overridden if the user can override it at the command line. +*/ +enum hookdir_opt run_hookdir; +}; + +void run_hooks_opt_init (struct run_hooks_opt *o); +void run_hooks_opt_clear (struct run_hooks_opt *o); + +/* +* Runs all hooks associated to the 'hookname' event in order. Each hook will be +* passed 'env' and 'args'. +*/ +int run_hooks (const char *hookname,struct run_hooks_opt *options); +/* Free memory associated with a 'struct hook' */ void free_hook (struct hook *ptr);/* Empties the list at 'head', calling 'free_hook()' on each entry */
t/t1360-config-based-hooks.sh +69 -3
102 lines not on this page · raw message
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
Re: [PATCH v8 08/37] hook: add 'run' subcommand
Quoted 2 lines · Emily Shaffer
'git hook' list <hook-name>
+'git hook' run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] <hook-name>
[...]
hook.c enum hookdir_opt configured_hookdir_opt(void) · line 159
Quoted 3 lines
hook.c +3
-0+switch (cfg) + { +case HOOKDIR_ERROR :
Overly indented case statements again.
Quoted 7 lines
hook.c +7
-0+fprintf (stderr,_ ("Skipping legacy hook at '%s'\n" ), + path); +/* FALLTHROUGH */ +case HOOKDIR_NO : +return 0 ; +case HOOKDIR_WARN : + fprintf(stderr, _("Running legacy hook at '%s'\n"),
hook.c enum hookdir_opt configured_hookdir_opt(void) · line 176
Quoted 10 lines
hook.c +10
-0+ path); +return 1 ; +case HOOKDIR_INTERACTIVE : + do { + /* + * TRANSLATORS: Make sure to include [Y] and [n] + * in your translation. Only English input is + * accepted. Default option is "yes". + */ + fprintf(stderr, _("Run '%s'? [Yn] "), path);
Nit: [Y/n]
hook.c enum hookdir_opt configured_hookdir_opt(void) · line 182
Quoted 1 line
hook.c +1
-0+ }else if (starts_with (prompt.buf,"y" )) {
So also "Y", "yes" and "yellow"...
Quoted 46 lines
[...]
t/t1360-config-based-hooks.sh +42
-1git hook list pre-commit >actual && # the hookdir annotation is translated - test_i18ncmp expected actual + test_i18ncmp expected actual && + + test_write_lines n | git hook run pre-commit 2>actual && + ! grep "Legacy Hook" actual && + + test_write_lines y | git hook run pre-commit 2>actual && + grep "Legacy Hook" actual +' + +test_expect_success 'inline hook definitions execute oneliners' ' + test_config hook.pre-commit.command "echo \"Hello World\"" && + + echo "Hello World" >expected && + + # hooks are run with stdout_to_stderr = 1 + git hook run pre-commit 2>actual && + test_cmp expected actual +' + +test_expect_success 'inline hook definitions resolve paths' ' + write_script sample-hook.sh <<-EOF && + echo \"Sample Hook\" + EOF + + test_when_finished "rm sample-hook.sh" && + + test_config hook.pre-commit.command "\"$(pwd)/sample-hook.sh\"" && + + echo \"Sample Hook\" >expected && + + # hooks are run with stdout_to_stderr = 1 + git hook run pre-commit 2>actual && + test_cmp expected actual +' + +test_expect_success 'hookdir hook included in git hook run' ' + setup_hookdir && + + echo \"Legacy Hook\" >expected && + + # hooks are run with stdout_to_stderr = 1 + git hook run pre-commit 2>actual &&
t/t1360-config-based-hooks.sh test_expect_success 'hook.runHookDir = interactive is respected by list' ' · line 265
Quoted 10 lines
t/t1360-config-based-hooks.sh +7
-0+ test_cmp expected actual +' + +test_expect_success 'out-of-repo runs excluded' ' + setup_hooks && + + nongit test_must_fail git hook run pre-commit ' test_expect_success 'hook.runHookDir is tolerant to unknown values' '
No tests for --env or --arg?
Re: [PATCH v8 08/37] hook: add 'run' subcommand
Quoted 7 lines · Ævar Arnfjörð Bjarmason
Quoted 2 lines · Emily Shaffer
'git hook' list <hook-name>
+'git hook' run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] <hook-name>
Quoted 7 lines
[...]
hook.c enum hookdir_opt configured_hookdir_opt(void) · line 159
Quoted 3 lines
hook.c +3
-0+switch (cfg) + { +case HOOKDIR_ERROR :Overly indented case statements again.
Thanks. Probably need to fix my autoformatter or something.
Quoted 12 lines
Quoted 12 lines
hook.c +11
-0+fprintf (stderr,_ ("Skipping legacy hook at '%s'\n" ), + path); +/* FALLTHROUGH */ +case HOOKDIR_NO : +return 0 ; +case HOOKDIR_WARN : +fprintf (stderr,_ ("Running legacy hook at '%s'\n" ), + path); +return 1 ; +case HOOKDIR_INTERACTIVE : + do {
Quoted 8 lines
hook.c enum hookdir_opt configured_hookdir_opt(void) · line 176
Quoted 6 lines
hook.c +6
-0+/* +* TRANSLATORS: Make sure to include [Y] and [n] +* in your translation. Only English input is +* accepted. Default option is "yes". +*/ +fprintf (stderr,_ ("Run '%s'? [Yn] " ), path);Nit: [Y/n]
ACK
Quoted 4 lines
hook.c enum hookdir_opt configured_hookdir_opt(void) · line 182
Quoted 2 lines
hook.c +1
-0+ }else if (starts_with (prompt.buf,"y" )) {So also "Y", "yes" and "yellow"...
That's also how add-patch.c:prompt_yesno(), builtin/bisect--helper.c:decide_next(), and git-add--interactive.perl:prompt_yesno (assuming I'm grokking the perl correctly) work. builtin/clean.c:ask_each_cmd checks that the user's reply matches a substring anchored to the beginning (e.g. "y", "ye", "yes"). git-diftool--helper.sh:launch_merge_tool just checks for "not 'n'". And git-send-email.perl:ask just checks for "'y' or not".
So I think there's a little flexibility :) But I like builtin/clean.c's approach most out of these, so I'll switch.
Quoted 56 lines
Quoted 56 lines
[...]
t/t1360-config-based-hooks.sh +49
-1git hook list pre-commit >actual && # the hookdir annotation is translated - test_i18ncmp expected actual + test_i18ncmp expected actual && + + test_write_lines n | git hook run pre-commit 2>actual && + ! grep "Legacy Hook" actual && + + test_write_lines y | git hook run pre-commit 2>actual && + grep "Legacy Hook" actual +' + +test_expect_success 'inline hook definitions execute oneliners' ' + test_config hook.pre-commit.command "echo \"Hello World\"" && + + echo "Hello World" >expected && + + # hooks are run with stdout_to_stderr = 1 + git hook run pre-commit 2>actual && + test_cmp expected actual +' + +test_expect_success 'inline hook definitions resolve paths' ' + write_script sample-hook.sh <<-EOF && + echo \"Sample Hook\" + EOF + + test_when_finished "rm sample-hook.sh" && + + test_config hook.pre-commit.command "\"$(pwd)/sample-hook.sh\"" && + + echo \"Sample Hook\" >expected && + + # hooks are run with stdout_to_stderr = 1 + git hook run pre-commit 2>actual && + test_cmp expected actual +' + +test_expect_success 'hookdir hook included in git hook run' ' + setup_hookdir && + + echo \"Legacy Hook\" >expected && + + # hooks are run with stdout_to_stderr = 1 + git hook run pre-commit 2>actual && + test_cmp expected actual +' + +test_expect_success 'out-of-repo runs excluded' ' + setup_hooks && + + nongit test_must_fail git hook run pre-commit '
Quoted 3 lines
Quoted 1 line
test_expect_success'hook.runHookDir is tolerant to unknown values' 'No tests for --env or --arg?
Yikes, I guess not. Thanks, will add.
- Emily
Re: [PATCH v8 08/37] hook: add 'run' subcommand
In reply to Emily ShafferQuoted 23 lines · Emily Shaffer
builtin/hook.c +23
-0+static int run (int argc,const char **argv,const char *prefix) +{ +struct strbuf hookname =STRBUF_INIT ; +struct run_hooks_opt opt; +int rc =0 ; + +struct option run_options[] = { +OPT_STRVEC ('e' ,"env" , &opt.env,N_ ("var" ), +N_ ("environment variables for hook to use" )), +OPT_STRVEC ('a' ,"arg" , &opt.args,N_ ("args" ), +N_ ("argument to pass to hook" )), +OPT_END (), + }; + +run_hooks_opt_init (&opt); + + argc =parse_options (argc, argv, prefix, run_options, + builtin_hook_usage,0 ); + +if (argc <1 ) +usage_msg_opt (_ ("You must specify a hook event to run." ), + builtin_hook_usage, run_options); +
builtin/hook.c static int list(int argc, const char **argv, const char *prefix) · line 125
Quoted 10 lines
builtin/hook.c +10
-0+strbuf_addstr (&hookname, argv[0 ]); + opt.run_hookdir = should_run_hookdir; + + rc =run_hooks (hookname.buf, &opt); + +strbuf_release (&hookname); +run_hooks_opt_clear (&opt); + +return rc; +}
This looks like a small and clean example that is good for people to emulate when using the new run-hooks API. You opt_init(), futz with the its fields, call run_hooks(), and finally opt_clear() to release the resources.
PATCH v8 9/37 hook: introduce hook_exists()
In reply to Emily ShafferAdd a helper to easily determine whether any hooks exist for a given hook event.
Many callers want to check whether some state could be modified by a hook; that check should include the config-based hooks as well. Optimize by checking the config directly. Since commands which execute hooks might want to take args to replace 'hook.runHookDir', let 'hook_exists()' take a hookdir_opt to override that config.
In some cases, external callers today use find_hook() to discover the location of a hook and then run it manually with run-command.h (that is, not with run_hook_le()). Later, those cases will call hook.h:run_hook() directly instead.
Once the entire codebase is using hook_exists() instead of find_hook(), find_hook() can be safely rolled into hook_exists() and removed from run-command.h.
--- hook.c | 19 +++++++++++++++++++ hook.h | 10 ++++++++++ 2 files changed, 29 insertions(+)
hook.c +19 -0
@@ -243,6 +243,25 @@ void run_hooks_opt_init(struct run_hooks_opt *o) o->run_hookdir =configured_hookdir_opt (); } +int hook_exists (const char *hookname,enum hookdir_opt should_run_hookdir) +{ +const char *value =NULL ;/* throwaway */ +struct strbuf hook_key =STRBUF_INIT ; +int could_run_hookdir; + +if (should_run_hookdir ==HOOKDIR_USE_CONFIG ) + should_run_hookdir =configured_hookdir_opt (); + + could_run_hookdir = (should_run_hookdir ==HOOKDIR_INTERACTIVE || + should_run_hookdir ==HOOKDIR_WARN || + should_run_hookdir ==HOOKDIR_YES ) + && !!find_hook (hookname); + +strbuf_addf (&hook_key,"hook.%s.command" , hookname); + +return (!git_config_get_value (hook_key.buf, &value)) || could_run_hookdir; +} +void run_hooks_opt_clear (struct run_hooks_opt *o) {strvec_clear (&o->env);
hook.h +10 -0
@@ -23,6 +23,7 @@ struct list_head* hook_list(const struct strbuf *hookname);enum hookdir_opt { +HOOKDIR_USE_CONFIG ,HOOKDIR_NO ,HOOKDIR_ERROR ,HOOKDIR_WARN ,
@@ -56,6 +57,15 @@ struct run_hooks_optvoid run_hooks_opt_init (struct run_hooks_opt *o);void run_hooks_opt_clear (struct run_hooks_opt *o); +/* +* Returns 1 if any hooks are specified in the config or if a hook exists in the +* hookdir. Typically, invoke hook_exsts() like: +* hook_exists(hookname, configured_hookdir_opt()); +* Like with run_hooks, if you take a --run-hookdir flag, reflect that +* user-specified behavior here instead. +*/ +int hook_exists (const char *hookname,enum hookdir_opt should_run_hookdir); +/* * Runs all hooks associated to the 'hookname' event in order. Each hook will be * passed 'env' and 'args'.
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
PATCH v8 10/37 hook: support passing stdin to hooks
In reply to Emily ShafferSome hooks (such as post-rewrite) need to take input via stdin. Previously, callers provided stdin to hooks by setting run-command.h:child_process.in, which takes a FD. Callers would open the file in question themselves before calling run-command(). However, since we will now need to seek to the front of the file and read it again for every hook which runs, hook.h:run_command() takes a path and handles FD management itself. Since this file is opened for read only, it should not prevent later parallel execution support.
On the frontend, this is supported by asking for a file path, rather than by reading stdin. Reading directly from stdin would involve caching the entire stdin (to memory or to disk) and reading it back from the beginning to each hook. We'd want to support cases like insufficient memory or storage for the file. While this may prove useful later, for now the path of least resistance is to just ask the user to make this interim file themselves.
--- Documentation/git-hook.txt | 11 +++++++++-- builtin/hook.c | 5 ++++- hook.c | 8 +++++++- hook.h | 6 +++++- t/t1360-config-based-hooks.sh | 24 ++++++++++++++++++++++++ 5 files changed, 49 insertions(+), 5 deletions(-)
Documentation/git-hook.txt +9 -2
@@ -9,7 +9,8 @@ SYNOPSIS -------- [verse] 'git hook' list <hook-name> -'git hook' run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] <hook-name> +'git hook' run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] [--to-stdin=<path>] + <hook-name> DESCRIPTION -----------
@@ -98,7 +99,7 @@ in the order they should be run, and print the config scope where the relevant `hook.<hook-name>.command` was specified, not the `hookcmd` (if applicable). This output is human-readable and the format is subject to change over time. -run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] `<hook-name>`:: +run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] [--to-stdin=<path>] `<hook-name>`:: Runs hooks configured for `<hook-name>`, in the same order displayed by `git hook list`. Hooks configured this way may be run prepended with `sh -c`, so
@@ -124,6 +125,12 @@ Specify arguments to pass to every hook that is run. + Specify environment variables to set for every hook that is run. +--to-stdin:: + Only valid for `run`. ++ +Specify a file which will be streamed into stdin for every hook that is run. +Each hook will receive the entire file from beginning to EOF. + CONFIGURATION ------------- include::config/hook.txt[]
builtin/hook.c +4 -1
@@ -8,7 +8,8 @@static const char *const builtin_hook_usage[] = {N_ ("git hook list <hookname>" ), -N_ ("git hook run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] <hookname>" ), +N_ ("git hook run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...]" +"[--to-stdin=<path>] <hookname>" ),NULL };
@@ -101,6 +102,8 @@ static int run(int argc, const char **argv, const char *prefix) N_("environment variables for hook to use")),OPT_STRVEC ('a' ,"arg" , &opt.args,N_ ("args" ),N_ ("argument to pass to hook" )), +OPT_STRING (0 ,"to-stdin" , &opt.path_to_stdin,N_ ("path" ), +N_ ("file to read into hooks' stdin" )), OPT_END(), };
hook.c +7 -1
@@ -240,6 +240,7 @@ void run_hooks_opt_init(struct run_hooks_opt *o) {strvec_init (&o->env);strvec_init (&o->args); + o->path_to_stdin =NULL ; o->run_hookdir =configured_hookdir_opt (); }
@@ -274,7 +275,12 @@ static void prepare_hook_cp(struct hook *hook, struct run_hooks_opt *options,if (!hook)return ; - cp->no_stdin =1 ; +/* reopen the file for stdin; run_command closes it. */ +if (options->path_to_stdin) + cp->in =xopen (options->path_to_stdin,O_RDONLY ); +else + cp->no_stdin =1 ; + cp->env = options->env.v; cp->stdout_to_stderr =1 ; cp->trace2_hook_name = hook->command.buf;
hook.h +5 -1
@@ -52,6 +52,9 @@ struct run_hooks_opt* to be overridden if the user can override it at the command line. */ enum hookdir_opt run_hookdir; + +/* Path to file which should be piped to stdin for each hook */ +const char *path_to_stdin; };void run_hooks_opt_init (struct run_hooks_opt *o);
@@ -68,7 +71,8 @@ int hook_exists(const char *hookname, enum hookdir_opt should_run_hookdir);/* * Runs all hooks associated to the 'hookname' event in order. Each hook will be -* passed 'env' and 'args'. +* passed 'env' and 'args'. The file at 'stdin_path' will be closed and reopened +* for each hook that runs. */ int run_hooks (const char *hookname,struct run_hooks_opt *options);
t/t1360-config-based-hooks.sh +24 -0
@@ -276,4 +276,28 @@ test_expect_success 'hook.runHookDir is tolerant to unknown values' ' test_i18ncmp expected actual ' +test_expect_success 'stdin to multiple hooks' ' + git config --add hook.test.command "xargs -P1 -I% echo a%" && + git config --add hook.test.command "xargs -P1 -I% echo b%" && + test_when_finished "test_unconfig hook.test.command" && + + cat >input <<-EOF && + 1 + 2 + 3 + EOF + + cat >expected <<-EOF && + a1 + a2 + a3 + b1 + b2 + b3 + EOF + + git hook run --to-stdin=input test 2>actual && + test_cmp expected actual +' + test_done
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
Re: [PATCH v8 10/37] hook: support passing stdin to hooks
Quoted 7 lines · Emily Shaffer
On the frontend, this is supported by asking for a file path, rather than by reading stdin. Reading directly from stdin would involve caching the entire stdin (to memory or to disk) and reading it back from the beginning to each hook. We'd want to support cases like insufficient memory or storage for the file. While this may prove useful later, for now the path of least resistance is to just ask the user to make this interim file themselves.
We need to worry about cases where we wouldn't have enough memory to buffer the stdin, but still need to then do repo operations on the input from such a file, presumably some giant pre-receive update or something.
Seems unlikely, and the convenience of having stdin just work by just allocating that seems appealing, but let's read on to the rest of the series...
Re: [PATCH v8 10/37] hook: support passing stdin to hooks
In reply to Emily Shafferhook.c void run_hooks_opt_init(struct run_hooks_opt *o) · line 245
Quoted 11 lines · Emily Shaffer
hook.c +1
-0@@ -240,6 +240,7 @@ void run_hooks_opt_init(struct run_hooks_opt *o) {strvec_init (&o->env);strvec_init (&o->args); + o->path_to_stdin =NULL ; o->run_hookdir =configured_hookdir_opt (); }
The new member is initialized to NULL, and presumably the user of the API would point an existing string with it. Since there is no free() in opt_clear() introduced by this patch, the member is obviously a pointer to a borrowed piece of memory.
hook.h struct run_hooks_opt · line 58
Quoted 12 lines
hook.h +3
-0@@ -52,6 +52,9 @@ struct run_hooks_opt* to be overridden if the user can override it at the command line. */ enum hookdir_opt run_hookdir; + +/* Path to file which should be piped to stdin for each hook */ +const char *path_to_stdin; };
And we mark the fact that hook subsystem does not own it by making it "const char *". Looks quite consistent. Good.
PATCH v8 11/37 run-command: allow stdin for run_processes_parallel
In reply to Emily ShafferWhile it makes sense not to inherit stdin from the parent process to avoid deadlocking, it's not necessary to completely ban stdin to children. An informed user should be able to configure stdin safely. By setting some_child.process.no_stdin=1 before calling get_next_task() we provide a reasonable default behavior but enable users to set up stdin streaming for themselves during the callback.
some_child.process.stdout_to_stderr, however, remains unmodifiable by get_next_task() - the rest of the run_processes_parallel() API depends on child output in stderr.
--- run-command.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-)
run-command.c +8 -1
@@ -1693,6 +1693,14 @@ static int pp_start_one(struct parallel_processes *pp)if (i == pp->max_processes)BUG ("bookkeeping is hard" ); +/* +* By default, do not inherit stdin from the parent process - otherwise, +* all children would share stdin! Users may overwrite this to provide +* something to the child's stdin by having their 'get_next_task' +* callback assign 0 to .no_stdin and an appropriate integer to .in. +*/ + pp->children[i].process.no_stdin =1 ; + code = pp->get_next_task(&pp->children[i].process, &pp->children[i].err, pp->data,
@@ -1704,7 +1712,6 @@ static int pp_start_one(struct parallel_processes *pp) } pp->children[i].process.err =-1 ; pp->children[i].process.stdout_to_stderr =1 ; - pp->children[i].process.no_stdin =1 ;if (start_command (&pp->children[i].process)) { code = pp->start_failure(&pp->children[i].err,
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
PATCH v8 12/37 hook: allow parallel hook execution
In reply to Emily ShafferIn many cases, there's no reason not to allow hooks to execute in parallel. run_processes_parallel() is well-suited - it's a task queue that runs its housekeeping in series, which means users don't need to worry about thread safety on their callback data. True multithreaded execution with the async_* functions isn't necessary here. Synchronous hook execution can be achieved by only allowing 1 job to run at a time.
Teach run_hooks() to use that function for simple hooks which don't require stdin or capture of stderr.
---
Notes:
Per AEvar's request - parallel hook execution on day zero.
In most ways run_processes_parallel() worked great for me - but it didn't have great support for hooks where we pipe to and from. I had to add this support later in the series.
Since I modified an existing and in-use library I'd appreciate a keen look on these patches.
- Emily
Documentation/config/hook.txt | 5 ++ Documentation/git-hook.txt | 14 ++++- builtin/hook.c | 6 +- hook.c | 108 +++++++++++++++++++++++++++++----- hook.h | 21 ++++++- 5 files changed, 132 insertions(+), 22 deletions(-)
Documentation/config/hook.txt +5 -0
@@ -20,3 +20,8 @@ hook.runHookDir:: Controls how hooks contained in your hookdir are executed. Can be any of "yes", "warn", "interactive", or "no". Defaults to "yes". See linkgit:git-hook[1] and linkgit:git-config[1] "core.hooksPath"). + +hook.jobs:: + Specifies how many hooks can be run simultaneously during parallelized + hook execution. If unspecified, defaults to the number of processors on + the current system.
Documentation/git-hook.txt +12 -2
@@ -10,7 +10,7 @@ SYNOPSIS [verse] 'git hook' list <hook-name> 'git hook' run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] [--to-stdin=<path>] - <hook-name> + [(-j|--jobs) <n>] <hook-name> DESCRIPTION -----------
@@ -99,7 +99,7 @@ in the order they should be run, and print the config scope where the relevant `hook.<hook-name>.command` was specified, not the `hookcmd` (if applicable). This output is human-readable and the format is subject to change over time. -run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] [--to-stdin=<path>] `<hook-name>`:: +run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...] [--to-stdin=<path>] [(-j|--jobs)<n>] `<hook-name>`:: Runs hooks configured for `<hook-name>`, in the same order displayed by `git hook list`. Hooks configured this way may be run prepended with `sh -c`, so
@@ -131,6 +131,16 @@ Specify environment variables to set for every hook that is run. Specify a file which will be streamed into stdin for every hook that is run. Each hook will receive the entire file from beginning to EOF. +-j:: +--jobs:: + Only valid for `run`. ++ +Specify how many hooks to run simultaneously. If this flag is not specified, use +the value of the `hook.jobs` config. If the config is not specified, use the +number of CPUs on the current system. Some hooks may be ineligible for +parallelization: for example, 'commit-msg' intends hooks modify the commit +message body and cannot be parallelized. + CONFIGURATION ------------- include::config/hook.txt[]
builtin/hook.c +4 -2
@@ -9,7 +9,7 @@static const char *const builtin_hook_usage[] = {N_ ("git hook list <hookname>" ),N_ ("git hook run [(-e|--env)=<var>...] [(-a|--arg)=<arg>...]" -"[--to-stdin=<path>] <hookname>" ), +"[--to-stdin=<path>] [(-j|--jobs) <count>] <hookname>" ),NULL };
@@ -104,10 +104,12 @@ static int run(int argc, const char **argv, const char *prefix) N_("argument to pass to hook")),OPT_STRING (0 ,"to-stdin" , &opt.path_to_stdin,N_ ("path" ),N_ ("file to read into hooks' stdin" )), +OPT_INTEGER ('j' ,"jobs" , &opt.jobs, +N_ ("run up to <n> hooks simultaneously" )),OPT_END (), }; -run_hooks_opt_init (&opt); +run_hooks_opt_init_async (&opt); argc =parse_options (argc, argv, prefix, run_options, builtin_hook_usage,0 );
hook.c +92 -16
160 lines not on this page · raw message
hook.h +19 -2
@@ -38,6 +38,9 @@ enum hookdir_opt*/ enum hookdir_opt configured_hookdir_opt (void ); +/* Provides the number of threads to use for parallel hook execution. */ +int configured_hook_jobs (void ); +struct run_hooks_opt {/* Environment vars to be set for each hook */
@@ -48,16 +51,30 @@ struct run_hooks_opt/* * How should the hookdir be handled? -* Leave the RUN_HOOKS_OPT_INIT default in most cases; this only needs +* Leave the run_hooks_opt_init_*() default in most cases; this only needs * to be overridden if the user can override it at the command line. */ enum hookdir_opt run_hookdir;/* Path to file which should be piped to stdin for each hook */ const char *path_to_stdin; + +/* Number of threads to parallelize across */ +int jobs; +}; + +/* +* Callback provided to feed_pipe_fn and consume_sideband_fn. +*/ +struct hook_cb_data { +int rc; +struct list_head *head; +struct hook *run_me; +struct run_hooks_opt *options; }; -void run_hooks_opt_init (struct run_hooks_opt *o); +void run_hooks_opt_init_sync (struct run_hooks_opt *o); +void run_hooks_opt_init_async (struct run_hooks_opt *o);void run_hooks_opt_clear (struct run_hooks_opt *o);/*
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
PATCH v8 13/37 hook: allow specifying working directory for hooks
In reply to Emily ShafferHooks like "post-checkout" require that hooks have a different working directory than the initial process. Pipe that directly through to struct child_process.
Because we can just run 'git -C <some-dir> hook run ...' it shouldn't be necessary to pipe this option through the frontend. In fact, this reduces the possibility of users running hooks which affect some part of the filesystem outside of the repo in question.
---
Notes:
Needed later for "post-checkout" conversion.
hook.c | 2 ++ hook.h | 3 +++ 2 files changed, 5 insertions(+)
hook.c +2 -0
@@ -251,6 +251,7 @@ void run_hooks_opt_init_sync(struct run_hooks_opt *o) o->path_to_stdin =NULL ; o->run_hookdir =configured_hookdir_opt (); o->jobs =1 ; + o->dir =NULL ; }void run_hooks_opt_init_async (struct run_hooks_opt *o)
@@ -306,6 +307,7 @@ static int pick_next_hook(struct child_process *cp, cp->env = hook_cb->options->env.v; cp->stdout_to_stderr =1 ; cp->trace2_hook_name = hook->command.buf; + cp->dir = hook_cb->options->dir;/* * Commands from the config could be oneliners, but we know
hook.h +3 -0
@@ -61,6 +61,9 @@ struct run_hooks_opt/* Number of threads to parallelize across */ int jobs; + +/* Path to initial working directory for subprocess */ +const char *dir; };/*
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
PATCH v8 14/37 run-command: add stdin callback for parallelization
In reply to Emily ShafferIf a user of the run_processes_parallel() API wants to pipe a large amount of information to stdin of each parallel command, that information could exceed the buffer of the pipe allocated for that process's stdin. Generally this is solved by repeatedly writing to child_process.in between calls to start_command() and finish_command(); run_processes_parallel() did not provide users an opportunity to access child_process at that time.
Because the data might be extremely large (for example, a list of all refs received during a push from a client) simply taking a string_list or strbuf is not as scalable as using a callback; the rest of the run_processes_parallel() API also uses callbacks, so making this feature match the rest of the API reduces mental load on the user.
--- builtin/fetch.c | 1 + builtin/submodule--helper.c | 2 +- hook.c | 1 + run-command.c | 54 +++++++++++++++++++++++++++++++++++-- run-command.h | 17 +++++++++++- submodule.c | 1 + t/helper/test-run-command.c | 31 ++++++++++++++++++--- t/t0061-run-command.sh | 30 +++++++++++++++++++++ 8 files changed, 129 insertions(+), 8 deletions(-)
builtin/fetch.c +1 -0
@@ -1757,6 +1757,7 @@ static int fetch_multiple(struct string_list *list, int max_children) result =run_processes_parallel_tr2 (max_children, &fetch_next_remote, &fetch_failed_to_start, +NULL , &fetch_finished, &state,"fetch" ,"parallel/fetch" );
builtin/submodule--helper.c +1 -1
@@ -2294,7 +2294,7 @@ static int update_submodules(struct submodule_update_clone *suc)int i;run_processes_parallel_tr2 (suc->max_jobs, update_clone_get_next_task, - update_clone_start_failure, + update_clone_start_failure,NULL , update_clone_task_finished, suc,"submodule" ,"parallel/update" );
hook.c +1 -0
@@ -400,6 +400,7 @@ int run_hooks(const char *hookname, struct run_hooks_opt *options) run_processes_parallel_tr2(options->jobs, pick_next_hook, notify_start_failure, + NULL, notify_hook_finished, &cb_data, "hook",
run-command.c +52 -2
125 lines not on this page · raw message
run-command.h +16 -1
@@ -443,6 +443,20 @@ typedef int (*start_failure_fn)(struct strbuf *out,void *pp_cb, void *pp_task_cb); +/** +* This callback is called repeatedly on every child process who requests +* start_command() to create a pipe by setting child_process.in < 0. +* +* pp_cb is the callback cookie as passed into run_processes_parallel, and +* pp_task_cb is the callback cookie as passed into get_next_task_fn. +* The contents of 'send' will be read into the pipe and passed to the pipe. +* +* Return nonzero to close the pipe. +*/ +typedef int (*feed_pipe_fn )(struct strbuf *pipe, +void *pp_cb, +void *pp_task_cb); +/** * This callback is called on every child process that finished processing. *
@@ -477,10 +491,11 @@ typedef int (*task_finished_fn)(int result,int run_processes_parallel (int n,get_next_task_fn ,start_failure_fn , +feed_pipe_fn ,task_finished_fn ,void *pp_cb);int run_processes_parallel_tr2 (int n,get_next_task_fn ,start_failure_fn , -task_finished_fn ,void *pp_cb, +feed_pipe_fn ,task_finished_fn ,void *pp_cb,const char *tr2_category,const char *tr2_label);#endif
submodule.c +1 -0
@@ -1644,6 +1644,7 @@ int fetch_populated_submodules(struct repository *r,run_processes_parallel_tr2 (max_parallel_jobs, get_next_submodule, fetch_start_failure, +NULL , fetch_finish, &spf,"submodule" ,"parallel/fetch" );
t/helper/test-run-command.c +27 -4
@@ -32,8 +32,13 @@ static int parallel_next(struct child_process *cp,return 0 ;strvec_pushv (&cp->args, d->argv); + cp->in = d->in; + cp->no_stdin = d->no_stdin;strbuf_addstr (err,"preloaded output of a child\n" ); number_callbacks++; + + *task_cb =xmalloc (sizeof (int )); + *(int *)(*task_cb) =2 ;return 1 ; }
@@ -55,6 +60,17 @@ static int task_finished(int result,return 1 ; } +static int test_stdin (struct strbuf *pipe,void *cb,void *task_cb) +{ +int *lines_remaining = task_cb; + +if (*lines_remaining) +strbuf_addf (pipe,"sample stdin %d\n" , --(*lines_remaining)); + +return !(*lines_remaining); +} + +struct testsuite {struct string_list tests, failed;int next;
@@ -185,7 +201,7 @@ static int testsuite(int argc, const char **argv) suite.tests.nr, max_jobs); ret =run_processes_parallel (max_jobs, next_test, test_failed, - test_finished, &suite); + test_stdin, test_finished, &suite);if (suite.failed.nr >0 ) { ret =1 ;
@@ -413,15 +429,22 @@ int cmd__run_command(int argc, const char **argv)if (!strcmp (argv[1 ],"run-command-parallel" ))exit (run_processes_parallel (jobs, parallel_next, -NULL ,NULL , &proc)); +NULL ,NULL ,NULL , &proc));if (!strcmp (argv[1 ],"run-command-abort" ))exit (run_processes_parallel (jobs, parallel_next, -NULL , task_finished, &proc)); +NULL ,NULL , task_finished, &proc));if (!strcmp (argv[1 ],"run-command-no-jobs" ))exit (run_processes_parallel (jobs, no_job, -NULL , task_finished, &proc)); +NULL ,NULL , task_finished, &proc)); + +if (!strcmp (argv[1 ],"run-command-stdin" )) { + proc.in =-1 ; + proc.no_stdin =0 ; +exit (run_processes_parallel (jobs, parallel_next,NULL , + test_stdin,NULL , &proc)); + }fprintf (stderr,"check usage\n" );return 1 ;
t/t0061-run-command.sh +30 -0
@@ -143,6 +143,36 @@ test_expect_success 'run_command runs in parallel with more tasks than jobs avai test_cmp expect actual ' +cat >expect <<-EOF +preloaded output of a child +listening for stdin: +sample stdin 1 +sample stdin 0 +preloaded output of a child +listening for stdin: +sample stdin 1 +sample stdin 0 +preloaded output of a child +listening for stdin: +sample stdin 1 +sample stdin 0 +preloaded output of a child +listening for stdin: +sample stdin 1 +sample stdin 0 +EOF + +test_expect_success 'run_command listens to stdin' ' + write_script stdin-script <<-\EOF && + echo "listening for stdin:" + while read line; do + echo "$line" + done + EOF + test-tool run-command run-command-stdin 2 ./stdin-script 2>actual && + test_cmp expect actual +' + cat >expect <<-EOF preloaded output of a child asking for a quick stop
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
PATCH v8 15/37 hook: provide stdin by string_list or callback
In reply to Emily ShafferIn cases where a hook requires only a small amount of information via stdin, it should be simple for users to provide a string_list alone. But in more complicated cases where the stdin is too large to hold in memory, let's provide a callback the users can populate line after line with instead.
--- hook.c | 35 ++++++++++++++++++++++++++++++++++- hook.h | 28 ++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-)
hook.c +34 -1
@@ -9,6 +9,7 @@ void free_hook(struct hook *ptr) {if (ptr) {strbuf_release (&ptr->command); +free (ptr->feed_pipe_cb_data);free (ptr); } }
@@ -39,6 +40,7 @@ static void append_or_move_hook(struct list_head *head, const char *command)strbuf_init (&to_add->command,0 );strbuf_addstr (&to_add->command, command); to_add->from_hookdir =0 ; + to_add->feed_pipe_cb_data =NULL ; }/* re-set the scope so we show where an override was specified */
@@ -252,6 +254,8 @@ void run_hooks_opt_init_sync(struct run_hooks_opt *o) o->run_hookdir =configured_hookdir_opt (); o->jobs =1 ; o->dir =NULL ; + o->feed_pipe =NULL ; + o->feed_pipe_ctx =NULL ; }void run_hooks_opt_init_async (struct run_hooks_opt *o)
@@ -285,6 +289,28 @@ void run_hooks_opt_clear(struct run_hooks_opt *o)strvec_clear (&o->args); } +int pipe_from_string_list (struct strbuf *pipe,void *pp_cb,void *pp_task_cb) +{ +int *item_idx; +struct hook *ctx = pp_task_cb; +struct string_list *to_pipe = ((struct hook_cb_data *)pp_cb)->options->feed_pipe_ctx; + +/* Bootstrap the state manager if necessary. */ +if (!ctx->feed_pipe_cb_data) { + ctx->feed_pipe_cb_data =xmalloc (sizeof (unsigned int )); + *(int *)ctx->feed_pipe_cb_data =0 ; + } + + item_idx = ctx->feed_pipe_cb_data; + +if (*item_idx < to_pipe->nr) { +strbuf_addf (pipe,"%s\n" , to_pipe->items[*item_idx].string); + (*item_idx)++; +return 0 ; + } +return 1 ; +} + static int pick_next_hook(struct child_process *cp, struct strbuf *out, void *pp_cb,
@@ -300,6 +326,10 @@ static int pick_next_hook(struct child_process *cp,if (hook_cb->options->path_to_stdin) { cp->no_stdin =0 ; cp->in =xopen (hook_cb->options->path_to_stdin,O_RDONLY ); + }else if (hook_cb->options->feed_pipe) { +/* ask for start_command() to make a pipe for us */ + cp->in =-1 ; + cp->no_stdin =0 ; }else { cp->no_stdin =1 ; }
@@ -379,6 +409,9 @@ int run_hooks(const char *hookname, struct run_hooks_opt *options)if (!options)BUG ("a struct run_hooks_opt must be provided to run_hooks" ); +if (options->path_to_stdin && options->feed_pipe) +BUG ("choose only one method to populate stdin" ); +strbuf_addstr (&hookname_str, hookname); to_run =hook_list (&hookname_str);
@@ -400,7 +433,7 @@ int run_hooks(const char *hookname, struct run_hooks_opt *options) run_processes_parallel_tr2(options->jobs, pick_next_hook, notify_start_failure, - NULL, + options->feed_pipe, notify_hook_finished, &cb_data, "hook",
hook.h +28 -0
@@ -2,6 +2,7 @@#include "list.h" #include "strbuf.h" #include "strvec.h" +#include "run-command.h" struct hook {struct list_head list;
@@ -13,6 +14,12 @@ struct hook {/* The literal command to run. */ struct strbuf command; unsigned from_hookdir : 1; + +/* +* Use this to keep state for your feed_pipe_fn if you are using +* run_hooks_opt.feed_pipe. Otherwise, do not touch it. +*/ +void *feed_pipe_cb_data; };/*
@@ -58,14 +65,35 @@ struct run_hooks_opt/* Path to file which should be piped to stdin for each hook */ const char *path_to_stdin; +/* +* Callback and state pointer to ask for more content to pipe to stdin. +* Will be called repeatedly, for each hook. See +* hook.c:pipe_from_stdin() for an example. Keep per-hook state in +* hook.feed_pipe_cb_data (per process). Keep initialization context in +* feed_pipe_ctx (shared by all processes). +* +* See 'pipe_from_string_list()' for info about how to specify a +* string_list as the stdin input instead of writing your own handler. +*/ +feed_pipe_fn feed_pipe; +void *feed_pipe_ctx;/* Number of threads to parallelize across */ int jobs;/* Path to initial working directory for subprocess */ const char *dir; + }; +/* +* To specify a 'struct string_list', set 'run_hooks_opt.feed_pipe_ctx' to the +* string_list and set 'run_hooks_opt.feed_pipe' to 'pipe_from_string_list()'. +* This will pipe each string in the list to stdin, separated by newlines. (Do +* not inject your own newlines.) +*/ +int pipe_from_string_list (struct strbuf *pipe,void *pp_cb,void *pp_task_cb); +/* * Callback provided to feed_pipe_fn and consume_sideband_fn. */
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
PATCH v8 16/37 run-command: allow capturing of collated output
In reply to Emily ShafferSome callers, for example server-side hooks which wish to relay hook output to clients across a transport, want to capture what would normally print to stderr and do something else with it. Allow that via a callback.
By calling the callback regardless of whether there's output available, we allow clients to send e.g. a keepalive if necessary.
Because we expose a strbuf, not a fd or FILE*, there's no need to create a temporary pipe or similar - we can just skip the print to stderr and instead hand it to the caller.
---
builtin/fetch.c | 2 +- builtin/submodule--helper.c | 2 +- hook.c | 1 + run-command.c | 33 +++++++++++++++++++++++++-------- run-command.h | 18 +++++++++++++++++- submodule.c | 2 +- t/helper/test-run-command.c | 25 ++++++++++++++++++++----- t/t0061-run-command.sh | 7 +++++++ 8 files changed, 73 insertions(+), 17 deletions(-)
builtin/fetch.c +1 -1
@@ -1757,7 +1757,7 @@ static int fetch_multiple(struct string_list *list, int max_children) result =run_processes_parallel_tr2 (max_children, &fetch_next_remote, &fetch_failed_to_start, -NULL , +NULL ,NULL , &fetch_finished, &state,"fetch" ,"parallel/fetch" );
builtin/submodule--helper.c +1 -1
@@ -2294,7 +2294,7 @@ static int update_submodules(struct submodule_update_clone *suc)int i;run_processes_parallel_tr2 (suc->max_jobs, update_clone_get_next_task, - update_clone_start_failure,NULL , + update_clone_start_failure,NULL ,NULL , update_clone_task_finished, suc,"submodule" ,"parallel/update" );
hook.c +1 -0
@@ -434,6 +434,7 @@ int run_hooks(const char *hookname, struct run_hooks_opt *options) pick_next_hook, notify_start_failure, options->feed_pipe, +NULL , notify_hook_finished, &cb_data, "hook",
run-command.c +25 -8
@@ -1559,6 +1559,7 @@ struct parallel_processes {get_next_task_fn get_next_task;start_failure_fn start_failure;feed_pipe_fn feed_pipe; +consume_sideband_fn consume_sideband;task_finished_fn task_finished;struct {
@@ -1624,6 +1625,7 @@ static void pp_init(struct parallel_processes *pp,get_next_task_fn get_next_task, start_failure_fn start_failure, feed_pipe_fn feed_pipe, + consume_sideband_fn consume_sideband, task_finished_fn task_finished, void *data) {
@@ -1644,6 +1646,7 @@ static void pp_init(struct parallel_processes *pp, pp->start_failure = start_failure ? start_failure : default_start_failure; pp->feed_pipe = feed_pipe ? feed_pipe : default_feed_pipe; pp->task_finished = task_finished ? task_finished : default_task_finished; + pp->consume_sideband = consume_sideband; pp->nr_processes =0 ; pp->output_owner =0 ;
@@ -1680,7 +1683,10 @@ static void pp_cleanup(struct parallel_processes *pp)* When get_next_task added messages to the buffer in its last * iteration, the buffered output is non empty. */ -strbuf_write (&pp->buffered_output, stderr); +if (pp->consume_sideband) + pp->consume_sideband (&pp->buffered_output, pp->data); +else +strbuf_write (&pp->buffered_output, stderr);strbuf_release (&pp->buffered_output);sigchain_pop_common ();
@@ -1801,9 +1807,13 @@ static void pp_buffer_stderr(struct parallel_processes *pp, int output_timeout)static void pp_output (struct parallel_processes *pp) {int i = pp->output_owner; +if (pp->children[i].state ==GIT_CP_WORKING && pp->children[i].err.len) { -strbuf_write (&pp->children[i].err, stderr); +if (pp->consume_sideband) + pp->consume_sideband (&pp->children[i].err, pp->data); +else +strbuf_write (&pp->children[i].err, stderr);strbuf_reset (&pp->children[i].err); } }
@@ -1842,11 +1852,15 @@ static int pp_collect_finished(struct parallel_processes *pp)strbuf_addbuf (&pp->buffered_output, &pp->children[i].err);strbuf_reset (&pp->children[i].err); }else { -strbuf_write (&pp->children[i].err, stderr); +/* Output errors, then all other finished child processes */ +if (pp->consume_sideband) { + pp->consume_sideband (&pp->children[i].err, pp->data); + pp->consume_sideband (&pp->buffered_output, pp->data); + }else { +strbuf_write (&pp->children[i].err, stderr); +strbuf_write (&pp->buffered_output, stderr); + }strbuf_reset (&pp->children[i].err); - -/* Output all other finished child processes */ -strbuf_write (&pp->buffered_output, stderr);strbuf_reset (&pp->buffered_output);/*
@@ -1870,6 +1884,7 @@ int run_processes_parallel(int n,get_next_task_fn get_next_task, start_failure_fn start_failure, feed_pipe_fn feed_pipe, + consume_sideband_fn consume_sideband, task_finished_fn task_finished, void *pp_cb) {
@@ -1880,7 +1895,7 @@ int run_processes_parallel(int n,sigchain_push (SIGPIPE ,SIG_IGN ); -pp_init (&pp, n, get_next_task, start_failure, feed_pipe, task_finished, pp_cb); +pp_init (&pp, n, get_next_task, start_failure, feed_pipe, consume_sideband, task_finished, pp_cb);while (1 ) { for (i = 0; i < spawn_cap && !pp.shutdown &&
@@ -1918,6 +1933,7 @@ int run_processes_parallel(int n,int run_processes_parallel_tr2 (int n,get_next_task_fn get_next_task,start_failure_fn start_failure,feed_pipe_fn feed_pipe, +consume_sideband_fn consume_sideband,task_finished_fn task_finished,void *pp_cb,const char *tr2_category,const char *tr2_label) {
@@ -1927,7 +1943,8 @@ int run_processes_parallel_tr2(int n, get_next_task_fn get_next_task, ((n < 1) ? online_cpus() : n)); result =run_processes_parallel (n, get_next_task, start_failure, - feed_pipe, task_finished, pp_cb); + feed_pipe, consume_sideband, + task_finished, pp_cb);trace2_region_leave (tr2_category, tr2_label,NULL );
run-command.h +17 -1
@@ -457,6 +457,20 @@ typedef int (*feed_pipe_fn)(struct strbuf *pipe,void *pp_cb, void *pp_task_cb); +/** +* If this callback is provided, instead of collating process output to stderr, +* they will be collated into a new pipe. consume_sideband_fn will be called +* repeatedly. When output is available on that pipe, it will be contained in +* 'output'. But it will be called with an empty 'output' too, to allow for +* keepalives or similar operations if necessary. +* +* pp_cb is the callback cookie as passed into run_processes_parallel. +* +* Since this callback is provided with the collated output, no task cookie is +* provided. +*/ +typedef void (*consume_sideband_fn )(struct strbuf *output,void *pp_cb); +/** * This callback is called on every child process that finished processing. *
@@ -492,10 +506,12 @@ int run_processes_parallel(int n, get_next_task_fn, start_failure_fn, feed_pipe_fn, + consume_sideband_fn, task_finished_fn, void *pp_cb);int run_processes_parallel_tr2 (int n,get_next_task_fn ,start_failure_fn , -feed_pipe_fn ,task_finished_fn ,void *pp_cb, +feed_pipe_fn ,consume_sideband_fn , +task_finished_fn ,void *pp_cb,const char *tr2_category,const char *tr2_label);#endif
submodule.c +1 -1
@@ -1644,7 +1644,7 @@ int fetch_populated_submodules(struct repository *r,run_processes_parallel_tr2 (max_parallel_jobs, get_next_submodule, fetch_start_failure, -NULL , +NULL ,NULL , fetch_finish, &spf,"submodule" ,"parallel/fetch" );
t/helper/test-run-command.c +20 -5
@@ -51,6 +51,16 @@ static int no_job(struct child_process *cp,return 0 ; } +static void test_consume_sideband (struct strbuf *output,void *cb) +{ +FILE *sideband; + + sideband =fopen ("./sideband" ,"a" ); + +strbuf_write (output, sideband); +fclose (sideband); +} + static int task_finished(int result, struct strbuf *err, void *pp_cb,
@@ -201,7 +211,7 @@ static int testsuite(int argc, const char **argv) suite.tests.nr, max_jobs); ret =run_processes_parallel (max_jobs, next_test, test_failed, - test_stdin, test_finished, &suite); + test_stdin,NULL , test_finished, &suite);if (suite.failed.nr >0 ) { ret =1 ;
@@ -429,23 +439,28 @@ int cmd__run_command(int argc, const char **argv)if (!strcmp (argv[1 ],"run-command-parallel" ))exit (run_processes_parallel (jobs, parallel_next, -NULL ,NULL ,NULL , &proc)); +NULL ,NULL ,NULL ,NULL , &proc));if (!strcmp (argv[1 ],"run-command-abort" ))exit (run_processes_parallel (jobs, parallel_next, -NULL ,NULL , task_finished, &proc)); +NULL ,NULL ,NULL , task_finished, &proc));if (!strcmp (argv[1 ],"run-command-no-jobs" ))exit (run_processes_parallel (jobs, no_job, -NULL ,NULL , task_finished, &proc)); +NULL ,NULL ,NULL , task_finished, &proc));if (!strcmp (argv[1 ],"run-command-stdin" )) { proc.in =-1 ; proc.no_stdin =0 ;exit (run_processes_parallel (jobs, parallel_next,NULL , - test_stdin,NULL , &proc)); + test_stdin,NULL ,NULL , &proc)); } +if (!strcmp (argv[1 ],"run-command-sideband" )) +exit (run_processes_parallel (jobs, parallel_next,NULL ,NULL , + test_consume_sideband,NULL , + &proc)); +fprintf (stderr,"check usage\n" );return 1 ; }
t/t0061-run-command.sh +7 -0
@@ -143,6 +143,13 @@ test_expect_success 'run_command runs in parallel with more tasks than jobs avai test_cmp expect actual ' +test_expect_success 'run_command can divert output' ' + test_when_finished rm sideband && + test-tool run-command run-command-sideband 3 sh -c "printf \"%s\n%s\n\" Hello World" 2>actual && + test_must_be_empty actual && + test_cmp expect sideband +' + cat >expect <<-EOF preloaded output of a child listening for stdin:
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
PATCH v8 17/37 hooks: allow callers to capture output
In reply to Emily ShafferSome server-side hooks will require capturing output to send over sideband instead of printing directly to stderr. Expose that capability.
---
Notes:
You can see this in practice in the conversions for some of the push hooks, like 'receive-pack'.
hook.c | 3 ++- hook.h | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-)
hook.c +2 -1
@@ -256,6 +256,7 @@ void run_hooks_opt_init_sync(struct run_hooks_opt *o) o->dir =NULL ; o->feed_pipe =NULL ; o->feed_pipe_ctx =NULL ; + o->consume_sideband =NULL ; }void run_hooks_opt_init_async (struct run_hooks_opt *o)
@@ -434,7 +435,7 @@ int run_hooks(const char *hookname, struct run_hooks_opt *options) pick_next_hook, notify_start_failure, options->feed_pipe, -NULL , + options->consume_sideband, notify_hook_finished, &cb_data, "hook",
hook.h +8 -0
@@ -78,6 +78,14 @@ struct run_hooks_optfeed_pipe_fn feed_pipe;void *feed_pipe_ctx; +/* +* Populate this to capture output and prevent it from being printed to +* stderr. This will be passed directly through to +* run_command:run_parallel_processes(). See t/helper/test-run-command.c +* for an example. +*/ +consume_sideband_fn consume_sideband; +/* Number of threads to parallelize across */ int jobs;
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
Re: [PATCH v8 17/37] hooks: allow callers to capture output
Quoted 2 lines · Emily Shaffer
Some server-side hooks will require capturing output to send over sideband instead of printing directly to stderr. Expose that capability.
So added here in 17/37 and not used until 30/37. As a point on readability (this isn't the first such patch) I think it would be better to just squash those together with some "since we now need access to consume_sideband in hooks, do that ...".
If there's a much larger API it makes sense to do it as another step...
Quoted 33 lines
hook.c | 3 ++- hook.h | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-)hook.c +2
-1@@ -256,6 +256,7 @@ void run_hooks_opt_init_sync(struct run_hooks_opt *o) o->dir =NULL ; o->feed_pipe =NULL ; o->feed_pipe_ctx =NULL ; + o->consume_sideband =NULL ; }void run_hooks_opt_init_async (struct run_hooks_opt *o)@@ -434,7 +435,7 @@ int run_hooks(const char *hookname, struct run_hooks_opt *options) pick_next_hook, notify_start_failure, options->feed_pipe, -NULL , + options->consume_sideband, notify_hook_finished, &cb_data, "hook",hook.h +0
-0@@ -78,6 +78,14 @@ struct run_hooks_optfeed_pipe_fn feed_pipe;void *feed_pipe_ctx;
hook.h struct run_hooks_opt · line 90
Quoted 10 lines
hook.h +8
-0+/* +* Populate this to capture output and prevent it from being printed to +* stderr. This will be passed directly through to +* run_command:run_parallel_processes(). See t/helper/test-run-command.c +* for an example. +*/ +consume_sideband_fn consume_sideband; +/* Number of threads to parallelize across */ int jobs;
...but this scaffolding is rather trivial.
Re: [PATCH v8 17/37] hooks: allow callers to capture output
Quoted 11 lines · Ævar Arnfjörð Bjarmason
Quoted 2 lines · Emily Shaffer
Some server-side hooks will require capturing output to send over sideband instead of printing directly to stderr. Expose that capability.
So added here in 17/37 and not used until 30/37. As a point on readability (this isn't the first such patch) I think it would be better to just squash those together with some "since we now need access to
consume_sidebandin hooks, do that ...".
Yeah. When I was putting together the series I had two thoughts on how best to organize it:
1. Adding functionality just-in-time for the hook that needs it (like you describe)
or
2. Implementing the whole utility, then doing hook conversions in a separate chunk or series (what I went with).
I chose 2 for a couple reasons: that it would be easier for people who just care "did a hook I use start working differently?" to review only the second chunk of the change, and that it would be easier if we wanted to adopt the library part into the codebase without converting the hooks to use it (this was listed as a step in the design doc, but I think we ended up abandoning it). The differentiation was certainly easier when I had the two "chunks" separated into a part I and part II series, but Junio asked me to combine them starting with this revision so it would be easier to merge to 'seen' (as I understood it).
At this point, I'd prefer not to rearrange the series, though.
- Emily
PATCH v8 18/37 commit: use config-based hooks
In reply to Emily ShafferAs part of the adoption of config-based hooks, teach run_commit_hook() to call hook.h instead of run-command.h. This covers 'pre-commit', 'commit-msg', and 'prepare-commit-msg'. Additionally, ask the hook library - not run-command - whether any hooks will be run, as it's possible hooks may exist in the config but not the hookdir.
Because all but 'post-commit' hooks are expected to make some state change, force all but 'post-commit' hook to run in series. 'post-commit' "is meant primarily for notification, and cannot affect the outcome of git commit," so it is fine to run in parallel.
--- Documentation/githooks.txt | 13 +++++++++++ builtin/commit.c | 11 +++++----- builtin/merge.c | 9 ++++---- commit.c | 22 ++++++++++++++----- commit.h | 3 ++- sequencer.c | 7 +++--- ...3-pre-commit-and-pre-merge-commit-hooks.sh | 17 ++++++++++++-- 7 files changed, 61 insertions(+), 21 deletions(-)
Documentation/githooks.txt +13 -0
@@ -103,6 +103,8 @@ The default 'pre-commit' hook, when enabled--and with the `hooks.allownonascii` config option unset or set to false--prevents the use of non-ASCII filenames. +Hooks executed during 'pre-commit' will not be parallelized. + pre-merge-commit ~~~~~~~~~~~~~~~~
@@ -125,6 +127,8 @@ need to be resolved and the result committed separately (see linkgit:git-merge[1]). At that point, this hook will not be executed, but the 'pre-commit' hook will, if it is enabled. +Hooks executed during 'pre-merge-commit' will not be parallelized. + prepare-commit-msg ~~~~~~~~~~~~~~~~~~
@@ -150,6 +154,9 @@ be used as replacement for pre-commit hook. The sample `prepare-commit-msg` hook that comes with Git removes the help message found in the commented portion of the commit template. +Hooks executed during 'prepare-commit-msg' will not be parallelized, because +hooks are expected to edit the file containing the commit log message. + commit-msg ~~~~~~~~~~
@@ -166,6 +173,9 @@ file. The default 'commit-msg' hook, when enabled, detects duplicate `Signed-off-by` trailers, and aborts the commit if one is found. +Hooks executed during 'commit-msg' will not be parallelized, because hooks are +expected to edit the file containing the proposed commit log message. + post-commit ~~~~~~~~~~~
@@ -175,6 +185,9 @@ invoked after a commit is made. This hook is meant primarily for notification, and cannot affect the outcome of `git commit`. +Hooks executed during 'post-commit' will run in parallel, unless hook.jobs is +configured to 1. + pre-rebase ~~~~~~~~~~
builtin/commit.c +6 -5
@@ -36,6 +36,7 @@#include "help.h" #include "commit-reach.h" #include "commit-graph.h" +#include "hook.h" static const char *const builtin_commit_usage[] = {N_ ("git commit [<options>] [--] <pathspec>..." ),
@@ -699,7 +700,7 @@ static int prepare_to_commit(const char *index_file, const char *prefix,/* This checks and barfs if author is badly specified */ determine_author_info (author_ident); -if (!no_verify &&run_commit_hook (use_editor, index_file,"pre-commit" ,NULL )) +if (!no_verify &&run_commit_hook (use_editor,0 , index_file,"pre-commit" ,NULL ))return 0 ;if (squash_message) {
@@ -983,7 +984,7 @@ static int prepare_to_commit(const char *index_file, const char *prefix,return 0 ; } -if (!no_verify &&find_hook ("pre-commit" )) { +if (!no_verify &&hook_exists ("pre-commit" ,HOOKDIR_USE_CONFIG )) {/* * Re-read the index as pre-commit hook could have updated it, * and write it out as a tree. We must do this before we invoke
@@ -998,7 +999,7 @@ static int prepare_to_commit(const char *index_file, const char *prefix,return 0 ; } -if (run_commit_hook (use_editor, index_file,"prepare-commit-msg" , +if (run_commit_hook (use_editor,0 , index_file,"prepare-commit-msg" ,git_path_commit_editmsg (), hook_arg1, hook_arg2,NULL ))return 0 ;
@@ -1015,7 +1016,7 @@ static int prepare_to_commit(const char *index_file, const char *prefix, }if (!no_verify && -run_commit_hook (use_editor, index_file,"commit-msg" ,git_path_commit_editmsg (),NULL )) { +run_commit_hook (use_editor,0 , index_file,"commit-msg" ,git_path_commit_editmsg (),NULL )) {return 0 ; }
@@ -1701,7 +1702,7 @@ int cmd_commit(int argc, const char **argv, const char *prefix)repo_rerere (the_repository,0 );run_auto_maintenance (quiet); -run_commit_hook (use_editor,get_index_file (),"post-commit" ,NULL ); +run_commit_hook (use_editor,1 ,get_index_file (),"post-commit" ,NULL );if (amend && !no_post_rewrite) {commit_post_rewrite (the_repository, current_head, &oid); }
builtin/merge.c +5 -4
@@ -43,6 +43,7 @@#include "commit-reach.h" #include "wt-status.h" #include "commit-graph.h" +#include "hook.h" #define DEFAULT_TWOHEAD (1<<0)#define DEFAULT_OCTOPUS (1<<1)
@@ -837,14 +838,14 @@ static void prepare_to_commit(struct commit_list *remoteheads)struct strbuf msg =STRBUF_INIT ;const char *index_file =get_index_file (); -if (!no_verify &&run_commit_hook (0 < option_edit, index_file,"pre-merge-commit" ,NULL )) +if (!no_verify &&run_commit_hook (0 < option_edit,0 , index_file,"pre-merge-commit" ,NULL ))abort_commit (remoteheads,NULL );/* * Re-read the index as pre-merge-commit hook could have updated it, * and write it out as a tree. We must do this before we invoke * the editor and after we invoke run_status above. */ -if (find_hook ("pre-merge-commit" )) +if (hook_exists ("pre-merge-commit" ,HOOKDIR_USE_CONFIG ))discard_cache ();read_cache_from (index_file);strbuf_addbuf (&msg, &merge_msg);
@@ -865,7 +866,7 @@ static void prepare_to_commit(struct commit_list *remoteheads)append_signoff (&msg,ignore_non_trailer (msg.buf, msg.len),0 );write_merge_heads (remoteheads);write_file_buf (git_path_merge_msg (the_repository), msg.buf, msg.len); -if (run_commit_hook (0 < option_edit,get_index_file (),"prepare-commit-msg" , +if (run_commit_hook (0 < option_edit,0 ,get_index_file (),"prepare-commit-msg" ,git_path_merge_msg (the_repository),"merge" ,NULL ))abort_commit (remoteheads,NULL );if (0 < option_edit) {
@@ -873,7 +874,7 @@ static void prepare_to_commit(struct commit_list *remoteheads)abort_commit (remoteheads,NULL ); } -if (!no_verify &&run_commit_hook (0 < option_edit,get_index_file (), +if (!no_verify &&run_commit_hook (0 < option_edit,0 ,get_index_file (),"commit-msg" ,git_path_merge_msg (the_repository),NULL ))abort_commit (remoteheads,NULL );
commit.c +16 -6
@@ -21,6 +21,7 @@#include "commit-reach.h" #include "run-command.h" #include "shallow.h" +#include "hook.h" static struct commit_extra_header *read_commit_extra_header_lines (const char *buf,size_t len,const char **);
@@ -1681,25 +1682,34 @@ size_t ignore_non_trailer(const char *buf, size_t len)return boc ? len - boc : len - cutoff; } -int run_commit_hook (int editor_is_used,const char *index_file, +int run_commit_hook (int editor_is_used,int parallelize,const char *index_file,const char *name, ...) { -struct strvec hook_env =STRVEC_INIT ; +struct run_hooks_opt opt;va_list args; +const char *arg;int ret; -strvec_pushf (&hook_env,"GIT_INDEX_FILE=%s" , index_file); +run_hooks_opt_init_sync (&opt); + +if (parallelize) + opt.jobs =configured_hook_jobs (); + +strvec_pushf (&opt.env,"GIT_INDEX_FILE=%s" , index_file);/* * Let the hook know that no editor will be launched. */ if (!editor_is_used) -strvec_push (&hook_env,"GIT_EDITOR=:" ); +strvec_push (&opt.env,"GIT_EDITOR=:" );va_start (args, name); - ret =run_hook_ve (hook_env.v, name, args); + while ((arg = va_arg(args, const char *))) +strvec_push (&opt.args, arg);va_end (args); -strvec_clear (&hook_env); + + ret =run_hooks (name, &opt); +run_hooks_opt_clear (&opt);return ret; }
commit.h +2 -1
@@ -360,7 +360,8 @@ int compare_commits_by_commit_date(const void *a_, const void *b_, void *unused)int compare_commits_by_gen_then_commit_date (const void *a_,const void *b_,void *unused);LAST_ARG_MUST_BE_NULL -int run_commit_hook(int editor_is_used, const char *index_file, const char *name, ...); +int run_commit_hook(int editor_is_used, int parallelize, const char *index_file, +const char *name, ...);/* Sign a commit or tag buffer, storing the result in a header. */ int sign_with_header (struct strbuf *buf,const char *keyid);
sequencer.c +4 -3
@@ -34,6 +34,7 @@#include "commit-reach.h" #include "rebase-interactive.h" #include "reset.h" +#include "hook.h" #define GIT_REFLOG_ACTION "GIT_REFLOG_ACTION"
@@ -1206,7 +1207,7 @@ static int run_prepare_commit_msg_hook(struct repository *r, }else { arg1 ="message" ; } -if (run_commit_hook (0 , r->index_file,"prepare-commit-msg" , name, +if (run_commit_hook (0 ,0 , r->index_file,"prepare-commit-msg" , name, arg1, arg2,NULL )) ret =error (_ ("'prepare-commit-msg' hook failed" ));
@@ -1444,7 +1445,7 @@ static int try_to_commit(struct repository *r, } } -if (find_hook ("prepare-commit-msg" )) { +if (hook_exists ("prepare-commit-msg" ,HOOKDIR_USE_CONFIG )) { res =run_prepare_commit_msg_hook (r, msg, hook_commit);if (res) goto out;
@@ -1536,7 +1537,7 @@ static int try_to_commit(struct repository *r, goto out; } -run_commit_hook (0 , r->index_file,"post-commit" ,NULL ); +run_commit_hook (0 ,1 , r->index_file,"post-commit" ,NULL );if (flags &AMEND_MSG )commit_post_rewrite (r, current_head, oid);
t/t7503-pre-commit-and-pre-merge-commit-hooks.sh +15 -2
@@ -8,8 +8,8 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME . ./test-lib.sh HOOKDIR="$(git rev-parse --git-dir)/hooks" -PRECOMMIT="$HOOKDIR/pre-commit" -PREMERGE="$HOOKDIR/pre-merge-commit" +PRECOMMIT="$(pwd)/$HOOKDIR/pre-commit" +PREMERGE="$(pwd)/$HOOKDIR/pre-merge-commit" # Prepare sample scripts that write their $0 to actual_hooks test_expect_success 'sample script setup' '
@@ -106,6 +106,19 @@ test_expect_success 'with succeeding hook' ' test_cmp expected_hooks actual_hooks ' +# NEEDSWORK: when 'git hook add' and 'git hook remove' have been added, use that +# instead +test_expect_success 'with succeeding hook (config-based)' ' + test_when_finished "git config --unset hook.pre-commit.command success.sample" && + test_when_finished "rm -f expected_hooks actual_hooks" && + git config hook.pre-commit.command "$HOOKDIR/success.sample" && + echo "$HOOKDIR/success.sample" >expected_hooks && + echo "more" >>file && + git add file && + git commit -m "more" && + test_cmp expected_hooks actual_hooks +' + test_expect_success 'with succeeding hook (merge)' ' test_when_finished "rm -f \"$PREMERGE\" expected_hooks actual_hooks" && cp "$HOOKDIR/success.sample" "$PREMERGE" &&
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
Re: [PATCH v8 18/37] commit: use config-based hooks
Quoted 27 lines · Emily Shaffer
commit.c +10
-5-int run_commit_hook (int editor_is_used,const char *index_file, +int run_commit_hook (int editor_is_used,int parallelize,const char *index_file,const char *name, ...) { -struct strvec hook_env =STRVEC_INIT ; +struct run_hooks_opt opt;va_list args; +const char *arg;int ret; -strvec_pushf (&hook_env,"GIT_INDEX_FILE=%s" , index_file); +run_hooks_opt_init_sync (&opt); + +if (parallelize) + opt.jobs =configured_hook_jobs (); + +strvec_pushf (&opt.env,"GIT_INDEX_FILE=%s" , index_file);/* * Let the hook know that no editor will be launched. */ if (!editor_is_used) -strvec_push (&hook_env,"GIT_EDITOR=:" ); +strvec_push (&opt.env,"GIT_EDITOR=:" );va_start (args, name); - ret =run_hook_ve (hook_env.v, name, args);
commit.c size_t ignore_non_trailer(const char *buf, size_t len) · line 1715
Quoted 10 lines
commit.c +5
-1+ while ((arg = va_arg(args, const char *))) +strvec_push (&opt.args, arg);va_end (args); -strvec_clear (&hook_env); + + ret =run_hooks (name, &opt); +run_hooks_opt_clear (&opt);return ret; }
This follows the textbook pattern established earlier and demonstrated in [v8 08/37]. opt_init() to initialize, populate its members, call run_hooks() and finally opt_clear().
Quite nicely demonstrated.
PATCH v8 19/37 am: convert applypatch hooks to use config
In reply to Emily ShafferTeach pre-applypatch, post-applypatch, and applypatch-msg to use the hook.h library instead of the run-command.h library. This enables use of hooks specified in the config, in addition to those in the hookdir. These three hooks are called only by builtin/am.c.
--- Documentation/githooks.txt | 9 +++++++++ builtin/am.c | 14 +++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-)
Documentation/githooks.txt +9 -0
@@ -58,6 +58,9 @@ the message file. The default 'applypatch-msg' hook, when enabled, runs the 'commit-msg' hook, if the latter is enabled. +Hooks run during 'applypatch-msg' will not be parallelized, because hooks are +expected to edit the file holding the commit log message. + pre-applypatch ~~~~~~~~~~~~~~
@@ -73,6 +76,9 @@ make a commit if it does not pass certain test. The default 'pre-applypatch' hook, when enabled, runs the 'pre-commit' hook, if the latter is enabled. +Hooks run during 'pre-applypatch' will be run in parallel, unless hook.jobs is +configured to 1. + post-applypatch ~~~~~~~~~~~~~~~
@@ -82,6 +88,9 @@ and is invoked after the patch is applied and a commit is made. This hook is meant primarily for notification, and cannot affect the outcome of `git am`. +Hooks run during 'post-applypatch' will be run in parallel, unless hook.jobs is +configured to 1. + pre-commit ~~~~~~~~~~
builtin/am.c +11 -3
@@ -33,6 +33,7 @@#include "string-list.h" #include "packfile.h" #include "repository.h" +#include "hook.h" /** * Returns the length of the first line of msg.
@@ -426,9 +427,13 @@ static void am_destroy(const struct am_state *state)static int run_applypatch_msg_hook (struct am_state *state) {int ret; +struct run_hooks_opt opt; +run_hooks_opt_init_sync (&opt);assert (state->msg); - ret =run_hook_le (NULL ,"applypatch-msg" ,am_path (state,"final-commit" ),NULL ); +strvec_push (&opt.args,am_path (state,"final-commit" )); + ret =run_hooks ("applypatch-msg" , &opt); +run_hooks_opt_clear (&opt);if (!ret) {FREE_AND_NULL (state->msg);
@@ -1558,8 +1563,10 @@ static void do_commit(const struct am_state *state)struct commit_list *parents =NULL ;const char *reflog_msg, *author, *committer =NULL ;struct strbuf sb =STRBUF_INIT ; +struct run_hooks_opt hook_opt; +run_hooks_opt_init_async (&hook_opt); -if (run_hook_le (NULL ,"pre-applypatch" ,NULL )) +if (run_hooks ("pre-applypatch" , &hook_opt))exit (1 );if (write_cache_as_tree (&tree,0 ,NULL ))
@@ -1611,8 +1618,9 @@ static void do_commit(const struct am_state *state)fclose (fp); } -run_hook_le (NULL ,"post-applypatch" ,NULL ); +run_hooks ("post-applypatch" , &hook_opt); +run_hooks_opt_clear (&hook_opt);strbuf_release (&sb); }
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
Re: [PATCH v8 19/37] am: convert applypatch hooks to use config
Quoted 13 lines · Emily Shaffer
builtin/am.c +3
-1@@ -1558,8 +1563,10 @@ static void do_commit(const struct am_state *state)struct commit_list *parents =NULL ;const char *reflog_msg, *author, *committer =NULL ;struct strbuf sb =STRBUF_INIT ; +struct run_hooks_opt hook_opt; +run_hooks_opt_init_async (&hook_opt); -if (run_hook_le (NULL ,"pre-applypatch" ,NULL )) +if (run_hooks ("pre-applypatch" , &hook_opt))exit (1 );if (write_cache_as_tree (&tree,0 ,NULL ))
builtin/am.c static void do_commit(const struct am_state *state) · line 1625
Quoted 9 lines
builtin/am.c +2
-1@@ -1611,8 +1618,9 @@ static void do_commit(const struct am_state *state)fclose (fp); } -run_hook_le (NULL ,"post-applypatch" ,NULL ); +run_hooks ("post-applypatch" , &hook_opt); +run_hooks_opt_clear (&hook_opt);strbuf_release (&sb); }
This one does opt_init(), run_hooks(), and another run_hooks() and then opt_clear(). If run_hooks() is a read-only operation on the hook_opt, then that would be alright, but it just smells iffy that it is not done as two separate opt_init(), run_hooks(), opt_clear() sequences for two separate run_hooks() invocations. The same worry about future safety I meantioned elsewhere in the series also applies.
Thanks.
Re: [PATCH v8 19/37] am: convert applypatch hooks to use config
Quoted 22 lines · Junio C Hamano
Quoted 19 lines · Emily Shaffer
builtin/am.c +4
-2@@ -1558,8 +1563,10 @@ static void do_commit(const struct am_state *state)struct commit_list *parents =NULL ;const char *reflog_msg, *author, *committer =NULL ;struct strbuf sb =STRBUF_INIT ; +struct run_hooks_opt hook_opt; +run_hooks_opt_init_async (&hook_opt); -if (run_hook_le (NULL ,"pre-applypatch" ,NULL )) +if (run_hooks ("pre-applypatch" , &hook_opt))exit (1 );if (write_cache_as_tree (&tree,0 ,NULL ))@@ -1611,8 +1618,9 @@ static void do_commit(const struct am_state *state)fclose (fp); } -run_hook_le (NULL ,"post-applypatch" ,NULL ); +run_hooks ("post-applypatch" , &hook_opt);
Quoted 11 lines
builtin/am.c static void do_commit(const struct am_state *state) · line 1625
Quoted 3 lines
builtin/am.c +1
-0+run_hooks_opt_clear (&hook_opt);strbuf_release (&sb); }This one does
opt_init(),run_hooks(), and anotherrun_hooks()and thenopt_clear(). Ifrun_hooks()is a read-only operation on thehook_opt, then that would be alright, but it just smells iffy that it is not done as two separateopt_init(),run_hooks(),opt_clear()sequences for two separaterun_hooks()invocations. The same worry about future safety I meantioned elsewhere in the series also applies.
Interesting observation. I think the only thing that could be mutated in the run_hooks_opt struct today is the caller-provided callback data (run_hooks_opt.feed_pipe_ctx) - which presumably is being manipulated by the caller in a callback they wrote. But I don't think it hurts particularly to clear/init again between the two invocations, to be safe - so I will change the code here.
- Emily
PATCH v8 20/37 merge: use config-based hooks for post-merge hook
In reply to Emily ShafferTeach post-merge to use the hook.h library instead of the run-command.h library to run hooks. This means that post-merge hooks can come from the config as well as from the hookdir. post-merge is invoked only from builtin/merge.c.
--- Documentation/githooks.txt | 3 +++ builtin/merge.c | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-)
Documentation/githooks.txt +3 -0
@@ -242,6 +242,9 @@ save and restore any form of metadata associated with the working tree (e.g.: permissions/ownership, ACLS, etc). See contrib/hooks/setgitperms.perl for an example of how to do this. +Hooks executed during 'post-merge' will run in parallel, unless hook.jobs is +configured to 1. + pre-push ~~~~~~~~
builtin/merge.c +5 -1
@@ -444,7 +444,9 @@ static void finish(struct commit *head_commit, const struct object_id *new_head, const char *msg) { struct strbuf reflog_message = STRBUF_INIT; +struct run_hooks_opt opt;const struct object_id *head = &head_commit->object.oid; +run_hooks_opt_init_async (&opt);if (!msg)strbuf_addstr (&reflog_message,getenv ("GIT_REFLOG_ACTION" ));
@@ -485,7 +487,9 @@ static void finish(struct commit *head_commit, }/* Run a post-merge hook */ -run_hook_le (NULL ,"post-merge" , squash ?"1" :"0" ,NULL ); +strvec_push (&opt.args, squash ?"1" :"0" ); +run_hooks ("post-merge" , &opt); +run_hooks_opt_clear (&opt);apply_autostash (git_path_merge_autostash (the_repository));strbuf_release (&reflog_message);
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
PATCH v8 21/37 gc: use hook library for pre-auto-gc hook
In reply to Emily ShafferUsing the hook.h library instead of the run-command.h library to run pre-auto-gc means that those hooks can be set up in config files, as well as in the hookdir. pre-auto-gc is called only from builtin/gc.c.
--- Documentation/githooks.txt | 3 +++ builtin/gc.c | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-)
Documentation/githooks.txt +3 -0
@@ -560,6 +560,9 @@ This hook is invoked by `git gc --auto` (see linkgit:git-gc[1]). It takes no parameter, and exiting with non-zero status from this script causes the `git gc --auto` to abort. +Hooks run during 'pre-auto-gc' will be run in parallel, unless hook.jobs is +configured to 1. + post-rewrite ~~~~~~~~~~~~
builtin/gc.c +4 -1
@@ -32,6 +32,7 @@#include "remote.h" #include "object-store.h" #include "exec-cmd.h" +#include "hook.h" #define FAILED_RUN "failed to run %s"
@@ -348,6 +349,8 @@ static void add_repack_incremental_option(void)static int need_to_gc (void ) { +struct run_hooks_opt hook_opt; +run_hooks_opt_init_async (&hook_opt);/* * Setting gc.auto to 0 or negative can disable the * automatic gc.
@@ -394,7 +397,7 @@ static int need_to_gc(void)else return 0 ; -if (run_hook_le (NULL ,"pre-auto-gc" ,NULL )) +if (run_hooks ("pre-auto-gc" , &hook_opt))return 0 ;return 1 ; }
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
PATCH v8 22/37 rebase: teach pre-rebase to use hook.h
In reply to Emily ShafferBy using hook.h instead of run-command.h to run hooks, pre-rebase hooks can now be specified in the config as well as in the hookdir. pre-rebase is not called anywhere besides builtin/rebase.c.
--- Documentation/githooks.txt | 3 +++ builtin/rebase.c | 9 +++++++-- 2 files changed, 10 insertions(+), 2 deletions(-)
Documentation/githooks.txt +3 -0
@@ -206,6 +206,9 @@ two parameters. The first parameter is the upstream from which the series was forked. The second parameter is the branch being rebased, and is not set when rebasing the current branch. +Hooks executed during 'pre-rebase' will run in parallel, unless hook.jobs is +configured to 1. + post-checkout ~~~~~~~~~~~~~
builtin/rebase.c +7 -2
@@ -28,6 +28,7 @@#include "sequencer.h" #include "rebase-interactive.h" #include "reset.h" +#include "hook.h" #define DEFAULT_REFLOG_ACTION "rebase"
@@ -1318,6 +1319,7 @@ int cmd_rebase(int argc, const char **argv, const char *prefix)char *squash_onto_name =NULL ;int reschedule_failed_exec =-1 ;int allow_preemptive_ff =1 ; +struct run_hooks_opt hook_opt;struct option builtin_rebase_options[] = { OPT_STRING(0, "onto", &options.onto_name, N_("revision"),
@@ -1431,6 +1433,8 @@ int cmd_rebase(int argc, const char **argv, const char *prefix) };int i; +run_hooks_opt_init_async (&hook_opt); +if (argc ==2 && !strcmp (argv[1 ],"-h" ))usage_with_options (builtin_rebase_usage, builtin_rebase_options);
@@ -2032,9 +2036,9 @@ int cmd_rebase(int argc, const char **argv, const char *prefix) }/* If a hook exists, give it a chance to interrupt*/ +strvec_pushl (&hook_opt.args, options.upstream_arg, argc ? argv[0 ] :NULL ,NULL );if (!ok_to_skip_pre_rebase && -run_hook_le (NULL ,"pre-rebase" , options.upstream_arg, - argc ? argv[0 ] :NULL ,NULL )) +run_hooks ("pre-rebase" , &hook_opt))die (_ ("The pre-rebase hook refused to rebase." ));if (options.flags &REBASE_DIFFSTAT ) {
@@ -2114,6 +2118,7 @@ int cmd_rebase(int argc, const char **argv, const char *prefix) ret = !!run_specific_rebase (&options, action); cleanup: +run_hooks_opt_clear (&hook_opt);strbuf_release (&buf);strbuf_release (&revisions);free (options.head_name);
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
Re: [PATCH v8 22/37] rebase: teach pre-rebase to use hook.h
Quoted 18 lines · Emily Shaffer
builtin/rebase.c +3
-0@@ -1318,6 +1319,7 @@ int cmd_rebase(int argc, const char **argv, const char *prefix)char *squash_onto_name =NULL ;int reschedule_failed_exec =-1 ;int allow_preemptive_ff =1 ; +struct run_hooks_opt hook_opt;struct option builtin_rebase_options[] = { OPT_STRING(0, "onto", &options.onto_name, N_("revision"),@@ -1431,6 +1433,8 @@ int cmd_rebase(int argc, const char **argv, const char *prefix) };int i; +run_hooks_opt_init_async (&hook_opt); +if (argc ==2 && !strcmp (argv[1 ],"-h" ))usage_with_options (builtin_rebase_usage, builtin_rebase_options);
builtin/rebase.c int cmd_rebase(int argc, const char **argv, const char *prefix) · line 2042
Quoted 9 lines
builtin/rebase.c +2
-2@@ -2032,9 +2036,9 @@ int cmd_rebase(int argc, const char **argv, const char *prefix) }/* If a hook exists, give it a chance to interrupt*/ +strvec_pushl (&hook_opt.args, options.upstream_arg, argc ? argv[0 ] :NULL ,NULL );if (!ok_to_skip_pre_rebase && -run_hook_le (NULL ,"pre-rebase" , options.upstream_arg, - argc ? argv[0 ] :NULL ,NULL )) +run_hooks ("pre-rebase" , &hook_opt))die (_ ("The pre-rebase hook refused to rebase." ));
This may needlessly populate hook_opt.args even when run_hooks() is not triggered, but that probably is OK. Except for a place or two where we call die(), the exit path from this function after this point all eventually passes ...
builtin/rebase.c int cmd_rebase(int argc, const char **argv, const char *prefix) · line 2120
Quoted 5 lines
builtin/rebase.c +0
-0if (options.flags &REBASE_DIFFSTAT ) {@@ -2114,6 +2118,7 @@ int cmd_rebase(int argc, const char **argv, const char *prefix) ret = !!run_specific_rebase (&options, action); cleanup:
... this label, so everybody calls opt_clear() at the end, which is good.
Quoted 4 lines
builtin/rebase.c +1
-0+run_hooks_opt_clear (&hook_opt);strbuf_release (&buf);strbuf_release (&revisions);free (options.head_name);
PATCH v8 23/37 read-cache: convert post-index-change hook to use config
In reply to Emily ShafferBy using hook.h instead of run-command.h to run, post-index-change hooks can now be specified in the config in addition to the hookdir. post-index-change is not run anywhere besides in read-cache.c.
--- Documentation/githooks.txt | 3 +++ read-cache.c | 13 ++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-)
Documentation/githooks.txt +3 -0
@@ -720,6 +720,9 @@ and "0" meaning they were not. Only one parameter should be set to "1" when the hook runs. The hook running passing "1", "1" should not be possible. +Hooks run during 'post-index-change' will be run in parallel, unless hook.jobs +is configured to 1. + GIT --- Part of the linkgit:git[1] suite
read-cache.c +10 -3
@@ -25,6 +25,7 @@#include "fsmonitor.h" #include "thread-utils.h" #include "progress.h" +#include "hook.h" /* Mask for the name length in ce_flags in the on-disk index */
@@ -3070,6 +3071,8 @@ static int do_write_locked_index(struct index_state *istate, struct lock_file *l unsigned flags) {int ret; +struct run_hooks_opt hook_opt; +run_hooks_opt_init_async (&hook_opt);/* * TODO trace2: replace "the_repository" with the actual repo instance
@@ -3088,9 +3091,13 @@ static int do_write_locked_index(struct index_state *istate, struct lock_file *lelse ret =close_lock_file_gently (lock); -run_hook_le (NULL ,"post-index-change" , - istate->updated_workdir ?"1" :"0" , - istate->updated_skipworktree ?"1" :"0" ,NULL ); +strvec_pushl (&hook_opt.args, + istate->updated_workdir ?"1" :"0" , + istate->updated_skipworktree ?"1" :"0" , +NULL ); +run_hooks ("post-index-change" , &hook_opt); +run_hooks_opt_clear (&hook_opt); + istate->updated_workdir =0 ; istate->updated_skipworktree =0 ;
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
Re: [PATCH v8 23/37] read-cache: convert post-index-change hook to use config
read-cache.c static int do_write_locked_index(struct index_state *istate, struct lock_file *l · line 3076
Quoted 7 lines · Emily Shaffer
read-cache.c +2
-0@@ -3070,6 +3071,8 @@ static int do_write_locked_index(struct index_state *istate, struct lock_file *l unsigned flags) {int ret; +struct run_hooks_opt hook_opt; +run_hooks_opt_init_async (&hook_opt);
Nit. blank line between the last of decls and the first stmt (many identical nits exist everywhere in this series).
Quoted 5 lines
read-cache.c +0
-0/* * TODO trace2: replace "the_repository" with the actual repo instance @@ -3088,9 +3091,13 @@ static int do_write_locked_index(selse ret =close_lock_file_gently (lock);
read-cache.c static int do_write_locked_index(s · line 3099
Quoted 10 lines
read-cache.c +6
-3-run_hook_le (NULL ,"post-index-change" , - istate->updated_workdir ?"1" :"0" , - istate->updated_skipworktree ?"1" :"0" ,NULL ); +strvec_pushl (&hook_opt.args, + istate->updated_workdir ?"1" :"0" , + istate->updated_skipworktree ?"1" :"0" , +NULL ); +run_hooks ("post-index-change" , &hook_opt); +run_hooks_opt_clear (&hook_opt);
There is one early return before the precontext of this hunk that bypasses this opt_clear() call. It is before any member of hook_opt structure that was opt_init()'ed gets touched, so with the current code, there is no leak, but it probably is laying a landmine for the future, where opt_init() may allocate some resource to its member, with the expectation that all users of the API would call opt_clear() to release. Or the caller of the API (like this one) may start mucking with the opt structure before the existing early return, at which point the current assumption that it is safe to return from that point without opt_clear() would be broken.
I saw that there are other early returns in the series that are safe right now but may become unsafe when the API implementation gets extended that way. If it does not involve too much code churning, we may want to restructure the code to make these early returns into "goto"s that jump to a single exit point, so that we can always match opt_init() with opt_clear(), like the structure of the existing code allowed cmd_rebase() to use the hooks API cleanly in [v8 22/37].
Thanks.
Re: [PATCH v8 23/37] read-cache: convert post-index-change hook to use config
Quoted 41 lines · Junio C Hamano
read-cache.c static int do_write_locked_index(struct index_state *istate, struct lock_file *l · line 3076
Quoted 7 lines · Emily Shaffer
read-cache.c +2
-0@@ -3070,6 +3071,8 @@ static int do_write_locked_index(struct index_state *istate, struct lock_file *l unsigned flags) {int ret; +struct run_hooks_opt hook_opt; +run_hooks_opt_init_async (&hook_opt);Nit. blank line between the last of decls and the first stmt (many identical nits exist everywhere in this series).
Quoted 5 lines
read-cache.c +0
-0/* * TODO trace2: replace "the_repository" with the actual repo instance @@ -3088,9 +3091,13 @@ static int do_write_locked_index(selse ret =close_lock_file_gently (lock);read-cache.c static int do_write_locked_index(s · line 3099
Quoted 10 lines
read-cache.c +6
-3-run_hook_le (NULL ,"post-index-change" , - istate->updated_workdir ?"1" :"0" , - istate->updated_skipworktree ?"1" :"0" ,NULL ); +strvec_pushl (&hook_opt.args, + istate->updated_workdir ?"1" :"0" , + istate->updated_skipworktree ?"1" :"0" , +NULL ); +run_hooks ("post-index-change" , &hook_opt); +run_hooks_opt_clear (&hook_opt);There is one early return before the precontext of this hunk that bypasses this
opt_clear()call. It is before any member ofhook_optstructure that was opt_init()'ed gets touched, so with the current code, there is no leak, but it probably is laying a landmine for the future, whereopt_init()may allocate some resource to its member, with the expectation that all users of the API would callopt_clear()to release. Or the caller of the API (like this one) may start mucking with the opt structure before the existing early return, at which point the current assumption that it is safe to return from that point withoutopt_clear()would be broken.
Quoted 8 lines
I saw that there are other early returns in the series that are safe right now but may become unsafe when the API implementation gets extended that way. If it does not involve too much code churning, we may want to restructure the code to make these early returns into "goto"s that jump to a single exit point, so that we can always match
opt_init()withopt_clear(), like the structure of the existing code allowedcmd_rebase()to use the hooks API cleanly in [v8 22/37].
OK. I'll audit this second half of the series looking for this type of thing and try to clean up/use gotos if appropriate/etc. Thanks for pointing it out.
- Emily
PATCH v8 24/37 receive-pack: convert push-to-checkout hook to hook.h
In reply to Emily ShafferBy using hook.h instead of run-command.h to invoke push-to-checkout, hooks can now be specified in the config as well as in the hookdir. push-to-checkout is not called anywhere but in builtin/receive-pack.c.
--- Documentation/githooks.txt | 1 + builtin/receive-pack.c | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-)
Documentation/githooks.txt +1 -0
@@ -555,6 +555,7 @@ that switches branches while keeping the local changes in the working tree that do not interfere with the difference between the branches. +Hooks executed during 'push-to-checkout' will not be parallelized. pre-auto-gc ~~~~~~~~~~~
builtin/receive-pack.c +12 -4
@@ -29,6 +29,7 @@#include "commit-reach.h" #include "worktree.h" #include "shallow.h" +#include "hook.h" static const char *const receive_pack_usage[] = {N_ ("git receive-pack <git-dir>" ),
@@ -1435,12 +1436,19 @@ static const char *push_to_checkout(unsigned char *hash,struct strvec *env, const char *work_tree) { + struct run_hooks_opt opt; +run_hooks_opt_init_sync (&opt); +strvec_pushf (env,"GIT_WORK_TREE=%s" ,absolute_path (work_tree)); -if (run_hook_le (env->v, push_to_checkout_hook, -hash_to_hex (hash),NULL )) +strvec_pushv (&opt.env, env->v); +strvec_push (&opt.args,hash_to_hex (hash)); +if (run_hooks (push_to_checkout_hook, &opt)) { +run_hooks_opt_clear (&opt);return "push-to-checkout hook declined" ; -else + }else { +run_hooks_opt_clear (&opt);return NULL ; + } }static const char *update_worktree (unsigned char *sha1,const struct worktree *worktree)
@@ -1464,7 +1472,7 @@ static const char *update_worktree(unsigned char *sha1, const struct worktree *wstrvec_pushf (&env,"GIT_DIR=%s" ,absolute_path (git_dir)); -if (!find_hook (push_to_checkout_hook)) +if (!hook_exists (push_to_checkout_hook,HOOKDIR_USE_CONFIG )) retval =push_to_deploy (sha1, &env, work_tree);else retval =push_to_checkout (sha1, &env, work_tree);
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
Re: [PATCH v8 24/37] receive-pack: convert push-to-checkout hook to hook.h
Quoted 11 lines · Emily Shaffer
builtin/receive-pack.c +4
-2@@ -1435,12 +1436,19 @@ static const char *push_to_checkout(unsigned char *hash,struct strvec *env, const char *work_tree) { + struct run_hooks_opt opt; +run_hooks_opt_init_sync (&opt); +strvec_pushf (env,"GIT_WORK_TREE=%s" ,absolute_path (work_tree)); -if (run_hook_le (env->v, push_to_checkout_hook, -hash_to_hex (hash),NULL )) +strvec_pushv (&opt.env, env->v);
builtin/receive-pack.c static const char *push_to_checkout(unsigned char *hash, · line 1452
Quoted 10 lines
builtin/receive-pack.c +6
-1+strvec_push (&opt.args,hash_to_hex (hash)); +if (run_hooks (push_to_checkout_hook, &opt)) { +run_hooks_opt_clear (&opt);return "push-to-checkout hook declined" ; - else + }else { +run_hooks_opt_clear (&opt);return NULL ; + } }
OK, we opt_init(), futz with opt, call run_hooks() and opt_clear() regardless of the outcome from run_hooks(). Narrow-sighted me wonders if it makes the use of the API easier if run_hooks() did the opt_clear() before it returns, but I haven't yet seen enough use at this point to judge.
Thanks.
Re: [PATCH v8 24/37] receive-pack: convert push-to-checkout hook to hook.h
Quoted 22 lines · Junio C Hamano
Quoted 19 lines · Emily Shaffer
builtin/receive-pack.c +9
-3@@ -1435,12 +1436,19 @@ static const char *push_to_checkout(unsigned char *hash,struct strvec *env, const char *work_tree) { + struct run_hooks_opt opt; +run_hooks_opt_init_sync (&opt); +strvec_pushf (env,"GIT_WORK_TREE=%s" ,absolute_path (work_tree)); -if (run_hook_le (env->v, push_to_checkout_hook, -hash_to_hex (hash),NULL )) +strvec_pushv (&opt.env, env->v); +strvec_push (&opt.args,hash_to_hex (hash)); +if (run_hooks (push_to_checkout_hook, &opt)) { +run_hooks_opt_clear (&opt);return "push-to-checkout hook declined" ; -else + }else { +run_hooks_opt_clear (&opt);return NULL ;
Quoted 8 lines
builtin/receive-pack.c static const char *push_to_checkout(unsigned char *hash, · line 1452
Quoted 2 lines
builtin/receive-pack.c +1
-0+ } }OK, we
opt_init(), futz with opt, callrun_hooks()andopt_clear()regardless of the outcome fromrun_hooks(). Narrow-sighted me wonders if it makes the use of the API easier ifrun_hooks()did theopt_clear()before it returns, but I haven't yet seen enough use at this point to judge.
Hrm, is that idiomatic? I guess it would be convenient, and as long as it doesn't touch explicitly caller-managed context pointer it should be safe, but wouldn't it be surprising?
- Emily
Re: [PATCH v8 24/37] receive-pack: convert push-to-checkout hook to hook.h
Quoted 9 lines · Emily Shaffer
Quoted 5 lines
OK, we
opt_init(), futz with opt, callrun_hooks()andopt_clear()regardless of the outcome fromrun_hooks(). Narrow-sighted me wonders if it makes the use of the API easier ifrun_hooks()did theopt_clear()before it returns, but I haven't yet seen enough use at this point to judge.Hrm, is that idiomatic? I guess it would be convenient, and as long as it doesn't touch explicitly caller-managed context pointer it should be safe, but wouldn't it be surprising?
The precedent (at this point, I will not judge if it is a good pattern to emulate or an anti-pattern to stay away from) I had in mind was the run_command() which clears child_process structure as the side effect of internally calling finish_command().
Leaving them separate is of course more flexible, but depending on how small we can keep down the number of call patterns of this new API, always having to clear after run might become an unnecessary source of leaks. When I gave that comment, I didn't have enough input to decide, and now it has been so long since I gave my reviews, I do not quite remember what my impression after reading all the patches through was X-<.
PATCH v8 25/37 git-p4: use 'git hook' to run hooks
In reply to Emily ShafferInstead of duplicating the behavior of run-command.h:run_hook_le() in Python, we can directly call 'git hook run'. As a bonus, this means git-p4 learns how to find hook specifications from the Git config as well as from the hookdir.
---
Notes:
Maybe there is a better way to do this - I had a hard time getting this to run locally, and Python is not my forte, so if anybody has a better approach I'd love to just take that patch instead :)
Since v6, removed the developer debug print statements.... :X
Maybe there is a better way to do this - I had a hard time getting this to run locally, and Python is not my forte, so if anybody has a better approach I'd love to just take that patch instead :)
git-p4.py | 67 +++++-------------------------------------------------- 1 file changed, 6 insertions(+), 61 deletions(-)
git-p4.py +6 -61
@@ -208,70 +208,15 @@ def decode_path(path): def run_git_hook(cmd, param=[]): """Execute a hook if the hook exists.""" - if verbose: - sys.stderr.write("Looking for hook: %s\n" % cmd) - sys.stderr.flush() - - hooks_path = gitConfig("core.hooksPath") - if len(hooks_path) <= 0: - hooks_path = os.path.join(os.environ["GIT_DIR"], "hooks") - - if not isinstance(param, list): - param=[param] - - # resolve hook file name, OS depdenent - hook_file = os.path.join(hooks_path, cmd) - if platform.system() == 'Windows': - if not os.path.isfile(hook_file): - # look for the file with an extension - files = glob.glob(hook_file + ".*") - if not files: - return True - files.sort() - hook_file = files.pop() - while hook_file.upper().endswith(".SAMPLE"): - # The file is a sample hook. We don't want it - if len(files) > 0: - hook_file = files.pop() - else: - return True - - if not os.path.isfile(hook_file) or not os.access(hook_file, os.X_OK): + if not cmd: return True - return run_hook_command(hook_file, param) == 0 - -def run_hook_command(cmd, param): - """Executes a git hook command - cmd = the command line file to be executed. This can be - a file that is run by OS association. - - param = a list of parameters to pass to the cmd command - - On windows, the extension is checked to see if it should - be run with the Git for Windows Bash shell. If there - is no file extension, the file is deemed a bash shell - and will be handed off to sh.exe. Otherwise, Windows - will be called with the shell to handle the file assocation. - - For non Windows operating systems, the file is called - as an executable. - """ - cli = [cmd] + param - use_shell = False - if platform.system() == 'Windows': - (root,ext) = os.path.splitext(cmd) - if ext == "": - exe_path = os.environ.get("EXEPATH") - if exe_path is None: - exe_path = "" - else: - exe_path = os.path.join(exe_path, "bin") - cli = [os.path.join(exe_path, "SH.EXE")] + cli - else: - use_shell = True - return subprocess.call(cli, shell=use_shell) + """args are specified with -a <arg> -a <arg> -a <arg>""" + args = (['git', 'hook', 'run'] + + ["-a" + arg for arg in param] + + [cmd]) + return subprocess.call(args) == 0 def write_pipe(c, stdin): if verbose:
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
PATCH v8 26/37 hooks: convert 'post-checkout' hook to hook library
In reply to Emily ShafferBy using the 'hook.h' library, 'post-checkout' hooks can now be specified in the config as well as in the hook directory.
--- Documentation/githooks.txt | 2 ++ builtin/checkout.c | 19 ++++++++++++++----- builtin/clone.c | 8 ++++++-- builtin/worktree.c | 31 +++++++++++++++---------------- reset.c | 16 ++++++++++++---- 5 files changed, 49 insertions(+), 27 deletions(-)
Documentation/githooks.txt +2 -0
@@ -231,6 +231,8 @@ This hook can be used to perform repository validity checks, auto-display differences from the previous HEAD if different, or set working dir metadata properties. +Hooks executed during 'post-checkout' will not be parallelized. + post-merge ~~~~~~~~~~
builtin/checkout.c +14 -5
@@ -9,6 +9,7 @@#include "config.h" #include "diff.h" #include "dir.h" +#include "hook.h" #include "ll-merge.h" #include "lockfile.h" #include "merge-recursive.h"
@@ -104,13 +105,21 @@ struct branch_info {static int post_checkout_hook (struct commit *old_commit,struct commit *new_commit,int changed) { -return run_hook_le (NULL ,"post-checkout" , -oid_to_hex (old_commit ? &old_commit->object.oid : &null_oid), -oid_to_hex (new_commit ? &new_commit->object.oid : &null_oid), - changed ?"1" :"0" ,NULL ); +struct run_hooks_opt opt; +int rc; + +run_hooks_opt_init_sync (&opt); +/* "new_commit" can be NULL when checking out from the index before a commit exists. */ - +strvec_pushl (&opt.args, +oid_to_hex (old_commit ? &old_commit->object.oid : &null_oid), +oid_to_hex (new_commit ? &new_commit->object.oid : &null_oid), + changed ?"1" :"0" , +NULL ); + rc =run_hooks ("post-checkout" , &opt); +run_hooks_opt_clear (&opt); +return rc; } static int update_some(const struct object_id *oid, struct strbuf *base,
builtin/clone.c +6 -2
@@ -32,6 +32,7 @@#include "connected.h" #include "packfile.h" #include "list-objects-filter-options.h" +#include "hook.h" /* * Overall FIXMEs:
@@ -771,6 +772,8 @@ static int checkout(int submodule_progress)struct tree *tree;struct tree_desc t;int err =0 ; +struct run_hooks_opt hook_opt; +run_hooks_opt_init_sync (&hook_opt);if (option_no_checkout)return 0 ;
@@ -816,8 +819,9 @@ static int checkout(int submodule_progress)if (write_locked_index (&the_index, &lock_file,COMMIT_LOCK ))die (_ ("unable to write new index file" )); - err |=run_hook_le (NULL ,"post-checkout" ,oid_to_hex (&null_oid), -oid_to_hex (&oid),"1" ,NULL ); +strvec_pushl (&hook_opt.args,oid_to_hex (&null_oid),oid_to_hex (&oid),"1" ,NULL ); + err |=run_hooks ("post-checkout" , &hook_opt); +run_hooks_opt_clear (&hook_opt);if (!err && (option_recurse_submodules.nr >0 )) {struct strvec args =STRVEC_INIT ;
builtin/worktree.c +15 -16
@@ -13,6 +13,7 @@#include "utf8.h" #include "worktree.h" #include "quote.h" +#include "hook.h" static const char *const worktree_usage[] = {N_ ("git worktree add [<options>] <path> [<commit-ish>]" ),
@@ -383,22 +384,20 @@ static int add_worktree(const char *path, const char *refname,* is_junk is cleared, but do return appropriate code when hook fails. */ if (!ret && opts->checkout) { -const char *hook =find_hook ("post-checkout" ); -if (hook) { -const char *env[] = {"GIT_DIR" ,"GIT_WORK_TREE" ,NULL }; - cp.git_cmd =0 ; - cp.no_stdin =1 ; - cp.stdout_to_stderr =1 ; - cp.dir = path; - cp.env = env; - cp.argv =NULL ; - cp.trace2_hook_name ="post-checkout" ; -strvec_pushl (&cp.args,absolute_path (hook), -oid_to_hex (&null_oid), -oid_to_hex (&commit->object.oid), -"1" ,NULL ); - ret =run_command (&cp); - } +struct run_hooks_opt opt; +run_hooks_opt_init_sync (&opt); + +strvec_pushl (&opt.env,"GIT_DIR" ,"GIT_WORK_TREE" ,NULL ); +strvec_pushl (&opt.args, +oid_to_hex (&null_oid), +oid_to_hex (&commit->object.oid), +"1" , +NULL ); + opt.dir = path; + + ret =run_hooks ("post-checkout" , &opt); + +run_hooks_opt_clear (&opt); }strvec_clear (&child_env);
reset.c +12 -4
@@ -7,6 +7,7 @@#include "tree-walk.h" #include "tree.h" #include "unpack-trees.h" +#include "hook.h" int reset_head(struct repository *r, struct object_id *oid, const char *action, const char *switch_to_branch, unsigned flags,
@@ -126,10 +127,17 @@ int reset_head(struct repository *r, struct object_id *oid, const char *action, ret =create_symref ("HEAD" , switch_to_branch, reflog_head); } -if (run_hook) -run_hook_le (NULL ,"post-checkout" , -oid_to_hex (orig ? orig : &null_oid), -oid_to_hex (oid),"1" ,NULL ); +if (run_hook) { +struct run_hooks_opt opt; +run_hooks_opt_init_sync (&opt); +strvec_pushl (&opt.args, +oid_to_hex (orig ? orig : &null_oid), +oid_to_hex (oid), +"1" , +NULL ); +run_hooks ("post-checkout" , &opt); +run_hooks_opt_clear (&opt); + } leave_reset_head:strbuf_release (&msg);
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
PATCH v8 27/37 hook: convert 'post-rewrite' hook to config
In reply to Emily ShafferBy using 'hook.h' for 'post-rewrite', we simplify hook invocations by not needing to put together our own 'struct child_process' and we also learn to run hooks specified in the config as well as the hook dir.
The signal handling that's being removed by this commit now takes place in run-command.h:run_processes_parallel(), so it is OK to remove them here.
--- Documentation/githooks.txt | 3 ++ builtin/am.c | 19 +++------ sequencer.c | 83 +++++++++++++++++--------------------- 3 files changed, 45 insertions(+), 60 deletions(-)
Documentation/githooks.txt +3 -0
@@ -594,6 +594,9 @@ The hook always runs after the automatic note copying (see "notes.rewrite.<command>" in linkgit:git-config[1]) has happened, and thus has access to these notes. +Hooks run during 'post-rewrite' will be run in parallel, unless hook.jobs is +configured to 1. + The following command-specific comments apply: rebase::
builtin/am.c +6 -13
@@ -450,23 +450,16 @@ static int run_applypatch_msg_hook(struct am_state *state)*/ static int run_post_rewrite_hook (const struct am_state *state) { -struct child_process cp =CHILD_PROCESS_INIT ; -const char *hook =find_hook ("post-rewrite" ); +struct run_hooks_opt opt;int ret; +run_hooks_opt_init_async (&opt); -if (!hook) -return 0 ; - -strvec_push (&cp.args, hook); -strvec_push (&cp.args,"rebase" ); - - cp.in =xopen (am_path (state,"rewritten" ),O_RDONLY ); - cp.stdout_to_stderr =1 ; - cp.trace2_hook_name ="post-rewrite" ; +strvec_push (&opt.args,"rebase" ); + opt.path_to_stdin =am_path (state,"rewritten" ); - ret =run_command (&cp); + ret =run_hooks ("post-rewrite" , &opt); -close (cp.in); +run_hooks_opt_clear (&opt);return ret; }
sequencer.c +36 -47
105 lines not on this page · raw message
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
PATCH v8 28/37 transport: convert pre-push hook to use config
In reply to Emily ShafferBy using the hook.h:run_hooks API, pre-push hooks can be specified in the config as well as in the hookdir.
--- Documentation/githooks.txt | 3 ++ transport.c | 59 +++++++++++--------------------------- 2 files changed, 20 insertions(+), 42 deletions(-)
Documentation/githooks.txt +3 -0
@@ -279,6 +279,9 @@ If this hook exits with a non-zero status, `git push` will abort without pushing anything. Information about why the push is rejected may be sent to the user by writing to standard error. +Hooks executed during 'pre-push' will run in parallel, unless hook.jobs is +configured to 1. + [[pre-receive]] pre-receive ~~~~~~~~~~~
transport.c +17 -42
84 lines not on this page · raw message
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
PATCH v8 29/37 reference-transaction: look for hooks in config
In reply to Emily ShafferBy using the hook.h library, reference-transaction hooks can be specified in the config instead.
The expected output of the test is not fully updated to reflect the absolute path of the hook called because the 'update' hook has not yet been converted to use hook.h.
--- Documentation/githooks.txt | 3 +++ refs.c | 43 +++++++++++++------------------- t/t1416-ref-transaction-hooks.sh | 8 +++--- 3 files changed, 24 insertions(+), 30 deletions(-)
Documentation/githooks.txt +3 -0
@@ -530,6 +530,9 @@ The exit status of the hook is ignored for any state except for the cause the transaction to be aborted. The hook will not be called with "aborted" state in that case. +Hooks run during 'reference-transaction' will be run in parallel, unless +hook.jobs is configured to 1. + push-to-checkout ~~~~~~~~~~~~~~~~
refs.c +17 -26
70 lines not on this page · raw message
t/t1416-ref-transaction-hooks.sh +4 -4
@@ -125,11 +125,11 @@ test_expect_success 'interleaving hook calls succeed' ' cat >expect <<-EOF && hooks/update refs/tags/PRE $ZERO_OID $PRE_OID - hooks/reference-transaction prepared - hooks/reference-transaction committed + $(pwd)/target-repo.git/hooks/reference-transaction prepared + $(pwd)/target-repo.git/hooks/reference-transaction committed hooks/update refs/tags/POST $ZERO_OID $POST_OID - hooks/reference-transaction prepared - hooks/reference-transaction committed + $(pwd)/target-repo.git/hooks/reference-transaction prepared + $(pwd)/target-repo.git/hooks/reference-transaction committed EOF git push ./target-repo.git PRE POST &&
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
PATCH v8 30/37 receive-pack: convert 'update' hook to hook.h
In reply to Emily ShafferBy using hook.h to invoke the 'update' hook, now hooks can be specified in the config in addition to the hookdir.
--- Documentation/githooks.txt | 3 ++ builtin/receive-pack.c | 66 ++++++++++++++++++++++---------- t/t1416-ref-transaction-hooks.sh | 4 +- 3 files changed, 50 insertions(+), 23 deletions(-)
Documentation/githooks.txt +3 -0
@@ -368,6 +368,9 @@ The default 'update' hook, when enabled--and with `hooks.allowunannotated` config option unset or set to false--prevents unannotated tags to be pushed. +Hooks executed during 'update' are run in parallel, unless hook.jobs is +configured to 1. + [[proc-receive]] proc-receive ~~~~~~~~~~~~
builtin/receive-pack.c +45 -21
78 lines not on this page · raw message
t/t1416-ref-transaction-hooks.sh +2 -2
@@ -124,10 +124,10 @@ test_expect_success 'interleaving hook calls succeed' ' EOF cat >expect <<-EOF && - hooks/update refs/tags/PRE $ZERO_OID $PRE_OID + $(pwd)/target-repo.git/hooks/update refs/tags/PRE $ZERO_OID $PRE_OID $(pwd)/target-repo.git/hooks/reference-transaction prepared $(pwd)/target-repo.git/hooks/reference-transaction committed - hooks/update refs/tags/POST $ZERO_OID $POST_OID + $(pwd)/target-repo.git/hooks/update refs/tags/POST $ZERO_OID $POST_OID $(pwd)/target-repo.git/hooks/reference-transaction prepared $(pwd)/target-repo.git/hooks/reference-transaction committed EOF
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
PATCH v8 31/37 proc-receive: acquire hook list from hook.h
In reply to Emily ShafferThe proc-receive hook differs from most other hooks Git invokes because the hook and the parent Git process engage in bidirectional communication via stdin/stdout. This bidirectional communication is unsuitable for multiple hooks, whether they are in series or in parallel, and is incompatible with run-command.h:run_processes_parallel:
- The proc-receive hook is intended to modify the state of the Git repo. From 'git help githooks':
This [proc-receive] hook is responsible for updating the relevant references and reporting the results back to 'receive-pack'.
This prevents parallelization and implies, at least, specific ordering of hook execution.
- The proc-receive hook can reject a push by aborting early with an error code. If a former hook ran through the entire push contents successfully but a later hook rejects some of the push, the repo may be left in a partially-updated (and corrupt) state.
- The callback model of the run_processes_parallel() API is unsuited to the current implementation of proc-receive, which loops through "send-receive-consider" with the child process. proc-receive today relies on stateful communication with the child process, which would be unwieldy to implement with callbacks and saved state.
- Additionally, run_processes_parallel() is designed to collate the output of many child processes into a single output (stderr or callback), and would require significant work to tell the caller which process sent the output, and indeed to collect any output before the child process has exited.
So, rather than using hook.h:run_hooks() to invoke the proc-receive hook, receive-pack.c can learn to ask hook.h:hook_list() for the location of a hook to run. This allows users to configure their proc-receive in a global config for all repos if they want, or a local config if they just don't want to use the hookdir. Because running more than one proc-receive hook doesn't make sense from a repo state perspective, we can explicitly ban configuring more than one proc-receive hook at a time.
If a user wants to globally configure one proc-receive hook for most of their repos, but override that hook in a single repo, they should use 'skip' to manually remove the global hook in their special repo:
~/.gitconfig:
[hook.proc-receive]
command = /usr/bin/usual-proc-receive
~/special-repo/.git/config:
[hookcmd./usr/bin/usual-proc-receive]
skip = true
[hook.proc-receive]
command = /usr/bin/special-proc-receive
--- Documentation/githooks.txt | 4 ++ builtin/receive-pack.c | 33 +++++++++++++++- t/t5411/test-0015-too-many-hooks-error.sh | 47 +++++++++++++++++++++++ 3 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 t/t5411/test-0015-too-many-hooks-error.sh
Documentation/githooks.txt +4 -0
@@ -433,6 +433,10 @@ the input. The exit status of the 'proc-receive' hook only determines the success or failure of the group of commands sent to it, unless atomic push is in use. +It is forbidden to specify more than one hook for 'proc-receive'. If a +globally-configured 'proc-receive' must be overridden, use +'hookcmd.<global-hook>.skip = true' to ignore it. + [[post-receive]] post-receive ~~~~~~~~~~~~
builtin/receive-pack.c +31 -2
@@ -1146,11 +1146,40 @@ static int run_proc_receive_hook(struct command *commands,int version =0 ;int code; - argv[0 ] =find_hook ("proc-receive" ); -if (!argv[0 ]) { +struct strbuf hookname =STRBUF_INIT ; +struct hook *proc_receive =NULL ; +struct list_head *pos, *hooks; + +strbuf_addstr (&hookname,"proc-receive" ); + hooks =hook_list (&hookname); + +list_for_each (pos, hooks) { +if (proc_receive) { +rp_error ("only one 'proc-receive' hook can be specified" ); +return -1 ; + } + proc_receive = list_entry(pos, struct hook, list); +/* check if the hookdir hook should be ignored */ +if (proc_receive->from_hookdir) { +switch (configured_hookdir_opt ()) { +case HOOKDIR_INTERACTIVE : +case HOOKDIR_NO : + proc_receive =NULL ; +break ; +default : +break ; + } + } + + } + +if (!proc_receive) {rp_error ("cannot find hook 'proc-receive'" );return -1 ; } + + + argv[0 ] = proc_receive->command.buf; argv[1 ] =NULL ; proc.argv = argv;
t/t5411/test-0015-too-many-hooks-error.sh new file +47 -0
47 lines not on this page · raw message
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
PATCH v8 32/37 post-update: use hook.h library
In reply to Emily ShafferBy using run_hooks() instead of run_hook_le(), 'post-update' hooks can be specified in the config as well as the hookdir.
--- Documentation/githooks.txt | 3 +++ builtin/receive-pack.c | 27 ++++++++------------------- 2 files changed, 11 insertions(+), 19 deletions(-)
Documentation/githooks.txt +3 -0
@@ -508,6 +508,9 @@ Both standard output and standard error output are forwarded to `git send-pack` on the other end, so you can simply `echo` messages for the user. +Hooks run during 'post-update' will be run in parallel, unless hook.jobs is +configured to 1. + reference-transaction ~~~~~~~~~~~~~~~~~~~~~
builtin/receive-pack.c +8 -19
@@ -1688,33 +1688,22 @@ static const char *update(struct command *cmd, struct shallow_info *si)static void run_update_post_hook (struct command *commands) {struct command *cmd; -struct child_process proc =CHILD_PROCESS_INIT ; -const char *hook; - - hook =find_hook ("post-update" ); -if (!hook) -return ; +struct run_hooks_opt opt; +run_hooks_opt_init_async (&opt);for (cmd = commands; cmd; cmd = cmd->next) {if (cmd->error_string || cmd->did_not_exist)continue ; -if (!proc.args.nr) -strvec_push (&proc.args, hook); -strvec_push (&proc.args, cmd->ref_name); +strvec_push (&opt.args, cmd->ref_name); } -if (!proc.args.nr) +if (!opt.args.nr)return ; - proc.no_stdin =1 ; - proc.stdout_to_stderr =1 ; - proc.err = use_sideband ?-1 :0 ; - proc.trace2_hook_name ="post-update" ; +if (use_sideband) + opt.consume_sideband = hook_output_to_sideband; -if (!start_command (&proc)) { -if (use_sideband) -copy_to_sideband (proc.err,-1 ,NULL ); -finish_command (&proc); - } +run_hooks ("post-update" , &opt); +run_hooks_opt_clear (&opt); } static void check_aliased_update_internal(struct command *cmd,
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
Re: [PATCH v8 32/37] post-update: use hook.h library
Quoted 2 lines · Emily Shaffer
By using
run_hooks()instead ofrun_hook_le(), 'post-update' hooks can be specified in the config as well as the hookdir.
Looking ahead in the series no tests for this, seems like a good thing to have some at least trivial tests for each hook and their config invocation.
Re: [PATCH v8 32/37] post-update: use hook.h library
Quoted 10 lines · Ævar Arnfjörð Bjarmason
Quoted 2 lines · Emily Shaffer
By using
run_hooks()instead ofrun_hook_le(), 'post-update' hooks can be specified in the config as well as the hookdir.Looking ahead in the series no tests for this, seems like a good thing to have some at least trivial tests for each hook and their config invocation.
I'll look through the series and make sure the hooks being converted have at least some test to make sure they worked; I think I checked that for some of the early ones but got lazy :) I'll try and add some config-specified tests too. Thanks, it's on my todo list.
- Emily
PATCH v8 33/37 receive-pack: convert receive hooks to hook.h
In reply to Emily ShafferBy using the hook.h library to run receive hooks, they can be specified in the config as well as in the hookdir.
--- Documentation/githooks.txt | 5 + builtin/receive-pack.c | 199 +++++++++++++++++-------------------- 2 files changed, 97 insertions(+), 107 deletions(-)
Documentation/githooks.txt +5 -0
@@ -323,6 +323,8 @@ will be set to zero, `GIT_PUSH_OPTION_COUNT=0`. See the section on "Quarantine Environment" in linkgit:git-receive-pack[1] for some caveats. +Hooks executed during 'pre-receive' will not be parallelized. + [[update]] update ~~~~~~
@@ -476,6 +478,9 @@ environment variables will not be set. If the client selects to use push options, but doesn't transmit any, the count variable will be set to zero, `GIT_PUSH_OPTION_COUNT=0`. +Hooks executed during 'post-receive' are run in parallel, unless hook.jobs is +configured to 1. + [[post-update]] post-update ~~~~~~~~~~~
builtin/receive-pack.c +92 -107
256 lines not on this page · raw message
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog