bundle: arguments can be read from stdin
PATCH 1/1 bundle: arguments can be read from stdin
From: Jiang Xin <[email protected]>
In order to create an incremental bundle, we need to pass many arguments to let git-bundle ignore some already packed commits. It will be more convenient to pass args via stdin. But the current implementation does not allow us to do this.
This is because args are parsed twice when creating bundle. The first time for parsing args is in compute_and_write_prerequisites() by running git-rev-list command to write prerequisites in bundle file, and stdin is consumed in this step if "--stdin" option is provided for git-bundle. Later nothing can be read from stdin when running setup_revisions() in create_bundle().
Remove the entire compute_and_write_prerequisites() function, and parse the args once by setup_revisions(). The first step for creating a bundle is to write prerequisites ("-" obj-id SP comment LF), but after calling prepare_revision_walk(), the revs.pending is left empty. Following steps could not work properly without data in revs.pending. Therefore, before calling prepare_revision_walk() function, make a copy on revs.pending for later use.
--- bundle.c | 109 ++++++++++++++++++++++------------------ t/t5607-clone-bundle.sh | 4 +- 2 files changed, 61 insertions(+), 52 deletions(-)
bundle.c +59 -50
@@ -338,48 +338,6 @@ static int write_pack_data(int bundle_fd, struct rev_info *revs, struct strvec *return 0 ; } -static int compute_and_write_prerequisites (int bundle_fd, -struct rev_info *revs, -int argc,const char **argv) -{ -struct child_process rls =CHILD_PROCESS_INIT ; -struct strbuf buf =STRBUF_INIT ; -FILE *rls_fout; -int i; - -strvec_pushl (&rls.args, -"rev-list" ,"--boundary" ,"--pretty=oneline" , -NULL ); -for (i =1 ; i < argc; i++) -strvec_push (&rls.args, argv[i]); - rls.out =-1 ; - rls.git_cmd =1 ; -if (start_command (&rls)) -return -1 ; - rls_fout =xfdopen (rls.out,"r" ); -while (strbuf_getwholeline (&buf, rls_fout,'\n' ) !=EOF ) { -struct object_id oid; -if (buf.len >0 && buf.buf[0 ] =='-' ) { -write_or_die (bundle_fd, buf.buf, buf.len); -if (!get_oid_hex (buf.buf +1 , &oid)) { -struct object *object =parse_object_or_die (&oid, - buf.buf); - object->flags |=UNINTERESTING ; -add_pending_object (revs, object, buf.buf); - } - }else if (!get_oid_hex (buf.buf, &oid)) { -struct object *object =parse_object_or_die (&oid, - buf.buf); - object->flags |=SHOWN ; - } - } -strbuf_release (&buf); -fclose (rls_fout); -if (finish_command (&rls)) -return error (_ ("rev-list died" )); -return 0 ; -} -/* * Write out bundle refs based on the tips already * parsed into revs.pending. As a side effect, may
@@ -474,6 +432,38 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs)return ref_count; } +struct bundle_prerequisites_info { +struct object_array *pending; +int fd; +}; + +static void write_bundle_prerequisites (struct commit *commit,void *data) +{ +struct bundle_prerequisites_info *bpi = data; +struct object *object; +struct pretty_print_context ctx = {0 }; +struct strbuf buf =STRBUF_INIT ; + +if (!(commit->object.flags &BOUNDARY )) +return ; +strbuf_addf (&buf,"-%s " ,oid_to_hex (&commit->object.oid)); +write_or_die (bpi->fd, buf.buf, buf.len); + + ctx.fmt =CMIT_FMT_ONELINE ; + ctx.output_encoding =get_log_output_encoding (); +strbuf_reset (&buf); +pretty_print_commit (&ctx, commit, &buf); +strbuf_trim (&buf); + + object = (struct object *)commit; + object->flags |=UNINTERESTING ; +add_object_array_with_path (object, buf.buf, bpi->pending,S_IFINVALID , +NULL ); +strbuf_addch (&buf,'\n' ); +write_or_die (bpi->fd, buf.buf, buf.len); +strbuf_release (&buf); +} +int create_bundle (struct repository *r,const char *path,int argc,const char **argv,struct strvec *pack_options,int version) {
@@ -481,8 +471,10 @@ int create_bundle(struct repository *r, const char *path,int bundle_fd =-1 ;int bundle_to_stdout;int ref_count =0 ; -struct rev_info revs; +struct rev_info revs, revs_copy;int min_version = the_hash_algo == &hash_algos[GIT_HASH_SHA1 ] ?2 :3 ; +struct bundle_prerequisites_info bpi; +int i; bundle_to_stdout = !strcmp (path,"-" );if (bundle_to_stdout)
@@ -512,10 +504,6 @@ int create_bundle(struct repository *r, const char *path, save_commit_buffer =0 ;repo_init_revisions (r, &revs,NULL ); -/* write prerequisites */ -if (compute_and_write_prerequisites (bundle_fd, &revs, argc, argv)) - goto err; - argc =setup_revisions (argc, argv, &revs,NULL );if (argc >1 ) {
@@ -523,16 +511,37 @@ int create_bundle(struct repository *r, const char *path, goto err; } -object_array_remove_duplicates (&revs.pending); +/* save revs.pending in revs_copy for later use */ +memcpy (&revs_copy, &revs,sizeof (revs)); + revs_copy.pending.nr =0 ; + revs_copy.pending.alloc =0 ; + revs_copy.pending.objects =NULL ; +for (i =0 ; i < revs.pending.nr; i++) { +struct object_array_entry *e = revs.pending.objects + i; +if (e) +add_object_array_with_path (e->item, e->name, + &revs_copy.pending, + e->mode, e->path); + } - ref_count =write_bundle_refs (bundle_fd, &revs); +/* write prerequisites */ + revs.boundary =1 ; +if (prepare_revision_walk (&revs)) +die ("revision walk setup failed" ); + bpi.fd = bundle_fd; + bpi.pending = &revs_copy.pending; +traverse_commit_list (&revs, write_bundle_prerequisites,NULL , &bpi); +object_array_remove_duplicates (&revs_copy.pending); + +/* write bundle refs */ + ref_count =write_bundle_refs (bundle_fd, &revs_copy);if (!ref_count)die (_ ("Refusing to create empty bundle." ));else if (ref_count <0 ) goto err;/* write pack */ -if (write_pack_data (bundle_fd, &revs, pack_options)) +if (write_pack_data (bundle_fd, &revs_copy, pack_options)) goto err;if (!bundle_to_stdout) {
t/t5607-clone-bundle.sh +2 -2
@@ -38,13 +38,13 @@ test_expect_success 'die if bundle file cannot be created' ' test_must_fail git bundle create adir --all ' -test_expect_failure 'bundle --stdin' ' +test_expect_success 'bundle --stdin' ' echo master | git bundle create stdin-bundle.bdl --stdin && git ls-remote stdin-bundle.bdl >output && grep master output ' -test_expect_failure 'bundle --stdin <rev-list options>' ' +test_expect_success 'bundle --stdin <rev-list options>' ' echo master | git bundle create hybrid-bundle.bdl --stdin tag && git ls-remote hybrid-bundle.bdl >output && grep master output
Signature · 1 line
2.30.0.1.gade423aef4
Quoted 2 lines · Jiang Xin
Therefore, before calling
prepare_revision_walk()function, make a copy onrevs.pendingfor later use.
The in-core objects pointed by the list elements are shared between the original and the copy, and the object flag bits that are used to control the traversal (like SEEN, SHOWN and BOUNDARY bits) would be smudged during the traversal. So depending on how the "later use" uses the copied list, it may or may not be sufficient to just copy the list.
Apparently, you've tested the updated code well enough to send to the list, so it must be sufficient to make a copy of the list to support the way the updated code uses it, but it is not clear how it is so, only from what is in the proposed log message.
PATCH v2 1/2 bundle: lost objects when removing duplicate pendings
From: Jiang Xin <[email protected]>
git rev-list will list one commit for the following command:
$ git rev-list 'main^!'
<tip-commit-of-main-branch>But providing the same rev-list args to git bundle, fail to create a bundle file.
$ git bundle create - 'main^!'
# v2 git bundle
-<OID> <one-line-message>fatal: Refusing to create empty bundle.
This is because when removing duplicate objects in function object_array_remove_duplicates(), one unique pending object which has the same name is deleted by mistake. The revision arg 'main^!' in the above example is parsed by handle_revision_arg(), and at lease two different objects will be appended to revs.pending, one points to the parent commit of the "main" branch, and the other points to the tip commit of the "main" branch. These two objects have the same name "main". Only one object is left with the name "main" after calling the function object_array_remove_duplicates().
And what's worse, when adding boundary commits into pending list, we use one-line commit message as names, and the arbitory names may surprise git-bundle.
Only comparing objects themselves (".item") is also not good enough, because user may want to create a bundle with two identical objects but with different reference names, such as: "HEAD" and "refs/heads/main".
Add new function contains_object() which compare both the address and the name of the object.
--- object.c | 10 +- t/t6020-bundle-misc.sh | 413 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 419 insertions(+), 4 deletions(-) create mode 100755 t/t6020-bundle-misc.sh
object.c +6 -4
@@ -412,15 +412,16 @@ void object_array_clear(struct object_array *array) }/* -* Return true iff array already contains an entry with name. +* Return true if array already contains an entry. */ -static int contains_name (struct object_array *array,const char *name) +static int contains_object (struct object_array *array, +const struct object *item,const char *name) {unsigned nr = array->nr, i;struct object_array_entry *object = array->objects;for (i =0 ; i < nr; i++, object++) -if (!strcmp (object->name, name)) +if (item == object->item && !strcmp (object->name, name))return 1 ;return 0 ; }
@@ -432,7 +433,8 @@ void object_array_remove_duplicates(struct object_array *array) array->nr =0 ;for (src =0 ; src < nr; src++) { -if (!contains_name (array, objects[src].name)) { +if (!contains_object (array, objects[src].item, + objects[src].name)) {if (src != array->nr) objects[array->nr] = objects[src]; array->nr++;
t/t6020-bundle-misc.sh new file +413 -0
413 lines not on this page · raw message
Signature · 1 line
2.30.0.4.g09ee524cd5
PATCH v2 2/2 bundle: arguments can be read from stdin
In reply to Junio C HamanoFrom: Jiang Xin <[email protected]>
In order to create an incremental bundle, we need to pass many arguments to let git-bundle ignore some already packed commits. It will be more convenient to pass args via stdin. But the current implementation does not allow us to do this.
This is because args are parsed twice when creating bundle. The first time for parsing args is in compute_and_write_prerequisites() by running git-rev-list command to write prerequisites in bundle file, and stdin is consumed in this step if "--stdin" option is provided for git-bundle. Later nothing can be read from stdin when running setup_revisions() in create_bundle().
Remove the entire compute_and_write_prerequisites() function, and parse the args once by setup_revisions(). The first step for creating a bundle is to write prerequisites ("-" obj-id SP comment LF), but after calling prepare_revision_walk(), the revs.pending is left empty. Following steps could not work properly without data in revs.pending. Therefore, before calling prepare_revision_walk() function, make a copy revs_copy from revs for later use. Even though revs_copy and revs share the same objects, but changes on these objects will not change the behavior of function write_bundle_refs() and write_pack_data().
Also add testcases for git bundle in t6020, which read args from stdin.
--- bundle.c | 109 ++++++++++++++++++++++------------------ t/t5607-clone-bundle.sh | 4 +- t/t6020-bundle-misc.sh | 85 +++++++++++++++++++++++++++++-- 3 files changed, 142 insertions(+), 56 deletions(-)
bundle.c +59 -50
@@ -338,48 +338,6 @@ static int write_pack_data(int bundle_fd, struct rev_info *revs, struct strvec *return 0 ; } -static int compute_and_write_prerequisites (int bundle_fd, -struct rev_info *revs, -int argc,const char **argv) -{ -struct child_process rls =CHILD_PROCESS_INIT ; -struct strbuf buf =STRBUF_INIT ; -FILE *rls_fout; -int i; - -strvec_pushl (&rls.args, -"rev-list" ,"--boundary" ,"--pretty=oneline" , -NULL ); -for (i =1 ; i < argc; i++) -strvec_push (&rls.args, argv[i]); - rls.out =-1 ; - rls.git_cmd =1 ; -if (start_command (&rls)) -return -1 ; - rls_fout =xfdopen (rls.out,"r" ); -while (strbuf_getwholeline (&buf, rls_fout,'\n' ) !=EOF ) { -struct object_id oid; -if (buf.len >0 && buf.buf[0 ] =='-' ) { -write_or_die (bundle_fd, buf.buf, buf.len); -if (!get_oid_hex (buf.buf +1 , &oid)) { -struct object *object =parse_object_or_die (&oid, - buf.buf); - object->flags |=UNINTERESTING ; -add_pending_object (revs, object, buf.buf); - } - }else if (!get_oid_hex (buf.buf, &oid)) { -struct object *object =parse_object_or_die (&oid, - buf.buf); - object->flags |=SHOWN ; - } - } -strbuf_release (&buf); -fclose (rls_fout); -if (finish_command (&rls)) -return error (_ ("rev-list died" )); -return 0 ; -} -/* * Write out bundle refs based on the tips already * parsed into revs.pending. As a side effect, may
@@ -474,6 +432,38 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs)return ref_count; } +struct bundle_prerequisites_info { +struct object_array *pending; +int fd; +}; + +static void write_bundle_prerequisites (struct commit *commit,void *data) +{ +struct bundle_prerequisites_info *bpi = data; +struct object *object; +struct pretty_print_context ctx = {0 }; +struct strbuf buf =STRBUF_INIT ; + +if (!(commit->object.flags &BOUNDARY )) +return ; +strbuf_addf (&buf,"-%s " ,oid_to_hex (&commit->object.oid)); +write_or_die (bpi->fd, buf.buf, buf.len); + + ctx.fmt =CMIT_FMT_ONELINE ; + ctx.output_encoding =get_log_output_encoding (); +strbuf_reset (&buf); +pretty_print_commit (&ctx, commit, &buf); +strbuf_trim (&buf); + + object = (struct object *)commit; + object->flags |=UNINTERESTING ; +add_object_array_with_path (object, buf.buf, bpi->pending,S_IFINVALID , +NULL ); +strbuf_addch (&buf,'\n' ); +write_or_die (bpi->fd, buf.buf, buf.len); +strbuf_release (&buf); +} +int create_bundle (struct repository *r,const char *path,int argc,const char **argv,struct strvec *pack_options,int version) {
@@ -481,8 +471,10 @@ int create_bundle(struct repository *r, const char *path,int bundle_fd =-1 ;int bundle_to_stdout;int ref_count =0 ; -struct rev_info revs; +struct rev_info revs, revs_copy;int min_version = the_hash_algo == &hash_algos[GIT_HASH_SHA1 ] ?2 :3 ; +struct bundle_prerequisites_info bpi; +int i; bundle_to_stdout = !strcmp (path,"-" );if (bundle_to_stdout)
@@ -512,10 +504,6 @@ int create_bundle(struct repository *r, const char *path, save_commit_buffer =0 ;repo_init_revisions (r, &revs,NULL ); -/* write prerequisites */ -if (compute_and_write_prerequisites (bundle_fd, &revs, argc, argv)) - goto err; - argc =setup_revisions (argc, argv, &revs,NULL );if (argc >1 ) {
@@ -523,16 +511,37 @@ int create_bundle(struct repository *r, const char *path, goto err; } -object_array_remove_duplicates (&revs.pending); +/* save revs.pending in revs_copy for later use */ +memcpy (&revs_copy, &revs,sizeof (revs)); + revs_copy.pending.nr =0 ; + revs_copy.pending.alloc =0 ; + revs_copy.pending.objects =NULL ; +for (i =0 ; i < revs.pending.nr; i++) { +struct object_array_entry *e = revs.pending.objects + i; +if (e) +add_object_array_with_path (e->item, e->name, + &revs_copy.pending, + e->mode, e->path); + } - ref_count =write_bundle_refs (bundle_fd, &revs); +/* write prerequisites */ + revs.boundary =1 ; +if (prepare_revision_walk (&revs)) +die ("revision walk setup failed" ); + bpi.fd = bundle_fd; + bpi.pending = &revs_copy.pending; +traverse_commit_list (&revs, write_bundle_prerequisites,NULL , &bpi); +object_array_remove_duplicates (&revs_copy.pending); + +/* write bundle refs */ + ref_count =write_bundle_refs (bundle_fd, &revs_copy);if (!ref_count)die (_ ("Refusing to create empty bundle." ));else if (ref_count <0 ) goto err;/* write pack */ -if (write_pack_data (bundle_fd, &revs, pack_options)) +if (write_pack_data (bundle_fd, &revs_copy, pack_options)) goto err;if (!bundle_to_stdout) {
t/t5607-clone-bundle.sh +2 -2
@@ -38,13 +38,13 @@ test_expect_success 'die if bundle file cannot be created' ' test_must_fail git bundle create adir --all ' -test_expect_failure 'bundle --stdin' ' +test_expect_success 'bundle --stdin' ' echo master | git bundle create stdin-bundle.bdl --stdin && git ls-remote stdin-bundle.bdl >output && grep master output ' -test_expect_failure 'bundle --stdin <rev-list options>' ' +test_expect_success 'bundle --stdin <rev-list options>' ' echo master | git bundle create hybrid-bundle.bdl --stdin tag && git ls-remote hybrid-bundle.bdl >output && grep master output
t/t6020-bundle-misc.sh +81 -4
147 lines not on this page · raw message
Signature · 1 line
2.30.0.4.g09ee524cd5
v3 cover improvements for git-bundle
In reply to Junio C HamanoFrom: Jiang Xin <[email protected]>
## Introduce two improvements for git-bundle
+ Commit "bundle: lost objects when removing duplicate pendings", which fixes command like:
$ git bundle create <file> 'master^!'
+ Commits "bundle: arguments can be read from stdin", which add "--stdin" option support for git-bundle, like:
$ git bundle create <file> <input
## Changes of v3
1. Forgot to add shebang in file "t/t6020-bundle-misc.sh", which breaks build and test on Windows.
2. Add more testcases in t6020.
## Range diff of v2...v3
1: ba13820340 ! 1: 9df48434f3 bundle: lost objects when removing duplicate pendings
@@ object.c: void object_array_remove_duplicates(struct object_array *array)
## t/t6020-bundle-misc.sh (new) ##
@@
++#!/bin/sh
++#
++# Copyright (c) 2021 Jiang Xin
++#
++
+test_description='Test git-bundle'
+
+GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
@@ t/t6020-bundle-misc.sh (new)
+. ./test-lib.sh
+
+test_bundle_object_count () {
-+ git verify-pack -v "$1" >verify.out &&
++ bundle=$1 &&
++ pack=${bundle%.bdl}.pack &&
++ convert_bundle_to_pack <"$bundle" >"$pack" &&
++ git index-pack "$pack" &&
++ git verify-pack -v "$pack" >verify.out &&
++ count=$(grep "^$OID_REGEX " verify.out | wc -l) &&
++ test $2 = $count && return 0
++ echo object count for $bundle is $count, not $2
++ return 1
++}
++
++
++test_thin_bundle_object_count () {
++ bundle=$1 &&
++ pack=${bundle%.bdl}.pack &&
++ convert_bundle_to_pack <"$bundle" |
++ test_must_fail git index-pack --stdin "$pack" &&
++ rm -f "$pack" &&
++ convert_bundle_to_pack <"$bundle" |
++ git index-pack --stdin --fix-thin "$pack" &&
++ git verify-pack -v "$pack" >verify.out &&
+ count=$(grep "^$OID_REGEX " verify.out | wc -l) &&
+ test $2 = $count && return 0
-+ echo object count is $count, not $2
++ echo object count for $bundle is $count, not $2
+ return 1
+}
+
@@ t/t6020-bundle-misc.sh (new)
+
+# (C) (D, pull/1/head, topic/1)
+# o --- o
-+# / \ (L, tags/v1)
-+# / \ o (H, topic/2) (M, tags/v2)
-+# / (F) \ / (N, tags/v3)
++# / \ (L)
++# / \ o (H, topic/2) (M, tag:v2)
++# / (F) \ / (N, tag:v3)
+# / o --------- o (G, pull/2/head) o --- o --- o (release)
+# / / \ \ / \
+# o --- o --- o -------- o -- o ------------------ o ------- o --- o (main)
-+# (A) (B) (E) (I) (J) (K) (O) (P)
++# (A) (B) (E, tag:v1) (I) (J) (K) (O) (P)
+#
+test_expect_success 'setup' '
+ # commit A & B
@@ t/t6020-bundle-misc.sh (new)
+ test_tick &&
+ git commit -m "Commit E" &&
+ E=$(git rev-parse HEAD) &&
++ test_tick &&
++ git tag -m "v1" v1 HEAD &&
++ TAG1=$(git rev-parse refs/tags/v1) &&
+
+ # branch topic/2
+ git checkout -b topic/2 &&
@@ t/t6020-bundle-misc.sh (new)
+ git add release.txt &&
+ test_tick &&
+ git commit -m "Commit L" &&
-+ test_tick &&
-+ git tag -m "v1" v1 HEAD &&
+
+ cat >release.txt <<-EOF &&
+ Commit M
@@ t/t6020-bundle-misc.sh (new)
+ L=$(git rev-parse HEAD~2) &&
+ M=$(git rev-parse HEAD~) &&
+ N=$(git rev-parse HEAD) &&
-+ TAG1=$(git rev-parse refs/tags/v1) &&
+ TAG2=$(git rev-parse refs/tags/v2) &&
+ TAG3=$(git rev-parse refs/tags/v3) &&
+
@@ t/t6020-bundle-misc.sh (new)
+ EOF
+ test_i18ncmp expect actual &&
+
-+ convert_bundle_to_pack <special-rev.bdl >special-rev.pack &&
-+ git index-pack special-rev.pack &&
-+ test_bundle_object_count special-rev.pack 3
++ test_bundle_object_count special-rev.bdl 3
++'
++
++test_expect_success 'create bundle with --max-count option' '
++ git bundle create max-count.bdl --max-count 1 \
++ main \
++ "^release" \
++ refs/tags/v1 \
++ refs/pull/1/head \
++ refs/pull/2/head &&
++
++ git bundle list-heads max-count.bdl |
++ make_user_friendly_and_stable_output >actual &&
++ cat >expect <<-EOF &&
++ <COMMIT-P> refs/heads/main
++ <TAG-1> refs/tags/v1
++ EOF
++ test_i18ncmp expect actual &&
++
++ git bundle verify max-count.bdl |
++ make_user_friendly_and_stable_output >actual &&
++ cat >expect <<-EOF &&
++ The bundle contains these 2 refs:
++ <COMMIT-P> refs/heads/main
++ <TAG-1> refs/tags/v1
++ The bundle requires this ref:
++ <COMMIT-O>
++ EOF
++ test_i18ncmp expect actual &&
++
++ test_bundle_object_count max-count.bdl 4
++'
++
++test_expect_success 'create bundle with --since option' '
++ git bundle create since.bdl \
++ --since "Thu Apr 7 15:26:13 2005 -0700" \
++ --all &&
++
++ git bundle list-heads since.bdl |
++ make_user_friendly_and_stable_output >actual &&
++ cat >expect <<-EOF &&
++ <COMMIT-P> refs/heads/main
++ <COMMIT-N> refs/heads/release
++ <TAG-2> refs/tags/v2
++ <TAG-3> refs/tags/v3
++ <COMMIT-P> HEAD
++ EOF
++ test_i18ncmp expect actual &&
++
++ git bundle verify since.bdl |
++ make_user_friendly_and_stable_output >actual &&
++ cat >expect <<-EOF &&
++ The bundle contains these 5 refs:
++ <COMMIT-P> refs/heads/main
++ <COMMIT-N> refs/heads/release
++ <TAG-2> refs/tags/v2
++ <TAG-3> refs/tags/v3
++ <COMMIT-P> HEAD
++ The bundle requires these 2 refs:
++ <COMMIT-L>
++ <COMMIT-K>
++ EOF
++ test_i18ncmp expect actual &&
++
++ test_thin_bundle_object_count since.bdl 16
+'
+
+test_expect_success 'create bundle 1 - no prerequisites' '
-+ # create bundle from args
+ git bundle create 1.bdl topic/1 topic/2 &&
+
+ cat >expect <<-EOF &&
@@ t/t6020-bundle-misc.sh (new)
+ make_user_friendly_and_stable_output >actual &&
+ test_i18ncmp expect actual &&
+
-+ convert_bundle_to_pack <1.bdl >1.pack &&
-+ git index-pack 1.pack &&
-+ test_bundle_object_count 1.pack 24
++ test_bundle_object_count 1.bdl 24
+'
+
+test_expect_success 'create bundle 2 - has prerequisites' '
-+ # create bundle from args
+ git bundle create 2.bdl \
+ --ignore-missing \
+ ^topic/deleted \
@@ t/t6020-bundle-misc.sh (new)
+ make_user_friendly_and_stable_output >actual &&
+ test_i18ncmp expect actual &&
+
-+ convert_bundle_to_pack <2.bdl >2.pack &&
-+ git index-pack 2.pack &&
-+ test_bundle_object_count 2.pack 16
++ test_bundle_object_count 2.bdl 16
+'
+
+test_expect_success 'fail to verify bundle without prerequisites' '
@@ t/t6020-bundle-misc.sh (new)
+'
+
+test_expect_success 'create bundle 3 - two refs, same object' '
-+ # create bundle from args
+ git bundle create --version=3 3.bdl \
+ ^release \
+ ^topic/1 \
@@ t/t6020-bundle-misc.sh (new)
+ make_user_friendly_and_stable_output >actual &&
+ test_i18ncmp expect actual &&
+
-+ convert_bundle_to_pack <3.bdl >3.pack &&
-+ git index-pack 3.pack &&
-+ test_bundle_object_count 3.pack 4
++ test_bundle_object_count 3.bdl 4
+'
+
+test_expect_success 'create bundle 4 - with tags' '
-+ # create bundle from args
+ git bundle create 4.bdl \
+ ^main \
+ ^release \
@@ t/t6020-bundle-misc.sh (new)
+ make_user_friendly_and_stable_output >actual &&
+ test_i18ncmp expect actual &&
+
-+ convert_bundle_to_pack <4.bdl >4.pack &&
-+ git index-pack 4.pack &&
-+ test_bundle_object_count 4.pack 3
++ test_bundle_object_count 4.bdl 3
+'
+
+test_expect_success 'clone from bundle' '2: a4662f44a8 ! 2: 86ad41e4d4 bundle: arguments can be read from stdin
@@ Commit message
git-bundle. Later nothing can be read from stdin when running setup_revisions() in create_bundle().
- Remove the entire compute_and_write_prerequisites() function, and
- parse the args once by setup_revisions(). The first step for creating
- a bundle is to write prerequisites ("-" obj-id SP comment LF), but after
- calling prepare_revision_walk(), the revs.pending is left empty.
- Following steps could not work properly without data in revs.pending.
- Therefore, before calling prepare_revision_walk() function, make a
- copy revs_copy from revs for later use. Even though revs_copy and
- revs share the same objects, but changes on these objects will not
- change the behavior of function write_bundle_refs() and
- write_pack_data().
+ The solution is to parse args once by removing the entire function
+ compute_and_write_prerequisites() and then calling function
+ setup_revisions(). In order to write prerequisites for bundle, will
+ call prepare_revision_walk() and traverse_commit_list(). But after
+ calling prepare_revision_walk(), the object array revs.pending is
+ left empty, and the following steps could not work properly with the
+ empty object array (revs.pending). Therefore, make a copy of revs
+ to revs_copy for later use right after calling setup_revisions().
+
+ The copy of revs_copy is not a deep copy, it shares the same objects
+ with revs. The object array of revs has been cleared, but objects
+ themselves are still kept. Flags of objects may change after calling
+ prepare_revision_walk(), we can use these changed flags without
+ calling the git rev-list command and parsing its output like the
+ former implementation.
Also add testcases for git bundle in t6020, which read args from stdin.
@@ bundle.c: static int write_pack_data(int bundle_fd, struct rev_info *revs, struc
/*
* Write out bundle refs based on the tips already
* parsed into revs.pending. As a side effect, may
+@@ bundle.c: static int write_bundle_refs(int bundle_fd, struct rev_info *revs)
+ * constraints.
+ */
+ if (!(e->item->flags & SHOWN) && e->item->type == OBJ_COMMIT) {
+- warning(_("ref '%s' is excluded by the rev-list options"),
++ warning(_("ref '%s' is excluded by the limiting options"),
+ e->name);
+ goto skip_write_ref;
+ }
@@ bundle.c: static int write_bundle_refs(int bundle_fd, struct rev_info *revs)
return ref_count;
}
@@ t/t5607-clone-bundle.sh: test_expect_success 'die if bundle file cannot be creat
grep master output
## t/t6020-bundle-misc.sh ##
-@@ t/t6020-bundle-misc.sh: test_expect_success 'create bundle 1 - no prerequisites' '
- # create bundle from args
+@@ t/t6020-bundle-misc.sh: test_expect_success 'create bundle with --since option' '
+ '
+
+ test_expect_success 'create bundle 1 - no prerequisites' '
++ # create bundle from args
git bundle create 1.bdl topic/1 topic/2 &&
+ # create bundle from stdin
@@ t/t6020-bundle-misc.sh: test_expect_success 'create bundle 1 - no prerequisites'
make_user_friendly_and_stable_output >actual &&
test_i18ncmp expect actual &&
+- test_bundle_object_count 1.bdl 24
+ git bundle verify stdin-1.bdl |
+ make_user_friendly_and_stable_output >actual &&
+ test_i18ncmp expect actual &&
+
- convert_bundle_to_pack <1.bdl >1.pack &&
- git index-pack 1.pack &&
-- test_bundle_object_count 1.pack 24
-+ test_bundle_object_count 1.pack 24 &&
-+
-+ convert_bundle_to_pack <stdin-1.bdl >stdin-1.pack &&
-+ git index-pack stdin-1.pack &&
-+ test_bundle_object_count stdin-1.pack 24
++ test_bundle_object_count 1.bdl 24 &&
++ test_bundle_object_count stdin-1.bdl 24
'
test_expect_success 'create bundle 2 - has prerequisites' '
++ # create bundle from args
+ git bundle create 2.bdl \
+ --ignore-missing \
+ ^topic/deleted \
@@ t/t6020-bundle-misc.sh: test_expect_success 'create bundle 2 - has prerequisites' '
^topic/2 \
release &&
@@ t/t6020-bundle-misc.sh: test_expect_success 'create bundle 2 - has prerequisites
make_user_friendly_and_stable_output >actual &&
test_i18ncmp expect actual &&
+- test_bundle_object_count 2.bdl 16
+ git bundle verify stdin-2.bdl |
+ make_user_friendly_and_stable_output >actual &&
+ test_i18ncmp expect actual &&
+
- convert_bundle_to_pack <2.bdl >2.pack &&
- git index-pack 2.pack &&
-- test_bundle_object_count 2.pack 16
-+ test_bundle_object_count 2.pack 16 &&
-+
-+ convert_bundle_to_pack <stdin-2.bdl >stdin-2.pack &&
-+ git index-pack stdin-2.pack &&
-+ test_bundle_object_count stdin-2.pack 16
++ test_bundle_object_count 2.bdl 16 &&
++ test_bundle_object_count stdin-2.bdl 16
'
test_expect_success 'fail to verify bundle without prerequisites' '
@@ t/t6020-bundle-misc.sh: test_expect_success 'fail to verify bundle without prere
test_i18ncmp expect actual
'
+ test_expect_success 'create bundle 3 - two refs, same object' '
++ # create bundle from args
+ git bundle create --version=3 3.bdl \
+ ^release \
+ ^topic/1 \
@@ t/t6020-bundle-misc.sh: test_expect_success 'create bundle 3 - two refs, same object' '
main \
HEAD &&
@@ t/t6020-bundle-misc.sh: test_expect_success 'create bundle 3 - two refs, same ob
make_user_friendly_and_stable_output >actual &&
test_i18ncmp expect actual &&
+- test_bundle_object_count 3.bdl 4
+ git bundle verify stdin-3.bdl |
+ make_user_friendly_and_stable_output >actual &&
+ test_i18ncmp expect actual &&
+
- convert_bundle_to_pack <3.bdl >3.pack &&
- git index-pack 3.pack &&
-- test_bundle_object_count 3.pack 4
-+ test_bundle_object_count 3.pack 4 &&
-+
-+ convert_bundle_to_pack <stdin-3.bdl >stdin-3.pack &&
-+ git index-pack stdin-3.pack &&
-+ test_bundle_object_count stdin-3.pack 4
++ test_bundle_object_count 3.bdl 4 &&
++ test_bundle_object_count stdin-3.bdl 4
'
test_expect_success 'create bundle 4 - with tags' '
++ # create bundle from args
+ git bundle create 4.bdl \
+ ^main \
+ ^release \
@@ t/t6020-bundle-misc.sh: test_expect_success 'create bundle 4 - with tags' '
^topic/2 \
--all &&
@@ t/t6020-bundle-misc.sh: test_expect_success 'create bundle 4 - with tags' '
make_user_friendly_and_stable_output >actual &&
test_i18ncmp expect actual &&
+- test_bundle_object_count 4.bdl 3
+ git bundle verify stdin-4.bdl |
+ make_user_friendly_and_stable_output >actual &&
+ test_i18ncmp expect actual &&
+
- convert_bundle_to_pack <4.bdl >4.pack &&
- git index-pack 4.pack &&
-- test_bundle_object_count 4.pack 3
-+ test_bundle_object_count 4.pack 3 &&
-+
-+ convert_bundle_to_pack <stdin-4.bdl >stdin-4.pack &&
-+ git index-pack stdin-4.pack &&
-+ test_bundle_object_count stdin-4.pack 3
++ test_bundle_object_count 4.bdl 3 &&
++ test_bundle_object_count stdin-4.bdl 3
'
test_expect_success 'clone from bundle' '
--
Jiang Xin (2):
bundle: lost objects when removing duplicate pendings
bundle: arguments can be read from stdin
bundle.c | 111 ++++---- object.c | 10 +- t/t5607-clone-bundle.sh | 4 +- t/t6020-bundle-misc.sh | 557 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 625 insertions(+), 57 deletions(-) create mode 100755 t/t6020-bundle-misc.sh
Signature · 1 line
2.30.0.2.g06d2f50715
PATCH v3 1/2 bundle: lost objects when removing duplicate pendings
In reply to Junio C HamanoFrom: Jiang Xin <[email protected]>
git rev-list will list one commit for the following command:
$ git rev-list 'main^!'
<tip-commit-of-main-branch>But providing the same rev-list args to git bundle, fail to create a bundle file.
$ git bundle create - 'main^!'
# v2 git bundle
-<OID> <one-line-message>fatal: Refusing to create empty bundle.
This is because when removing duplicate objects in function object_array_remove_duplicates(), one unique pending object which has the same name is deleted by mistake. The revision arg 'main^!' in the above example is parsed by handle_revision_arg(), and at lease two different objects will be appended to revs.pending, one points to the parent commit of the "main" branch, and the other points to the tip commit of the "main" branch. These two objects have the same name "main". Only one object is left with the name "main" after calling the function object_array_remove_duplicates().
And what's worse, when adding boundary commits into pending list, we use one-line commit message as names, and the arbitory names may surprise git-bundle.
Only comparing objects themselves (".item") is also not good enough, because user may want to create a bundle with two identical objects but with different reference names, such as: "HEAD" and "refs/heads/main".
Add new function contains_object() which compare both the address and the name of the object.
--- object.c | 10 +- t/t6020-bundle-misc.sh | 488 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 494 insertions(+), 4 deletions(-) create mode 100755 t/t6020-bundle-misc.sh
object.c +6 -4
@@ -412,15 +412,16 @@ void object_array_clear(struct object_array *array) }/* -* Return true iff array already contains an entry with name. +* Return true if array already contains an entry. */ -static int contains_name (struct object_array *array,const char *name) +static int contains_object (struct object_array *array, +const struct object *item,const char *name) {unsigned nr = array->nr, i;struct object_array_entry *object = array->objects;for (i =0 ; i < nr; i++, object++) -if (!strcmp (object->name, name)) +if (item == object->item && !strcmp (object->name, name))return 1 ;return 0 ; }
@@ -432,7 +433,8 @@ void object_array_remove_duplicates(struct object_array *array) array->nr =0 ;for (src =0 ; src < nr; src++) { -if (!contains_name (array, objects[src].name)) { +if (!contains_object (array, objects[src].item, + objects[src].name)) {if (src != array->nr) objects[array->nr] = objects[src]; array->nr++;
t/t6020-bundle-misc.sh new file +488 -0
488 lines not on this page · raw message
Signature · 1 line
2.30.0.2.g06d2f50715
PATCH v3 2/2 bundle: arguments can be read from stdin
In reply to Junio C HamanoFrom: Jiang Xin <[email protected]>
In order to create an incremental bundle, we need to pass many arguments to let git-bundle ignore some already packed commits. It will be more convenient to pass args via stdin. But the current implementation does not allow us to do this.
This is because args are parsed twice when creating bundle. The first time for parsing args is in compute_and_write_prerequisites() by running git-rev-list command to write prerequisites in bundle file, and stdin is consumed in this step if "--stdin" option is provided for git-bundle. Later nothing can be read from stdin when running setup_revisions() in create_bundle().
The solution is to parse args once by removing the entire function compute_and_write_prerequisites() and then calling function setup_revisions(). In order to write prerequisites for bundle, will call prepare_revision_walk() and traverse_commit_list(). But after calling prepare_revision_walk(), the object array revs.pending is left empty, and the following steps could not work properly with the empty object array (revs.pending). Therefore, make a copy of revs to revs_copy for later use right after calling setup_revisions().
The copy of revs_copy is not a deep copy, it shares the same objects with revs. The object array of revs has been cleared, but objects themselves are still kept. Flags of objects may change after calling prepare_revision_walk(), we can use these changed flags without calling the git rev-list command and parsing its output like the former implementation.
Also add testcases for git bundle in t6020, which read args from stdin.
--- bundle.c | 111 ++++++++++++++++++++++------------------ t/t5607-clone-bundle.sh | 4 +- t/t6020-bundle-misc.sh | 77 ++++++++++++++++++++++++++-- 3 files changed, 135 insertions(+), 57 deletions(-)
bundle.c +60 -51
@@ -338,48 +338,6 @@ static int write_pack_data(int bundle_fd, struct rev_info *revs, struct strvec *return 0 ; } -static int compute_and_write_prerequisites (int bundle_fd, -struct rev_info *revs, -int argc,const char **argv) -{ -struct child_process rls =CHILD_PROCESS_INIT ; -struct strbuf buf =STRBUF_INIT ; -FILE *rls_fout; -int i; - -strvec_pushl (&rls.args, -"rev-list" ,"--boundary" ,"--pretty=oneline" , -NULL ); -for (i =1 ; i < argc; i++) -strvec_push (&rls.args, argv[i]); - rls.out =-1 ; - rls.git_cmd =1 ; -if (start_command (&rls)) -return -1 ; - rls_fout =xfdopen (rls.out,"r" ); -while (strbuf_getwholeline (&buf, rls_fout,'\n' ) !=EOF ) { -struct object_id oid; -if (buf.len >0 && buf.buf[0 ] =='-' ) { -write_or_die (bundle_fd, buf.buf, buf.len); -if (!get_oid_hex (buf.buf +1 , &oid)) { -struct object *object =parse_object_or_die (&oid, - buf.buf); - object->flags |=UNINTERESTING ; -add_pending_object (revs, object, buf.buf); - } - }else if (!get_oid_hex (buf.buf, &oid)) { -struct object *object =parse_object_or_die (&oid, - buf.buf); - object->flags |=SHOWN ; - } - } -strbuf_release (&buf); -fclose (rls_fout); -if (finish_command (&rls)) -return error (_ ("rev-list died" )); -return 0 ; -} -/* * Write out bundle refs based on the tips already * parsed into revs.pending. As a side effect, may
@@ -425,7 +383,7 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs)* constraints. */ if (!(e->item->flags &SHOWN ) && e->item->type ==OBJ_COMMIT ) { -warning (_ ("ref '%s' is excluded by the rev-list options" ), +warning (_ ("ref '%s' is excluded by the limiting options" ), e->name); goto skip_write_ref; }
@@ -474,6 +432,38 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs)return ref_count; } +struct bundle_prerequisites_info { +struct object_array *pending; +int fd; +}; + +static void write_bundle_prerequisites (struct commit *commit,void *data) +{ +struct bundle_prerequisites_info *bpi = data; +struct object *object; +struct pretty_print_context ctx = {0 }; +struct strbuf buf =STRBUF_INIT ; + +if (!(commit->object.flags &BOUNDARY )) +return ; +strbuf_addf (&buf,"-%s " ,oid_to_hex (&commit->object.oid)); +write_or_die (bpi->fd, buf.buf, buf.len); + + ctx.fmt =CMIT_FMT_ONELINE ; + ctx.output_encoding =get_log_output_encoding (); +strbuf_reset (&buf); +pretty_print_commit (&ctx, commit, &buf); +strbuf_trim (&buf); + + object = (struct object *)commit; + object->flags |=UNINTERESTING ; +add_object_array_with_path (object, buf.buf, bpi->pending,S_IFINVALID , +NULL ); +strbuf_addch (&buf,'\n' ); +write_or_die (bpi->fd, buf.buf, buf.len); +strbuf_release (&buf); +} +int create_bundle (struct repository *r,const char *path,int argc,const char **argv,struct strvec *pack_options,int version) {
@@ -481,8 +471,10 @@ int create_bundle(struct repository *r, const char *path,int bundle_fd =-1 ;int bundle_to_stdout;int ref_count =0 ; -struct rev_info revs; +struct rev_info revs, revs_copy;int min_version = the_hash_algo == &hash_algos[GIT_HASH_SHA1 ] ?2 :3 ; +struct bundle_prerequisites_info bpi; +int i; bundle_to_stdout = !strcmp (path,"-" );if (bundle_to_stdout)
@@ -512,10 +504,6 @@ int create_bundle(struct repository *r, const char *path, save_commit_buffer =0 ;repo_init_revisions (r, &revs,NULL ); -/* write prerequisites */ -if (compute_and_write_prerequisites (bundle_fd, &revs, argc, argv)) - goto err; - argc =setup_revisions (argc, argv, &revs,NULL );if (argc >1 ) {
@@ -523,16 +511,37 @@ int create_bundle(struct repository *r, const char *path, goto err; } -object_array_remove_duplicates (&revs.pending); +/* save revs.pending in revs_copy for later use */ +memcpy (&revs_copy, &revs,sizeof (revs)); + revs_copy.pending.nr =0 ; + revs_copy.pending.alloc =0 ; + revs_copy.pending.objects =NULL ; +for (i =0 ; i < revs.pending.nr; i++) { +struct object_array_entry *e = revs.pending.objects + i; +if (e) +add_object_array_with_path (e->item, e->name, + &revs_copy.pending, + e->mode, e->path); + } - ref_count =write_bundle_refs (bundle_fd, &revs); +/* write prerequisites */ + revs.boundary =1 ; +if (prepare_revision_walk (&revs)) +die ("revision walk setup failed" ); + bpi.fd = bundle_fd; + bpi.pending = &revs_copy.pending; +traverse_commit_list (&revs, write_bundle_prerequisites,NULL , &bpi); +object_array_remove_duplicates (&revs_copy.pending); + +/* write bundle refs */ + ref_count =write_bundle_refs (bundle_fd, &revs_copy);if (!ref_count)die (_ ("Refusing to create empty bundle." ));else if (ref_count <0 ) goto err;/* write pack */ -if (write_pack_data (bundle_fd, &revs, pack_options)) +if (write_pack_data (bundle_fd, &revs_copy, pack_options)) goto err;if (!bundle_to_stdout) {
t/t5607-clone-bundle.sh +2 -2
@@ -38,13 +38,13 @@ test_expect_success 'die if bundle file cannot be created' ' test_must_fail git bundle create adir --all ' -test_expect_failure 'bundle --stdin' ' +test_expect_success 'bundle --stdin' ' echo master | git bundle create stdin-bundle.bdl --stdin && git ls-remote stdin-bundle.bdl >output && grep master output ' -test_expect_failure 'bundle --stdin <rev-list options>' ' +test_expect_success 'bundle --stdin <rev-list options>' ' echo master | git bundle create hybrid-bundle.bdl --stdin tag && git ls-remote hybrid-bundle.bdl >output && grep master output
t/t6020-bundle-misc.sh +73 -4
143 lines not on this page · raw message
Signature · 1 line
2.30.0.2.g06d2f50715
Re: [PATCH v3 1/2] bundle: lost objects when removing duplicate pendings
In reply to Jiang XinQuoted 55 lines · Jiang Xin
From: Jiang Xin <[email protected]>
git rev-listwill list one commit for the following command:$ git rev-list 'main^!' <tip-commit-of-main-branch>But providing the same rev-list args to
git bundle, fail to create a bundle file.$ git bundle create - 'main^!' # v2 git bundle -<OID> <one-line-message>fatal: Refusing to create empty bundle.
This is because when removing duplicate objects in function
object_array_remove_duplicates(), one unique pending object which has the same name is deleted by mistake. The revision arg 'main^!' in the above example is parsed byhandle_revision_arg(), and at lease two different objects will be appended torevs.pending, one points to the parent commit of the "main" branch, and the other points to the tip commit of the "main" branch. These two objects have the same name "main". Only one object is left with the name "main" after calling the functionobject_array_remove_duplicates().And what's worse, when adding boundary commits into pending list, we use one-line commit message as names, and the arbitory names may surprise git-bundle.
Only comparing objects themselves (".item") is also not good enough, because user may want to create a bundle with two identical objects but with different reference names, such as: "HEAD" and "refs/heads/main".
Add new function
contains_object()which compare both the address and the name of the object.Signed-off-by Jiang Xin <[email protected]>--- object.c | 10 +- t/t6020-bundle-misc.sh | 488 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 494 insertions(+), 4 deletions(-) create mode 100755 t/t6020-bundle-misc.shobject.c +1
-1@@ -412,15 +412,16 @@ void object_array_clear(struct object_array *array) }/* -* Return true iff array already contains an entry with name. +* Return true if array already contains an entry. */
object.c void object_array_clear(struct object_array *array) · line 424
Quoted 10 lines
object.c +3
-2-static int contains_name (struct object_array *array,const char *name) +static int contains_object (struct object_array *array, +const struct object *item,const char *name) {unsigned nr = array->nr, i;struct object_array_entry *object = array->objects;for (i =0 ; i < nr; i++, object++) -if (!strcmp (object->name, name)) +if (item == object->item && !strcmp (object->name, name))
I think the comparison of item == object->item is a bit too fragile. Yes, we all know item must be an entry of array.
However, it's separated from its usage may lead to misuse in the future. Perhaps using oidcmp or adding a comment to indicate that item must be an entry of array.
Quoted 28 lines
object.c +2
-1return 1 ;return 0 ; }@@ -432,7 +433,8 @@ void object_array_remove_duplicates(struct object_array *array) array->nr =0 ;for (src =0 ; src < nr; src++) { -if (!contains_name (array, objects[src].name)) { +if (!contains_object (array, objects[src].item, + objects[src].name)) {if (src != array->nr) objects[array->nr] = objects[src]; array->nr++;t/t6020-bundle-misc.sh new file +9
-0@@ -0,0 +1,488 @@ +#!/bin/sh +# +# Copyright (c) 2021 Jiang Xin +# + +test_description='Test git-bundle' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
t/t6020-bundle-misc.sh line 19
Quoted 10 lines
t/t6020-bundle-misc.sh new file +10
-0+ +. ./test-lib.sh + +test_bundle_object_count () { + bundle=$1 && + pack=${bundle%.bdl}.pack && + convert_bundle_to_pack <"$bundle" >"$pack" && + git index-pack "$pack" && + git verify-pack -v "$pack" >verify.out && + count=$(grep "^$OID_REGEX " verify.out | wc -l) &&
I think we can use 'grep -c' instead of grep .. | wc -l.
Or
grep "^$OID_REGEX " verify.out >verify.filtered && test_line_count = $2 verify.filtered
The same comment applied to test_thin_bundle_object_count
Quoted 17 lines
t/t6020-bundle-misc.sh new file +17
-0+ test $2 = $count && return 0 + echo object count for $bundle is $count, not $2 + return 1 +} + + +test_thin_bundle_object_count () { + bundle=$1 && + pack=${bundle%.bdl}.pack && + convert_bundle_to_pack <"$bundle" | + test_must_fail git index-pack --stdin "$pack" && + rm -f "$pack" && + convert_bundle_to_pack <"$bundle" | + git index-pack --stdin --fix-thin "$pack" && + git verify-pack -v "$pack" >verify.out && + count=$(grep "^$OID_REGEX " verify.out | wc -l) && + test $2 = $count && return 0
t/t6020-bundle-misc.sh line 46
Quoted 10 lines
t/t6020-bundle-misc.sh new file +10
-0+ echo object count for $bundle is $count, not $2 + return 1 +} + +convert_bundle_to_pack () { + while read x && test -n "$x" + do + :; + done + cat
I'm not sure what you would expect in this case, but in my experience, I replace this whole block with
sed '1,/^$/d'
also works.
IOW, I would apply this on top of your change:
----8<-----
t/t6020-bundle-misc.sh +5 -13
@@ -16,10 +16,8 @@ test_bundle_object_count () { convert_bundle_to_pack <"$bundle" >"$pack" && git index-pack "$pack" && git verify-pack -v "$pack" >verify.out && - count=$(grep "^$OID_REGEX " verify.out | wc -l) && - test $2 = $count && return 0 - echo object count for $bundle is $count, not $2 - return 1 + grep "^$OID_REGEX " verify.out >verify.filtered && + test_line_count = $2 verify.filtered }
@@ -32,18 +30,12 @@ test_thin_bundle_object_count () { convert_bundle_to_pack <"$bundle" | git index-pack --stdin --fix-thin "$pack" && git verify-pack -v "$pack" >verify.out && - count=$(grep "^$OID_REGEX " verify.out | wc -l) && - test $2 = $count && return 0 - echo object count for $bundle is $count, not $2 - return 1 + grep "^$OID_REGEX " verify.out >verify.filtered && + test_line_count = $2 verify.filtered } convert_bundle_to_pack () { - while read x && test -n "$x" - do - :; - done - cat + sed '1,/^$/d' } # Format the output of git commands to make a user-friendly and stable
----->8-----
For the below change, I haven't checked but I think test_commit should work, no?
-- Danh
Quoted 445 lines
t/t6020-bundle-misc.sh new file +442
-0+} + +# Format the output of git commands to make a user-friendly and stable +# text. We can easily prepare the expect text without having to worry +# about future changes of the commit ID and spaces of the output. +make_user_friendly_and_stable_output () { + sed \ + -e "s/ *\$//" \ + -e "s/$A/<COMMIT-A>/g" \ + -e "s/$B/<COMMIT-B>/g" \ + -e "s/$C/<COMMIT-C>/g" \ + -e "s/$D/<COMMIT-D>/g" \ + -e "s/$E/<COMMIT-E>/g" \ + -e "s/$F/<COMMIT-F>/g" \ + -e "s/$G/<COMMIT-G>/g" \ + -e "s/$H/<COMMIT-H>/g" \ + -e "s/$I/<COMMIT-I>/g" \ + -e "s/$J/<COMMIT-J>/g" \ + -e "s/$K/<COMMIT-K>/g" \ + -e "s/$L/<COMMIT-L>/g" \ + -e "s/$M/<COMMIT-M>/g" \ + -e "s/$N/<COMMIT-N>/g" \ + -e "s/$O/<COMMIT-O>/g" \ + -e "s/$P/<COMMIT-P>/g" \ + -e "s/$(echo $A | cut -c1-7)[0-9a-f]*/<OID-A>/g" \ + -e "s/$(echo $B | cut -c1-7)[0-9a-f]*/<OID-B>/g" \ + -e "s/$(echo $C | cut -c1-7)[0-9a-f]*/<OID-C>/g" \ + -e "s/$(echo $D | cut -c1-7)[0-9a-f]*/<OID-D>/g" \ + -e "s/$(echo $E | cut -c1-7)[0-9a-f]*/<OID-E>/g" \ + -e "s/$(echo $F | cut -c1-7)[0-9a-f]*/<OID-F>/g" \ + -e "s/$(echo $G | cut -c1-7)[0-9a-f]*/<OID-G>/g" \ + -e "s/$(echo $H | cut -c1-7)[0-9a-f]*/<OID-H>/g" \ + -e "s/$(echo $I | cut -c1-7)[0-9a-f]*/<OID-I>/g" \ + -e "s/$(echo $J | cut -c1-7)[0-9a-f]*/<OID-J>/g" \ + -e "s/$(echo $K | cut -c1-7)[0-9a-f]*/<OID-K>/g" \ + -e "s/$(echo $L | cut -c1-7)[0-9a-f]*/<OID-L>/g" \ + -e "s/$(echo $M | cut -c1-7)[0-9a-f]*/<OID-M>/g" \ + -e "s/$(echo $N | cut -c1-7)[0-9a-f]*/<OID-N>/g" \ + -e "s/$(echo $O | cut -c1-7)[0-9a-f]*/<OID-O>/g" \ + -e "s/$(echo $P | cut -c1-7)[0-9a-f]*/<OID-P>/g" \ + -e "s/$TAG1/<TAG-1>/g" \ + -e "s/$TAG2/<TAG-2>/g" \ + -e "s/$TAG3/<TAG-3>/g" \ + -e "s/$(echo $TAG1 | cut -c1-7)[0-9a-f]*/<OID-TAG-1>/g" \ + -e "s/$(echo $TAG2 | cut -c1-7)[0-9a-f]*/<OID-TAG-2>/g" \ + -e "s/$(echo $TAG3 | cut -c1-7)[0-9a-f]*/<OID-TAG-3>/g" \ + -e "s/$ZERO_OID/<ZERO-OID>/g" +} + +# (C) (D, pull/1/head, topic/1) +# o --- o +# / \ (L) +# / \ o (H, topic/2) (M, tag:v2) +# / (F) \ / (N, tag:v3) +# / o --------- o (G, pull/2/head) o --- o --- o (release) +# / / \ \ / \ +# o --- o --- o -------- o -- o ------------------ o ------- o --- o (main) +# (A) (B) (E, tag:v1) (I) (J) (K) (O) (P) +# +test_expect_success 'setup' ' + # commit A & B + cat >main.txt <<-EOF && + Commit A + EOF + git add main.txt && + test_tick && + git commit -m "Commit A" && + + cat >main.txt <<-EOF && + Commit B + EOF + git add main.txt && + test_tick && + git commit -m "Commit B" && + A=$(git rev-parse HEAD~) && + B=$(git rev-parse HEAD) && + + # branch topic/1 + git checkout -b topic/1 && + cat >topic-1.txt <<-EOF && + Commit C + EOF + git add topic-1.txt && + test_tick && + git commit -m "Commit C" && + + cat >topic-1.txt <<-EOF && + Commit D + EOF + git add -u && + test_tick && + git commit -m "Commit D" && + git update-ref refs/pull/1/head HEAD && + C=$(git rev-parse topic/1~) && + D=$(git rev-parse topic/1) && + + # commit E + git checkout main && + cat >main.txt <<-EOF && + Commit E + EOF + git add main.txt && + test_tick && + git commit -m "Commit E" && + E=$(git rev-parse HEAD) && + test_tick && + git tag -m "v1" v1 HEAD && + TAG1=$(git rev-parse refs/tags/v1) && + + # branch topic/2 + git checkout -b topic/2 && + cat >topic-2.txt <<-EOF && + Commit F + EOF + git add topic-2.txt && + test_tick && + git commit -m "Commit F" && + + cat >topic-2.txt <<-EOF && + Commit G + EOF + git add -u && + test_tick && + git commit -m "Commit G" && + git update-ref refs/pull/2/head HEAD && + + cat >topic-2.txt <<-EOF && + Commit H + EOF + git add -u && + test_tick && + git commit -m "Commit H" && + F=$(git rev-parse topic/2~2) && + G=$(git rev-parse topic/2~) && + H=$(git rev-parse topic/2) && + + # merge commit I & J + git checkout main && + test_tick && + git merge --no-ff --no-edit topic/1 && + test_tick && + git merge --no-ff --no-edit refs/pull/2/head && + I=$(git rev-parse HEAD~) && + J=$(git rev-parse HEAD) && + + # commit K + git checkout main && + cat >main.txt <<-EOF && + Commit K + EOF + git add main.txt && + test_tick && + git commit -m "Commit K" && + K=$(git rev-parse HEAD) && + + # branch release + git checkout -b release && + cat >release.txt <<-EOF && + Commit L + EOF + git add release.txt && + test_tick && + git commit -m "Commit L" && + + cat >release.txt <<-EOF && + Commit M + EOF + git add -u && + test_tick && + git commit -m "Commit M" && + test_tick && + git tag -m "v2" v2 HEAD && + + cat >release.txt <<-EOF && + Commit N + EOF + git add -u && + test_tick && + git commit -m "Commit N" && + test_tick && + git tag -m "v3" v3 HEAD && + L=$(git rev-parse HEAD~2) && + M=$(git rev-parse HEAD~) && + N=$(git rev-parse HEAD) && + TAG2=$(git rev-parse refs/tags/v2) && + TAG3=$(git rev-parse refs/tags/v3) && + + # merge commit O + git checkout main && + test_tick && + git merge --no-ff --no-edit tags/v2 && + O=$(git rev-parse HEAD) && + + # commit P + git checkout main && + cat >main.txt <<-EOF && + Commit P + EOF + git add main.txt && + test_tick && + git commit -m "Commit P" && + P=$(git rev-parse HEAD) +' + +test_expect_success 'create bundle from special rev: main^!' ' + git bundle create special-rev.bdl "main^!" && + + git bundle list-heads special-rev.bdl | + make_user_friendly_and_stable_output >actual && + cat >expect <<-EOF && + <COMMIT-P> refs/heads/main + EOF + test_i18ncmp expect actual && + + git bundle verify special-rev.bdl | + make_user_friendly_and_stable_output >actual && + cat >expect <<-EOF && + The bundle contains this ref: + <COMMIT-P> refs/heads/main + The bundle requires this ref: + <COMMIT-O> + EOF + test_i18ncmp expect actual && + + test_bundle_object_count special-rev.bdl 3 +' + +test_expect_success 'create bundle with --max-count option' ' + git bundle create max-count.bdl --max-count 1 \ + main \ + "^release" \ + refs/tags/v1 \ + refs/pull/1/head \ + refs/pull/2/head && + + git bundle list-heads max-count.bdl | + make_user_friendly_and_stable_output >actual && + cat >expect <<-EOF && + <COMMIT-P> refs/heads/main + <TAG-1> refs/tags/v1 + EOF + test_i18ncmp expect actual && + + git bundle verify max-count.bdl | + make_user_friendly_and_stable_output >actual && + cat >expect <<-EOF && + The bundle contains these 2 refs: + <COMMIT-P> refs/heads/main + <TAG-1> refs/tags/v1 + The bundle requires this ref: + <COMMIT-O> + EOF + test_i18ncmp expect actual && + + test_bundle_object_count max-count.bdl 4 +' + +test_expect_success 'create bundle with --since option' ' + git bundle create since.bdl \ + --since "Thu Apr 7 15:26:13 2005 -0700" \ + --all && + + git bundle list-heads since.bdl | + make_user_friendly_and_stable_output >actual && + cat >expect <<-EOF && + <COMMIT-P> refs/heads/main + <COMMIT-N> refs/heads/release + <TAG-2> refs/tags/v2 + <TAG-3> refs/tags/v3 + <COMMIT-P> HEAD + EOF + test_i18ncmp expect actual && + + git bundle verify since.bdl | + make_user_friendly_and_stable_output >actual && + cat >expect <<-EOF && + The bundle contains these 5 refs: + <COMMIT-P> refs/heads/main + <COMMIT-N> refs/heads/release + <TAG-2> refs/tags/v2 + <TAG-3> refs/tags/v3 + <COMMIT-P> HEAD + The bundle requires these 2 refs: + <COMMIT-L> + <COMMIT-K> + EOF + test_i18ncmp expect actual && + + test_thin_bundle_object_count since.bdl 16 +' + +test_expect_success 'create bundle 1 - no prerequisites' ' + git bundle create 1.bdl topic/1 topic/2 && + + cat >expect <<-EOF && + The bundle contains these 2 refs: + <COMMIT-D> refs/heads/topic/1 + <COMMIT-H> refs/heads/topic/2 + The bundle records a complete history. + EOF + + # verify bundle, which has no prerequisites + git bundle verify 1.bdl | + make_user_friendly_and_stable_output >actual && + test_i18ncmp expect actual && + + test_bundle_object_count 1.bdl 24 +' + +test_expect_success 'create bundle 2 - has prerequisites' ' + git bundle create 2.bdl \ + --ignore-missing \ + ^topic/deleted \ + ^$D \ + ^topic/2 \ + release && + + cat >expect <<-EOF && + The bundle contains this ref: + <COMMIT-N> refs/heads/release + The bundle requires these 3 refs: + <COMMIT-D> + <COMMIT-E> + <COMMIT-G> + EOF + + git bundle verify 2.bdl | + make_user_friendly_and_stable_output >actual && + test_i18ncmp expect actual && + + test_bundle_object_count 2.bdl 16 +' + +test_expect_success 'fail to verify bundle without prerequisites' ' + git init --bare test1.git && + + cat >expect <<-EOF && + error: Repository lacks these prerequisite commits: + error: <COMMIT-D> + error: <COMMIT-E> + error: <COMMIT-G> + EOF + + test_must_fail git -C test1.git bundle verify ../2.bdl 2>&1 | + make_user_friendly_and_stable_output >actual && + test_i18ncmp expect actual +' + +test_expect_success 'create bundle 3 - two refs, same object' ' + git bundle create --version=3 3.bdl \ + ^release \ + ^topic/1 \ + ^topic/2 \ + main \ + HEAD && + + cat >expect <<-EOF && + The bundle contains these 2 refs: + <COMMIT-P> refs/heads/main + <COMMIT-P> HEAD + The bundle requires these 2 refs: + <COMMIT-M> + <COMMIT-K> + EOF + + git bundle verify 3.bdl | + make_user_friendly_and_stable_output >actual && + test_i18ncmp expect actual && + + test_bundle_object_count 3.bdl 4 +' + +test_expect_success 'create bundle 4 - with tags' ' + git bundle create 4.bdl \ + ^main \ + ^release \ + ^topic/1 \ + ^topic/2 \ + --all && + + cat >expect <<-EOF && + The bundle contains these 3 refs: + <TAG-1> refs/tags/v1 + <TAG-2> refs/tags/v2 + <TAG-3> refs/tags/v3 + The bundle records a complete history. + EOF + + git bundle verify 4.bdl | + make_user_friendly_and_stable_output >actual && + test_i18ncmp expect actual && + + test_bundle_object_count 4.bdl 3 +' + +test_expect_success 'clone from bundle' ' + git clone --mirror 1.bdl mirror.git && + git -C mirror.git show-ref | + make_user_friendly_and_stable_output >actual && + cat >expect <<-EOF && + <COMMIT-D> refs/heads/topic/1 + <COMMIT-H> refs/heads/topic/2 + EOF + test_cmp expect actual && + + git -C mirror.git fetch ../2.bdl "+refs/*:refs/*" && + git -C mirror.git show-ref | + make_user_friendly_and_stable_output >actual && + cat >expect <<-EOF && + <COMMIT-N> refs/heads/release + <COMMIT-D> refs/heads/topic/1 + <COMMIT-H> refs/heads/topic/2 + EOF + test_cmp expect actual && + + git -C mirror.git fetch ../3.bdl "+refs/*:refs/*" && + git -C mirror.git show-ref | + make_user_friendly_and_stable_output >actual && + cat >expect <<-EOF && + <COMMIT-P> refs/heads/main + <COMMIT-N> refs/heads/release + <COMMIT-D> refs/heads/topic/1 + <COMMIT-H> refs/heads/topic/2 + EOF + test_cmp expect actual && + + git -C mirror.git fetch ../4.bdl "+refs/*:refs/*" && + git -C mirror.git show-ref | + make_user_friendly_and_stable_output >actual && + cat >expect <<-EOF && + <COMMIT-P> refs/heads/main + <COMMIT-N> refs/heads/release + <COMMIT-D> refs/heads/topic/1 + <COMMIT-H> refs/heads/topic/2 + <TAG-1> refs/tags/v1 + <TAG-2> refs/tags/v2 + <TAG-3> refs/tags/v3 + EOF + test_cmp expect actual +' + +test_doneSignature · 1 line
2.30.0.2.g06d2f50715
Signature · 1 line
Danh
Re: [PATCH v3 1/2] bundle: lost objects when removing duplicate pendings
Quoted 8 lines · Đoàn Trần Công Danh <[email protected]> 于2021年1月7日周四 下午11:37写道:
Quoted 8 lines
(quoted diff) +2
-1-static int contains_name(struct object_array *array, const char *name) +static int contains_object(struct object_array *array, + const struct object *item, const char *name) { unsigned nr = array->nr, i; struct object_array_entry *object = array->objects;
Quoted 9 lines
Review
Quoted 3 lines
for (i = 0; i < nr; i++, object++)(quoted diff) +1
-1- if (!strcmp(object->name, name)) + if (item == object->item && !strcmp(object->name, name))I think the comparison of
item == object->itemis a bit too fragile. Yes, we all knowitemmust be an entry of array.
However, it's separated from its usage may lead to misuse in the future. Perhaps usingoidcmpor adding a comment to indicate thatitemmust be an entry ofarray.
You can find the same usage on comparing address of objects in other places:
+ https://github.com/git/git/blob/v2.30.0/bundle.c#L447
+ https://github.com/git/git/blob/v2.30.0/commit.c#L954
Both item and object->item point to address of a shared object in parsed_object_pool of the repository, we can compare two objects by checking address of them safely.
Quoted 9 lines
Review
Quoted 7 lines
(quoted diff) +7
-0+test_bundle_object_count () { + bundle=$1 && + pack=${bundle%.bdl}.pack && + convert_bundle_to_pack <"$bundle" >"$pack" && + git index-pack "$pack" && + git verify-pack -v "$pack" >verify.out && + count=$(grep "^$OID_REGEX " verify.out | wc -l) &&I think we can use 'grep -c' instead of
grep .. | wc -l.
This function is borrowed from t5510-fetch.sh, and will change like this. Thanks.
Quoted 11 lines
Review
Quoted 7 lines
(quoted diff) +7
-0+ +convert_bundle_to_pack () { + while read x && test -n "$x" + do + :; + done + catI'm not sure what you would expect in this case, but in my experience, I replace this whole block with
Quoted 1 line
sed '1,/^$/d'
This function is used to convert bundle file to pack file by strip the header, which has a signature, prerequisites, references. This function is also borrowed from "t5510-fetch.sh".
Quoted 10 lines
For the below change, I haven't checked but I think
test_commitshould work, no?Quoted 9 lines
(quoted diff) +9
-0+test_expect_success 'setup' ' + # commit A & B + cat >main.txt <<-EOF && + Commit A + EOF + git add main.txt && + test_tick && + git commit -m "Commit A" && +
Quoted 8 lines
Review
Quoted 8 lines
(quoted diff) +8
-0+ cat >main.txt <<-EOF && + Commit B + EOF + git add main.txt && + test_tick && + git commit -m "Commit B" && + A=$(git rev-parse HEAD~) && + B=$(git rev-parse HEAD) &&
I should refactor these code, but I forgot. After examine the test_commit function, I have to write a new helper, because it does not meet my needs.
1. It should not create a tag every time.
2. The tag it created is not an annotated tag.
3. I need to store the object id to a variable.
So I write a new helper test_commit_setvar() in next reroll. And make this testcase much simpler.
test_expect_success 'setup' '
# branch main: commit A & B
test_commit_setvar A "Commit A" main.txt &&
test_commit_setvar B "Commit B" main.txt && # branch topic/1: commit C & D, refs/pull/1/head
git checkout -b topic/1 &&
test_commit_setvar C "Commit C" topic-1.txt &&
test_commit_setvar D "Commit D" topic-1.txt &&
git update-ref refs/pull/1/head HEAD && # branch topic/1: commit E, tag v1
git checkout main &&
test_commit_setvar E "Commit E" main.txt &&
test_commit_setvar TAG1 --tag v1 && # branch topic/2: commit F & G, refs/pull/2/head
git checkout -b topic/2 &&
test_commit_setvar F "Commit F" topic-2.txt &&
test_commit_setvar G "Commit G" topic-2.txt &&
git update-ref refs/pull/2/head HEAD &&
test_commit_setvar H "Commit H" topic-2.txt && # branch main: merge commit I & J
git checkout main &&
test_tick &&
test_commit_setvar I --merge topic/1 "Merge commit I" &&
test_commit_setvar J --merge refs/pull/2/head "Merge commit J" && # branch main: commit K
git checkout main &&
test_commit_setvar K "Commit K" main.txt && # branch release:
git checkout -b release &&
test_commit_setvar L "Commit L" release.txt &&
test_commit_setvar M "Commit M" release.txt &&
test_commit_setvar TAG2 --tag v2 &&
test_commit_setvar N "Commit N" release.txt &&
test_commit_setvar TAG3 --tag v3 && # branch main: merge commit O, commit P
git checkout main &&
test_commit_setvar O --merge tags/v2 "Merge commit O" &&
test_commit_setvar P "Commit P" main.txt
'--
Jiang Xin
v4 cover Improvements for git-bundle
In reply to Đoàn Trần Công DanhFrom: Jiang Xin <[email protected]>
## Two improvements for git-bundle
1. Commit "bundle: lost objects when removing duplicate pendings", which fixes command like:
$ git bundle create <file> 'master^!'
2. Commits "bundle: arguments can be read from stdin", which add "--stdin" option support for git-bundle, like:
$ git bundle create <file> <input
## Changes of v4
+ Refactor t6020.
## Range diff of v3...v4
1: 9df48434f3 ! 1: 0abad6486a bundle: lost objects when removing duplicate pendings
@@ t/t6020-bundle-misc.sh (new)
+
+. ./test-lib.sh
+
++# Check count of objects in a bundle file.
++# We can use "--thin" opiton to check thin pack, which must be fixed by
++# command `git-index-pack --fix-thin --stdin`.
+test_bundle_object_count () {
++ thin= &&
++ if test "$1" = "--thin"
++ then
++ thin=yes
++ shift
++ fi &&
++ if test $# -ne 2
++ then
++ echo >&2 "args should be: <bundle> <count>"
++ return 1
++ fi
+ bundle=$1 &&
+ pack=${bundle%.bdl}.pack &&
+ convert_bundle_to_pack <"$bundle" >"$pack" &&
-+ git index-pack "$pack" &&
-+ git verify-pack -v "$pack" >verify.out &&
-+ count=$(grep "^$OID_REGEX " verify.out | wc -l) &&
++ if test -n "$thin"
++ then
++ test_must_fail git index-pack "$pack" &&
++ mv "$pack" "$pack"-thin &&
++ cat "$pack"-thin |
++ git index-pack --stdin --fix-thin "$pack"
++ else
++ git index-pack "$pack"
++ fi &&
++ git verify-pack -v "$pack" >verify.out
++ if test $? -ne 0
++ then
++ echo >&2 "error: fail to convert $bundle to $pack"
++ return 1
++ fi
++ count=$(grep -c "^$OID_REGEX " verify.out) &&
+ test $2 = $count && return 0
-+ echo object count for $bundle is $count, not $2
-+ return 1
-+}
-+
-+
-+test_thin_bundle_object_count () {
-+ bundle=$1 &&
-+ pack=${bundle%.bdl}.pack &&
-+ convert_bundle_to_pack <"$bundle" |
-+ test_must_fail git index-pack --stdin "$pack" &&
-+ rm -f "$pack" &&
-+ convert_bundle_to_pack <"$bundle" |
-+ git index-pack --stdin --fix-thin "$pack" &&
-+ git verify-pack -v "$pack" >verify.out &&
-+ count=$(grep "^$OID_REGEX " verify.out | wc -l) &&
-+ test $2 = $count && return 0
-+ echo object count for $bundle is $count, not $2
++ echo >&2 "error: object count for $bundle is $count, not $2"
+ return 1
+}
+
++# Display the pack data contained in the bundle file, bypassing the
++# header that contains the signature, prerequisites and references.
+convert_bundle_to_pack () {
+ while read x && test -n "$x"
+ do
@@ t/t6020-bundle-misc.sh (new)
+ cat
+}
+
++# Create a commit or tag and set the variable with the object ID.
++test_commit_setvar () {
++ notick= &&
++ signoff= &&
++ indir= &&
++ merge= &&
++ tag= &&
++ var= &&
++ while test $# != 0
++ do
++ case "$1" in
++ --merge)
++ merge=yes
++ ;;
++ --tag)
++ tag=yes
++ ;;
++ --notick)
++ notick=yes
++ ;;
++ --signoff)
++ signoff="$1"
++ ;;
++ -C)
++ indir="$2"
++ shift
++ ;;
++ -*)
++ echo >&2 "error: unknown option $1"
++ return 1
++ ;;
++ *)
++ test -n "$var" && break
++ var=$1
++ ;;
++ esac
++ shift
++ done &&
++ indir=${indir:+"$indir"/} &&
++ if test $# -eq 0
++ then
++ echo >&2 "no args provided"
++ return 1
++ fi &&
++ if test -z "$notick"
++ then
++ test_tick
++ fi &&
++ if test -n "$merge"
++ then
++ git ${indir:+ -C "$indir"} merge --no-edit --no-ff \
++ ${2:+-m "$2"} "$1" &&
++ oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD)
++ elif test -n "$tag"
++ then
++ git ${indir:+ -C "$indir"} tag -m "$1" "$1" &&
++ oid=$(git ${indir:+ -C "$indir"} rev-parse "$1")
++ else
++ file=${2:-"$1.t"} &&
++ echo "${3-$1}" > "$indir$file" &&
++ git ${indir:+ -C "$indir"} add "$file" &&
++ git ${indir:+ -C "$indir"} commit $signoff -m "$1" &&
++ oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD)
++ fi &&
++ eval $var=$oid
++}
++
++
+# Format the output of git commands to make a user-friendly and stable
+# text. We can easily prepare the expect text without having to worry
+# about future changes of the commit ID and spaces of the output.
+make_user_friendly_and_stable_output () {
+ sed \
-+ -e "s/ *\$//" \
-+ -e "s/$A/<COMMIT-A>/g" \
-+ -e "s/$B/<COMMIT-B>/g" \
-+ -e "s/$C/<COMMIT-C>/g" \
-+ -e "s/$D/<COMMIT-D>/g" \
-+ -e "s/$E/<COMMIT-E>/g" \
-+ -e "s/$F/<COMMIT-F>/g" \
-+ -e "s/$G/<COMMIT-G>/g" \
-+ -e "s/$H/<COMMIT-H>/g" \
-+ -e "s/$I/<COMMIT-I>/g" \
-+ -e "s/$J/<COMMIT-J>/g" \
-+ -e "s/$K/<COMMIT-K>/g" \
-+ -e "s/$L/<COMMIT-L>/g" \
-+ -e "s/$M/<COMMIT-M>/g" \
-+ -e "s/$N/<COMMIT-N>/g" \
-+ -e "s/$O/<COMMIT-O>/g" \
-+ -e "s/$P/<COMMIT-P>/g" \
++ -e "s/$A/<COMMIT-A>/" \
++ -e "s/$B/<COMMIT-B>/" \
++ -e "s/$C/<COMMIT-C>/" \
++ -e "s/$D/<COMMIT-D>/" \
++ -e "s/$E/<COMMIT-E>/" \
++ -e "s/$F/<COMMIT-F>/" \
++ -e "s/$G/<COMMIT-G>/" \
++ -e "s/$H/<COMMIT-H>/" \
++ -e "s/$I/<COMMIT-I>/" \
++ -e "s/$J/<COMMIT-J>/" \
++ -e "s/$K/<COMMIT-K>/" \
++ -e "s/$L/<COMMIT-L>/" \
++ -e "s/$M/<COMMIT-M>/" \
++ -e "s/$N/<COMMIT-N>/" \
++ -e "s/$O/<COMMIT-O>/" \
++ -e "s/$P/<COMMIT-P>/" \
++ -e "s/$TAG1/<TAG-1>/" \
++ -e "s/$TAG2/<TAG-2>/" \
++ -e "s/$TAG3/<TAG-3>/" \
+ -e "s/$(echo $A | cut -c1-7)[0-9a-f]*/<OID-A>/g" \
+ -e "s/$(echo $B | cut -c1-7)[0-9a-f]*/<OID-B>/g" \
+ -e "s/$(echo $C | cut -c1-7)[0-9a-f]*/<OID-C>/g" \
@@ t/t6020-bundle-misc.sh (new)
+ -e "s/$(echo $N | cut -c1-7)[0-9a-f]*/<OID-N>/g" \
+ -e "s/$(echo $O | cut -c1-7)[0-9a-f]*/<OID-O>/g" \
+ -e "s/$(echo $P | cut -c1-7)[0-9a-f]*/<OID-P>/g" \
-+ -e "s/$TAG1/<TAG-1>/g" \
-+ -e "s/$TAG2/<TAG-2>/g" \
-+ -e "s/$TAG3/<TAG-3>/g" \
+ -e "s/$(echo $TAG1 | cut -c1-7)[0-9a-f]*/<OID-TAG-1>/g" \
+ -e "s/$(echo $TAG2 | cut -c1-7)[0-9a-f]*/<OID-TAG-2>/g" \
+ -e "s/$(echo $TAG3 | cut -c1-7)[0-9a-f]*/<OID-TAG-3>/g" \
-+ -e "s/$ZERO_OID/<ZERO-OID>/g"
++ -e "s/ *\$//"
+}
+
+# (C) (D, pull/1/head, topic/1)
@@ t/t6020-bundle-misc.sh (new)
+# (A) (B) (E, tag:v1) (I) (J) (K) (O) (P)
+#
+test_expect_success 'setup' '
-+ # commit A & B
-+ cat >main.txt <<-EOF &&
-+ Commit A
-+ EOF
-+ git add main.txt &&
-+ test_tick &&
-+ git commit -m "Commit A" &&
++ # Try to make a stable fixed width for abbreviated commit ID,
++ # this fixed-width oid will be replaced with "<OID>".
++ git config core.abbrev 7 &&
+
-+ cat >main.txt <<-EOF &&
-+ Commit B
-+ EOF
-+ git add main.txt &&
-+ test_tick &&
-+ git commit -m "Commit B" &&
-+ A=$(git rev-parse HEAD~) &&
-+ B=$(git rev-parse HEAD) &&
++ # branch main: commit A & B
++ test_commit_setvar A "Commit A" main.txt &&
++ test_commit_setvar B "Commit B" main.txt &&
+
-+ # branch topic/1
++ # branch topic/1: commit C & D, refs/pull/1/head
+ git checkout -b topic/1 &&
-+ cat >topic-1.txt <<-EOF &&
-+ Commit C
-+ EOF
-+ git add topic-1.txt &&
-+ test_tick &&
-+ git commit -m "Commit C" &&
-+
-+ cat >topic-1.txt <<-EOF &&
-+ Commit D
-+ EOF
-+ git add -u &&
-+ test_tick &&
-+ git commit -m "Commit D" &&
++ test_commit_setvar C "Commit C" topic-1.txt &&
++ test_commit_setvar D "Commit D" topic-1.txt &&
+ git update-ref refs/pull/1/head HEAD &&
-+ C=$(git rev-parse topic/1~) &&
-+ D=$(git rev-parse topic/1) &&
+
-+ # commit E
++ # branch topic/1: commit E, tag v1
+ git checkout main &&
-+ cat >main.txt <<-EOF &&
-+ Commit E
-+ EOF
-+ git add main.txt &&
-+ test_tick &&
-+ git commit -m "Commit E" &&
-+ E=$(git rev-parse HEAD) &&
-+ test_tick &&
-+ git tag -m "v1" v1 HEAD &&
-+ TAG1=$(git rev-parse refs/tags/v1) &&
-+
-+ # branch topic/2
-+ git checkout -b topic/2 &&
-+ cat >topic-2.txt <<-EOF &&
-+ Commit F
-+ EOF
-+ git add topic-2.txt &&
-+ test_tick &&
-+ git commit -m "Commit F" &&
++ test_commit_setvar E "Commit E" main.txt &&
++ test_commit_setvar TAG1 --tag v1 &&
+
-+ cat >topic-2.txt <<-EOF &&
-+ Commit G
-+ EOF
-+ git add -u &&
-+ test_tick &&
-+ git commit -m "Commit G" &&
++ # branch topic/2: commit F & G, refs/pull/2/head
++ git checkout -b topic/2 &&
++ test_commit_setvar F "Commit F" topic-2.txt &&
++ test_commit_setvar G "Commit G" topic-2.txt &&
+ git update-ref refs/pull/2/head HEAD &&
++ test_commit_setvar H "Commit H" topic-2.txt &&
+
-+ cat >topic-2.txt <<-EOF &&
-+ Commit H
-+ EOF
-+ git add -u &&
-+ test_tick &&
-+ git commit -m "Commit H" &&
-+ F=$(git rev-parse topic/2~2) &&
-+ G=$(git rev-parse topic/2~) &&
-+ H=$(git rev-parse topic/2) &&
-+
-+ # merge commit I & J
++ # branch main: merge commit I & J
+ git checkout main &&
-+ test_tick &&
-+ git merge --no-ff --no-edit topic/1 &&
-+ test_tick &&
-+ git merge --no-ff --no-edit refs/pull/2/head &&
-+ I=$(git rev-parse HEAD~) &&
-+ J=$(git rev-parse HEAD) &&
-+
-+ # commit K
++ test_commit_setvar I --merge topic/1 "Merge commit I" &&
++ test_commit_setvar J --merge refs/pull/2/head "Merge commit J" &&
++
++ # branch main: commit K
+ git checkout main &&
-+ cat >main.txt <<-EOF &&
-+ Commit K
-+ EOF
-+ git add main.txt &&
-+ test_tick &&
-+ git commit -m "Commit K" &&
-+ K=$(git rev-parse HEAD) &&
++ test_commit_setvar K "Commit K" main.txt &&
+
-+ # branch release
++ # branch release:
+ git checkout -b release &&
-+ cat >release.txt <<-EOF &&
-+ Commit L
-+ EOF
-+ git add release.txt &&
-+ test_tick &&
-+ git commit -m "Commit L" &&
++ test_commit_setvar L "Commit L" release.txt &&
++ test_commit_setvar M "Commit M" release.txt &&
++ test_commit_setvar TAG2 --tag v2 &&
++ test_commit_setvar N "Commit N" release.txt &&
++ test_commit_setvar TAG3 --tag v3 &&
+
-+ cat >release.txt <<-EOF &&
-+ Commit M
-+ EOF
-+ git add -u &&
-+ test_tick &&
-+ git commit -m "Commit M" &&
-+ test_tick &&
-+ git tag -m "v2" v2 HEAD &&
-+
-+ cat >release.txt <<-EOF &&
-+ Commit N
-+ EOF
-+ git add -u &&
-+ test_tick &&
-+ git commit -m "Commit N" &&
-+ test_tick &&
-+ git tag -m "v3" v3 HEAD &&
-+ L=$(git rev-parse HEAD~2) &&
-+ M=$(git rev-parse HEAD~) &&
-+ N=$(git rev-parse HEAD) &&
-+ TAG2=$(git rev-parse refs/tags/v2) &&
-+ TAG3=$(git rev-parse refs/tags/v3) &&
-+
-+ # merge commit O
-+ git checkout main &&
-+ test_tick &&
-+ git merge --no-ff --no-edit tags/v2 &&
-+ O=$(git rev-parse HEAD) &&
-+
-+ # commit P
++ # branch main: merge commit O, commit P
+ git checkout main &&
-+ cat >main.txt <<-EOF &&
-+ Commit P
-+ EOF
-+ git add main.txt &&
-+ test_tick &&
-+ git commit -m "Commit P" &&
-+ P=$(git rev-parse HEAD)
++ test_commit_setvar O --merge tags/v2 "Merge commit O" &&
++ test_commit_setvar P "Commit P" main.txt
+'
+
+test_expect_success 'create bundle from special rev: main^!' '
@@ t/t6020-bundle-misc.sh (new)
+'
+
+test_expect_success 'create bundle with --since option' '
++ since="Thu Apr 7 15:26:13 2005 -0700" &&
++ git log -1 --pretty="%ad" $M >actual &&
++ echo "$since" >expect &&
++ test_cmp expect actual &&
++
+ git bundle create since.bdl \
-+ --since "Thu Apr 7 15:26:13 2005 -0700" \
-+ --all &&
++ --since "$since" --all &&
+
+ git bundle list-heads since.bdl |
+ make_user_friendly_and_stable_output >actual &&
@@ t/t6020-bundle-misc.sh (new)
+ EOF
+ test_i18ncmp expect actual &&
+
-+ test_thin_bundle_object_count since.bdl 16
++ test_bundle_object_count --thin since.bdl 16
+'
+
+test_expect_success 'create bundle 1 - no prerequisites' '2: 86ad41e4d4 = 2: 48ef4aa44e bundle: arguments can be read from stdin
--
Jiang Xin (2):
bundle: lost objects when removing duplicate pendings
bundle: arguments can be read from stdin
bundle.c | 111 ++++---- object.c | 10 +- t/t5607-clone-bundle.sh | 4 +- t/t6020-bundle-misc.sh | 546 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 614 insertions(+), 57 deletions(-) create mode 100755 t/t6020-bundle-misc.sh
Signature · 1 line
2.30.0.2.g06d2f50715
PATCH v4 1/2 bundle: lost objects when removing duplicate pendings
In reply to Đoàn Trần Công DanhFrom: Jiang Xin <[email protected]>
git rev-list will list one commit for the following command:
$ git rev-list 'main^!'
<tip-commit-of-main-branch>But providing the same rev-list args to git bundle, fail to create a bundle file.
$ git bundle create - 'main^!'
# v2 git bundle
-<OID> <one-line-message>fatal: Refusing to create empty bundle.
This is because when removing duplicate objects in function object_array_remove_duplicates(), one unique pending object which has the same name is deleted by mistake. The revision arg 'main^!' in the above example is parsed by handle_revision_arg(), and at lease two different objects will be appended to revs.pending, one points to the parent commit of the "main" branch, and the other points to the tip commit of the "main" branch. These two objects have the same name "main". Only one object is left with the name "main" after calling the function object_array_remove_duplicates().
And what's worse, when adding boundary commits into pending list, we use one-line commit message as names, and the arbitory names may surprise git-bundle.
Only comparing objects themselves (".item") is also not good enough, because user may want to create a bundle with two identical objects but with different reference names, such as: "HEAD" and "refs/heads/main".
Add new function contains_object() which compare both the address and the name of the object.
--- object.c | 10 +- t/t6020-bundle-misc.sh | 477 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 483 insertions(+), 4 deletions(-) create mode 100755 t/t6020-bundle-misc.sh
object.c +6 -4
@@ -412,15 +412,16 @@ void object_array_clear(struct object_array *array) }/* -* Return true iff array already contains an entry with name. +* Return true if array already contains an entry. */ -static int contains_name (struct object_array *array,const char *name) +static int contains_object (struct object_array *array, +const struct object *item,const char *name) {unsigned nr = array->nr, i;struct object_array_entry *object = array->objects;for (i =0 ; i < nr; i++, object++) -if (!strcmp (object->name, name)) +if (item == object->item && !strcmp (object->name, name))return 1 ;return 0 ; }
@@ -432,7 +433,8 @@ void object_array_remove_duplicates(struct object_array *array) array->nr =0 ;for (src =0 ; src < nr; src++) { -if (!contains_name (array, objects[src].name)) { +if (!contains_object (array, objects[src].item, + objects[src].name)) {if (src != array->nr) objects[array->nr] = objects[src]; array->nr++;
t/t6020-bundle-misc.sh new file +477 -0
477 lines not on this page · raw message
Signature · 1 line
2.30.0.2.g06d2f50715
PATCH v4 2/2 bundle: arguments can be read from stdin
In reply to Đoàn Trần Công DanhFrom: Jiang Xin <[email protected]>
In order to create an incremental bundle, we need to pass many arguments to let git-bundle ignore some already packed commits. It will be more convenient to pass args via stdin. But the current implementation does not allow us to do this.
This is because args are parsed twice when creating bundle. The first time for parsing args is in compute_and_write_prerequisites() by running git-rev-list command to write prerequisites in bundle file, and stdin is consumed in this step if "--stdin" option is provided for git-bundle. Later nothing can be read from stdin when running setup_revisions() in create_bundle().
The solution is to parse args once by removing the entire function compute_and_write_prerequisites() and then calling function setup_revisions(). In order to write prerequisites for bundle, will call prepare_revision_walk() and traverse_commit_list(). But after calling prepare_revision_walk(), the object array revs.pending is left empty, and the following steps could not work properly with the empty object array (revs.pending). Therefore, make a copy of revs to revs_copy for later use right after calling setup_revisions().
The copy of revs_copy is not a deep copy, it shares the same objects with revs. The object array of revs has been cleared, but objects themselves are still kept. Flags of objects may change after calling prepare_revision_walk(), we can use these changed flags without calling the git rev-list command and parsing its output like the former implementation.
Also add testcases for git bundle in t6020, which read args from stdin.
--- bundle.c | 111 ++++++++++++++++++++++------------------ t/t5607-clone-bundle.sh | 4 +- t/t6020-bundle-misc.sh | 77 ++++++++++++++++++++++++++-- 3 files changed, 135 insertions(+), 57 deletions(-)
bundle.c +60 -51
@@ -338,48 +338,6 @@ static int write_pack_data(int bundle_fd, struct rev_info *revs, struct strvec *return 0 ; } -static int compute_and_write_prerequisites (int bundle_fd, -struct rev_info *revs, -int argc,const char **argv) -{ -struct child_process rls =CHILD_PROCESS_INIT ; -struct strbuf buf =STRBUF_INIT ; -FILE *rls_fout; -int i; - -strvec_pushl (&rls.args, -"rev-list" ,"--boundary" ,"--pretty=oneline" , -NULL ); -for (i =1 ; i < argc; i++) -strvec_push (&rls.args, argv[i]); - rls.out =-1 ; - rls.git_cmd =1 ; -if (start_command (&rls)) -return -1 ; - rls_fout =xfdopen (rls.out,"r" ); -while (strbuf_getwholeline (&buf, rls_fout,'\n' ) !=EOF ) { -struct object_id oid; -if (buf.len >0 && buf.buf[0 ] =='-' ) { -write_or_die (bundle_fd, buf.buf, buf.len); -if (!get_oid_hex (buf.buf +1 , &oid)) { -struct object *object =parse_object_or_die (&oid, - buf.buf); - object->flags |=UNINTERESTING ; -add_pending_object (revs, object, buf.buf); - } - }else if (!get_oid_hex (buf.buf, &oid)) { -struct object *object =parse_object_or_die (&oid, - buf.buf); - object->flags |=SHOWN ; - } - } -strbuf_release (&buf); -fclose (rls_fout); -if (finish_command (&rls)) -return error (_ ("rev-list died" )); -return 0 ; -} -/* * Write out bundle refs based on the tips already * parsed into revs.pending. As a side effect, may
@@ -425,7 +383,7 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs)* constraints. */ if (!(e->item->flags &SHOWN ) && e->item->type ==OBJ_COMMIT ) { -warning (_ ("ref '%s' is excluded by the rev-list options" ), +warning (_ ("ref '%s' is excluded by the limiting options" ), e->name); goto skip_write_ref; }
@@ -474,6 +432,38 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs)return ref_count; } +struct bundle_prerequisites_info { +struct object_array *pending; +int fd; +}; + +static void write_bundle_prerequisites (struct commit *commit,void *data) +{ +struct bundle_prerequisites_info *bpi = data; +struct object *object; +struct pretty_print_context ctx = {0 }; +struct strbuf buf =STRBUF_INIT ; + +if (!(commit->object.flags &BOUNDARY )) +return ; +strbuf_addf (&buf,"-%s " ,oid_to_hex (&commit->object.oid)); +write_or_die (bpi->fd, buf.buf, buf.len); + + ctx.fmt =CMIT_FMT_ONELINE ; + ctx.output_encoding =get_log_output_encoding (); +strbuf_reset (&buf); +pretty_print_commit (&ctx, commit, &buf); +strbuf_trim (&buf); + + object = (struct object *)commit; + object->flags |=UNINTERESTING ; +add_object_array_with_path (object, buf.buf, bpi->pending,S_IFINVALID , +NULL ); +strbuf_addch (&buf,'\n' ); +write_or_die (bpi->fd, buf.buf, buf.len); +strbuf_release (&buf); +} +int create_bundle (struct repository *r,const char *path,int argc,const char **argv,struct strvec *pack_options,int version) {
@@ -481,8 +471,10 @@ int create_bundle(struct repository *r, const char *path,int bundle_fd =-1 ;int bundle_to_stdout;int ref_count =0 ; -struct rev_info revs; +struct rev_info revs, revs_copy;int min_version = the_hash_algo == &hash_algos[GIT_HASH_SHA1 ] ?2 :3 ; +struct bundle_prerequisites_info bpi; +int i; bundle_to_stdout = !strcmp (path,"-" );if (bundle_to_stdout)
@@ -512,10 +504,6 @@ int create_bundle(struct repository *r, const char *path, save_commit_buffer =0 ;repo_init_revisions (r, &revs,NULL ); -/* write prerequisites */ -if (compute_and_write_prerequisites (bundle_fd, &revs, argc, argv)) - goto err; - argc =setup_revisions (argc, argv, &revs,NULL );if (argc >1 ) {
@@ -523,16 +511,37 @@ int create_bundle(struct repository *r, const char *path, goto err; } -object_array_remove_duplicates (&revs.pending); +/* save revs.pending in revs_copy for later use */ +memcpy (&revs_copy, &revs,sizeof (revs)); + revs_copy.pending.nr =0 ; + revs_copy.pending.alloc =0 ; + revs_copy.pending.objects =NULL ; +for (i =0 ; i < revs.pending.nr; i++) { +struct object_array_entry *e = revs.pending.objects + i; +if (e) +add_object_array_with_path (e->item, e->name, + &revs_copy.pending, + e->mode, e->path); + } - ref_count =write_bundle_refs (bundle_fd, &revs); +/* write prerequisites */ + revs.boundary =1 ; +if (prepare_revision_walk (&revs)) +die ("revision walk setup failed" ); + bpi.fd = bundle_fd; + bpi.pending = &revs_copy.pending; +traverse_commit_list (&revs, write_bundle_prerequisites,NULL , &bpi); +object_array_remove_duplicates (&revs_copy.pending); + +/* write bundle refs */ + ref_count =write_bundle_refs (bundle_fd, &revs_copy);if (!ref_count)die (_ ("Refusing to create empty bundle." ));else if (ref_count <0 ) goto err;/* write pack */ -if (write_pack_data (bundle_fd, &revs, pack_options)) +if (write_pack_data (bundle_fd, &revs_copy, pack_options)) goto err;if (!bundle_to_stdout) {
t/t5607-clone-bundle.sh +2 -2
@@ -38,13 +38,13 @@ test_expect_success 'die if bundle file cannot be created' ' test_must_fail git bundle create adir --all ' -test_expect_failure 'bundle --stdin' ' +test_expect_success 'bundle --stdin' ' echo master | git bundle create stdin-bundle.bdl --stdin && git ls-remote stdin-bundle.bdl >output && grep master output ' -test_expect_failure 'bundle --stdin <rev-list options>' ' +test_expect_success 'bundle --stdin <rev-list options>' ' echo master | git bundle create hybrid-bundle.bdl --stdin tag && git ls-remote hybrid-bundle.bdl >output && grep master output
t/t6020-bundle-misc.sh +73 -4
143 lines not on this page · raw message
Signature · 1 line
2.30.0.2.g06d2f50715
Re: [PATCH v4 1/2] bundle: lost objects when removing duplicate pendings
In reply to Jiang XinQuoted 29 lines · Jiang Xin
t/t6020-bundle-misc.sh new file +23
-0@@ -0,0 +1,477 @@ +#!/bin/sh +# +# Copyright (c) 2021 Jiang Xin +# + +test_description='Test git-bundle' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +. ./test-lib.sh + +# Check count of objects in a bundle file. +# We can use "--thin" opiton to check thin pack, which must be fixed by +# command `git-index-pack --fix-thin --stdin`. +test_bundle_object_count () { + thin= && + if test "$1" = "--thin" + then + thin=yes + shift + fi && + if test $# -ne 2
t/t6020-bundle-misc.sh line 33
Quoted 10 lines
t/t6020-bundle-misc.sh new file +10
-0+ then + echo >&2 "args should be: <bundle> <count>" + return 1 + fi + bundle=$1 && + pack=${bundle%.bdl}.pack && + convert_bundle_to_pack <"$bundle" >"$pack" && + if test -n "$thin" + then + test_must_fail git index-pack "$pack" &&
This is overly strict, isn't it?
Imagine a case where the objects newer revisions introduce have *no* resemblance to the objects in the prerequisites' trees---the resulting pack will have no object that is expressed as a delta against anything outside the pack, and the above "index-pack" would succeed.
Besides, "git pack-objects --thin" is *not* obligated to create a pack that lacks one or more objects. The "--thin" option merely *allows* pack-objects to omit base objects if it is convenient to do so.
t/t6020-bundle-misc.sh line 36
Quoted 3 lines
t/t6020-bundle-misc.sh new file +3
-0+ mv "$pack" "$pack"-thin && + cat "$pack"-thin | + git index-pack --stdin --fix-thin "$pack"
This side is good, but do not cat a single file into a pipe. The whole "then" clause would become
then mv "$pack" "$pack-thin" && git index-pack --stdin --fix-thin "$pack" <"$pack-thin" else
I would think.
t/t6020-bundle-misc.sh line 45
Quoted 9 lines
t/t6020-bundle-misc.sh new file +9
-0+ else + git index-pack "$pack" + fi && + git verify-pack -v "$pack" >verify.out + if test $? -ne 0 + then + echo >&2 "error: fail to convert $bundle to $pack" + return 1 + fi
At this point, we are not testing the bundle subcommand, but is testing "git index-pack --fix-thin" that we run ourselves. Is it essential to ensure $pack is sane here? I doubt it.
t/t6020-bundle-misc.sh line 46
Quoted 1 line
t/t6020-bundle-misc.sh new file +1
-0+ count=$(grep -c "^$OID_REGEX " verify.out) &&
And if there is no need to run verify-pack, then we can do count=$(git show-index "${pack%pack}idx" | wc -l) instead, perhaps?
Quoted 4 lines
t/t6020-bundle-misc.sh new file +4
-0+ test $2 = $count && return 0 + echo >&2 "error: object count for $bundle is $count, not $2" + return 1 +}
t/t6020-bundle-misc.sh line 60
Quoted 10 lines
t/t6020-bundle-misc.sh new file +10
-0+ +# Display the pack data contained in the bundle file, bypassing the +# header that contains the signature, prerequisites and references. +convert_bundle_to_pack () { + while read x && test -n "$x" + do + :; + done + cat +}
This looks somewhat familiar. Perhaps extract out necessary helpers including this one into t/test-bundle-lib or something similar in a preparatory step before this patch?
Quoted 28 lines
t/t6020-bundle-misc.sh new file +28
-0+# Create a commit or tag and set the variable with the object ID. +test_commit_setvar () { + notick= && + signoff= && + indir= && + merge= && + tag= && + var= && + while test $# != 0 + do + case "$1" in + --merge) + merge=yes + ;; + --tag) + tag=yes + ;; + --notick) + notick=yes + ;; + --signoff) + signoff="$1" + ;; + -C) + indir="$2" + shift + ;; + -*)
t/t6020-bundle-misc.sh line 99
Quoted 10 lines
t/t6020-bundle-misc.sh new file +10
-0+ echo >&2 "error: unknown option $1" + return 1 + ;; + *) + test -n "$var" && break + var=$1 + ;; + esac + shift + done &&
At this point, if $var is still empty, the caller is buggy, and ...
Quoted 18 lines
t/t6020-bundle-misc.sh new file +18
-0+ indir=${indir:+"$indir"/} && + if test $# -eq 0 + then + echo >&2 "no args provided" + return 1 + fi && + if test -z "$notick" + then + test_tick + fi && + if test -n "$merge" + then + git ${indir:+ -C "$indir"} merge --no-edit --no-ff \ + ${2:+-m "$2"} "$1" && + oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD) + elif test -n "$tag" + then + git ${indir:+ -C "$indir"} tag -m "$1" "$1" &&
t/t6020-bundle-misc.sh line 127
Quoted 10 lines
t/t6020-bundle-misc.sh new file +10
-0+ oid=$(git ${indir:+ -C "$indir"} rev-parse "$1") + else + file=${2:-"$1.t"} && + echo "${3-$1}" > "$indir$file" && + git ${indir:+ -C "$indir"} add "$file" && + git ${indir:+ -C "$indir"} commit $signoff -m "$1" && + oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD) + fi && + eval $var=$oid +}
... it will cause a failure in 'eval' we have here. Not good.
t/t6020-bundle-misc.sh line 132
Quoted 3 lines
t/t6020-bundle-misc.sh new file +3
-0+# Format the output of git commands to make a user-friendly and stable +# text. We can easily prepare the expect text without having to worry +# about future changes of the commit ID and spaces of the output.
Hmph. This relies on 7 hexdigits being sufficient to uniquely identify all objects involved in the test? It should be OK in practice.
Is there a point in having both <COMMIT-A> and <OID-A>? I would have expected that all these "full object name" conversions are unneeded.
Quoted 41 lines
t/t6020-bundle-misc.sh new file +41
-0+make_user_friendly_and_stable_output () { + sed \ + -e "s/$A/<COMMIT-A>/" \ + -e "s/$B/<COMMIT-B>/" \ + -e "s/$C/<COMMIT-C>/" \ + -e "s/$D/<COMMIT-D>/" \ + -e "s/$E/<COMMIT-E>/" \ + -e "s/$F/<COMMIT-F>/" \ + -e "s/$G/<COMMIT-G>/" \ + -e "s/$H/<COMMIT-H>/" \ + -e "s/$I/<COMMIT-I>/" \ + -e "s/$J/<COMMIT-J>/" \ + -e "s/$K/<COMMIT-K>/" \ + -e "s/$L/<COMMIT-L>/" \ + -e "s/$M/<COMMIT-M>/" \ + -e "s/$N/<COMMIT-N>/" \ + -e "s/$O/<COMMIT-O>/" \ + -e "s/$P/<COMMIT-P>/" \ + -e "s/$TAG1/<TAG-1>/" \ + -e "s/$TAG2/<TAG-2>/" \ + -e "s/$TAG3/<TAG-3>/" \ + -e "s/$(echo $A | cut -c1-7)[0-9a-f]*/<OID-A>/g" \ + -e "s/$(echo $B | cut -c1-7)[0-9a-f]*/<OID-B>/g" \ + -e "s/$(echo $C | cut -c1-7)[0-9a-f]*/<OID-C>/g" \ + -e "s/$(echo $D | cut -c1-7)[0-9a-f]*/<OID-D>/g" \ + -e "s/$(echo $E | cut -c1-7)[0-9a-f]*/<OID-E>/g" \ + -e "s/$(echo $F | cut -c1-7)[0-9a-f]*/<OID-F>/g" \ + -e "s/$(echo $G | cut -c1-7)[0-9a-f]*/<OID-G>/g" \ + -e "s/$(echo $H | cut -c1-7)[0-9a-f]*/<OID-H>/g" \ + -e "s/$(echo $I | cut -c1-7)[0-9a-f]*/<OID-I>/g" \ + -e "s/$(echo $J | cut -c1-7)[0-9a-f]*/<OID-J>/g" \ + -e "s/$(echo $K | cut -c1-7)[0-9a-f]*/<OID-K>/g" \ + -e "s/$(echo $L | cut -c1-7)[0-9a-f]*/<OID-L>/g" \ + -e "s/$(echo $M | cut -c1-7)[0-9a-f]*/<OID-M>/g" \ + -e "s/$(echo $N | cut -c1-7)[0-9a-f]*/<OID-N>/g" \ + -e "s/$(echo $O | cut -c1-7)[0-9a-f]*/<OID-O>/g" \ + -e "s/$(echo $P | cut -c1-7)[0-9a-f]*/<OID-P>/g" \ + -e "s/$(echo $TAG1 | cut -c1-7)[0-9a-f]*/<OID-TAG-1>/g" \ + -e "s/$(echo $TAG2 | cut -c1-7)[0-9a-f]*/<OID-TAG-2>/g" \ + -e "s/$(echo $TAG3 | cut -c1-7)[0-9a-f]*/<OID-TAG-3>/g" \ + -e "s/ *\$//"
t/t6020-bundle-misc.sh line 243
Quoted 10 lines
t/t6020-bundle-misc.sh new file +9
-0+} +test_expect_success 'create bundle from special rev: main^!' ' + git bundle create special-rev.bdl "main^!" && + + git bundle list-heads special-rev.bdl | + make_user_friendly_and_stable_output >actual && + cat >expect <<-EOF && + <COMMIT-P> refs/heads/main + EOF
We prefer to indent these more like so:
cat >expect <<-\EOF && <COMMIT-P> refs/heads/main EOF
i.e. the indent of the line with <<EOF on it and the indent of the line with the matching EOF are the same. Also, quote EOF to signal that the body of the here text should be taken as-is without $var substitution.
Quoted 10 lines · Jiang Xin
t/t5607-clone-bundle.sh +1
-1@@ -38,13 +38,13 @@ test_expect_success 'die if bundle file cannot be created' ' test_must_fail git bundle create adir --all ' -test_expect_failure 'bundle --stdin' ' +test_expect_success 'bundle --stdin' '
t/t5607-clone-bundle.sh test_expect_success 'die if bundle file cannot be created' ' · line 50
Quoted 10 lines
t/t5607-clone-bundle.sh +1
-1echo master | git bundle create stdin-bundle.bdl --stdin && git ls-remote stdin-bundle.bdl >output && grep master output ' -test_expect_failure 'bundle --stdin <rev-list options>' ' +test_expect_success 'bundle --stdin <rev-list options>' ' echo master | git bundle create hybrid-bundle.bdl --stdin tag && git ls-remote hybrid-bundle.bdl >output && grep master output
I think these tests that documented the known limitation came from f62e0a39 (t5704 (bundle): add tests for bundle --stdin, 2010-04-19).
The author(s) of the original test deserves to be notified ;-)
Re: [PATCH v4 1/2] bundle: lost objects when removing duplicate pendings
In reply to Junio C HamanoQuoted 36 lines · Junio C Hamano <[email protected]> 于2021年1月9日周六 上午10:11写道:
Quoted 33 lines · Jiang Xin
t/t6020-bundle-misc.sh new file +27
-0@@ -0,0 +1,477 @@ +#!/bin/sh +# +# Copyright (c) 2021 Jiang Xin +# + +test_description='Test git-bundle' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +. ./test-lib.sh + +# Check count of objects in a bundle file. +# We can use "--thin" opiton to check thin pack, which must be fixed by +# command `git-index-pack --fix-thin --stdin`. +test_bundle_object_count () { + thin= && + if test "$1" = "--thin" + then + thin=yes + shift + fi && + if test $# -ne 2 + then + echo >&2 "args should be: <bundle> <count>" + return 1 + fi
Quoted 8 lines
t/t6020-bundle-misc.sh line 33
Quoted 6 lines
t/t6020-bundle-misc.sh new file +6
-0+ bundle=$1 && + pack=${bundle%.bdl}.pack && + convert_bundle_to_pack <"$bundle" >"$pack" && + if test -n "$thin" + then + test_must_fail git index-pack "$pack" &&This is overly strict, isn't it?
I want to make sure that the created bundle file which contains a thin pack is not changed, but now I think this check is not necessary, testing the count of objects is enough.
Quoted 18 lines
Imagine a case where the objects newer revisions introduce have *no* resemblance to the objects in the prerequisites' trees---the resulting pack will have no object that is expressed as a delta against anything outside the pack, and the above "index-pack" would succeed.
Besides, "git pack-objects --thin" is *not* obligated to create a pack that lacks one or more objects. The "--thin" option merely *allows* pack-objects to omit base objects if it is convenient to do so.
t/t6020-bundle-misc.sh line 36
Quoted 3 lines
t/t6020-bundle-misc.sh new file +3
-0+ mv "$pack" "$pack"-thin && + cat "$pack"-thin | + git index-pack --stdin --fix-thin "$pack"This side is good, but do not cat a single file into a pipe. The whole "then" clause would become
Quoted 6 lines
then mv "$pack" "$pack-thin" && git index-pack --stdin --fix-thin "$pack" <"$pack-thin" elseI would think.
That's better.
Quoted 10 lines
t/t6020-bundle-misc.sh line 45
Quoted 9 lines
t/t6020-bundle-misc.sh new file +9
-0+ else + git index-pack "$pack" + fi && + git verify-pack -v "$pack" >verify.out + if test $? -ne 0 + then + echo >&2 "error: fail to convert $bundle to $pack" + return 1 + fi
Quoted 3 lines
At this point, we are not testing the bundle subcommand, but is testing "git index-pack --fix-thin" that we run ourselves. Is it essential to ensure $pack is sane here? I doubt it.
If not check the return error code of git index-pack, the report message will be "error: object count for $bundle is , not $2". I want to give a specific error message for developer.
Quoted 4 lines
t/t6020-bundle-misc.sh line 46
Quoted 1 line
t/t6020-bundle-misc.sh new file +1
-0+ count=$(grep -c "^$OID_REGEX " verify.out) &&And if there is no need to run verify-pack, then we can do count=$(git show-index "${pack%pack}idx" | wc -l) instead, perhaps?
Will do.
Quoted 45 lines
Quoted 4 lines
t/t6020-bundle-misc.sh new file +4
-0+ test $2 = $count && return 0 + echo >&2 "error: object count for $bundle is $count, not $2" + return 1 +}t/t6020-bundle-misc.sh line 60
Quoted 10 lines
t/t6020-bundle-misc.sh new file +10
-0+ +# Display the pack data contained in the bundle file, bypassing the +# header that contains the signature, prerequisites and references. +convert_bundle_to_pack () { + while read x && test -n "$x" + do + :; + done + cat +}This looks somewhat familiar. Perhaps extract out necessary helpers including this one into t/test-bundle-lib or something similar in a preparatory step before this patch?
Quoted 26 lines
t/t6020-bundle-misc.sh new file +26
-0+# Create a commit or tag and set the variable with the object ID. +test_commit_setvar () { + notick= && + signoff= && + indir= && + merge= && + tag= && + var= && + while test $# != 0 + do + case "$1" in + --merge) + merge=yes + ;; + --tag) + tag=yes + ;; + --notick) + notick=yes + ;; + --signoff) + signoff="$1" + ;; + -C) + indir="$2" + shift
Quoted 8 lines
t/t6020-bundle-misc.sh line 95
Quoted 8 lines
t/t6020-bundle-misc.sh new file +8
-0+ ;; + -*) + echo >&2 "error: unknown option $1" + return 1 + ;; + *) + test -n "$var" && break + var=$1
The loop ends only if $var has been assigned a value, or no other args. Will report error if no other args later.
Quoted 6 lines
t/t6020-bundle-misc.sh line 99
Quoted 4 lines
t/t6020-bundle-misc.sh new file +4
-0+ ;; + esac + shift + done &&At this point, if $var is still empty, the caller is buggy, and ...
See the above note.
Quoted 39 lines
Quoted 18 lines
t/t6020-bundle-misc.sh new file +18
-0+ indir=${indir:+"$indir"/} && + if test $# -eq 0 + then + echo >&2 "no args provided" + return 1 + fi && + if test -z "$notick" + then + test_tick + fi && + if test -n "$merge" + then + git ${indir:+ -C "$indir"} merge --no-edit --no-ff \ + ${2:+-m "$2"} "$1" && + oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD) + elif test -n "$tag" + then + git ${indir:+ -C "$indir"} tag -m "$1" "$1" &&t/t6020-bundle-misc.sh line 127
Quoted 10 lines
t/t6020-bundle-misc.sh new file +10
-0+ oid=$(git ${indir:+ -C "$indir"} rev-parse "$1") + else + file=${2:-"$1.t"} && + echo "${3-$1}" > "$indir$file" && + git ${indir:+ -C "$indir"} add "$file" && + git ${indir:+ -C "$indir"} commit $signoff -m "$1" && + oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD) + fi && + eval $var=$oid +}... it will cause a failure in 'eval' we have here. Not good.
t/t6020-bundle-misc.sh line 132
Quoted 3 lines
t/t6020-bundle-misc.sh new file +3
-0+# Format the output of git commands to make a user-friendly and stable +# text. We can easily prepare the expect text without having to worry +# about future changes of the commit ID and spaces of the output.Hmph. This relies on 7 hexdigits being sufficient to uniquely identify all objects involved in the test? It should be OK in practice.
Quoted 3 lines
Is there a point in having both <COMMIT-A> and <OID-A>? I would have expected that all these "full object name" conversions are unneeded.
Will do.
Quoted 63 lines
Quoted 41 lines
t/t6020-bundle-misc.sh new file +41
-0+make_user_friendly_and_stable_output () { + sed \ + -e "s/$A/<COMMIT-A>/" \ + -e "s/$B/<COMMIT-B>/" \ + -e "s/$C/<COMMIT-C>/" \ + -e "s/$D/<COMMIT-D>/" \ + -e "s/$E/<COMMIT-E>/" \ + -e "s/$F/<COMMIT-F>/" \ + -e "s/$G/<COMMIT-G>/" \ + -e "s/$H/<COMMIT-H>/" \ + -e "s/$I/<COMMIT-I>/" \ + -e "s/$J/<COMMIT-J>/" \ + -e "s/$K/<COMMIT-K>/" \ + -e "s/$L/<COMMIT-L>/" \ + -e "s/$M/<COMMIT-M>/" \ + -e "s/$N/<COMMIT-N>/" \ + -e "s/$O/<COMMIT-O>/" \ + -e "s/$P/<COMMIT-P>/" \ + -e "s/$TAG1/<TAG-1>/" \ + -e "s/$TAG2/<TAG-2>/" \ + -e "s/$TAG3/<TAG-3>/" \ + -e "s/$(echo $A | cut -c1-7)[0-9a-f]*/<OID-A>/g" \ + -e "s/$(echo $B | cut -c1-7)[0-9a-f]*/<OID-B>/g" \ + -e "s/$(echo $C | cut -c1-7)[0-9a-f]*/<OID-C>/g" \ + -e "s/$(echo $D | cut -c1-7)[0-9a-f]*/<OID-D>/g" \ + -e "s/$(echo $E | cut -c1-7)[0-9a-f]*/<OID-E>/g" \ + -e "s/$(echo $F | cut -c1-7)[0-9a-f]*/<OID-F>/g" \ + -e "s/$(echo $G | cut -c1-7)[0-9a-f]*/<OID-G>/g" \ + -e "s/$(echo $H | cut -c1-7)[0-9a-f]*/<OID-H>/g" \ + -e "s/$(echo $I | cut -c1-7)[0-9a-f]*/<OID-I>/g" \ + -e "s/$(echo $J | cut -c1-7)[0-9a-f]*/<OID-J>/g" \ + -e "s/$(echo $K | cut -c1-7)[0-9a-f]*/<OID-K>/g" \ + -e "s/$(echo $L | cut -c1-7)[0-9a-f]*/<OID-L>/g" \ + -e "s/$(echo $M | cut -c1-7)[0-9a-f]*/<OID-M>/g" \ + -e "s/$(echo $N | cut -c1-7)[0-9a-f]*/<OID-N>/g" \ + -e "s/$(echo $O | cut -c1-7)[0-9a-f]*/<OID-O>/g" \ + -e "s/$(echo $P | cut -c1-7)[0-9a-f]*/<OID-P>/g" \ + -e "s/$(echo $TAG1 | cut -c1-7)[0-9a-f]*/<OID-TAG-1>/g" \ + -e "s/$(echo $TAG2 | cut -c1-7)[0-9a-f]*/<OID-TAG-2>/g" \ + -e "s/$(echo $TAG3 | cut -c1-7)[0-9a-f]*/<OID-TAG-3>/g" \ + -e "s/ *\$//"t/t6020-bundle-misc.sh line 243
Quoted 10 lines
t/t6020-bundle-misc.sh new file +9
-0+} +test_expect_success 'create bundle from special rev: main^!' ' + git bundle create special-rev.bdl "main^!" && + + git bundle list-heads special-rev.bdl | + make_user_friendly_and_stable_output >actual && + cat >expect <<-EOF && + <COMMIT-P> refs/heads/main + EOFWe prefer to indent these more like so:
cat >expect <<-\EOF && <COMMIT-P> refs/heads/main EOFi.e. the indent of the line with <<EOF on it and the indent of the line with the matching EOF are the same. Also, quote EOF to signal that the body of the here text should be taken as-is without $var substitution.
Re: [PATCH v4 1/2] bundle: lost objects when removing duplicate pendings
In reply to Junio C HamanoQuoted 5 lines · Junio C Hamano <[email protected]> 于2021年1月9日周六 上午10:11写道:
Quoted 5 lines
t/t6020-bundle-misc.sh new file +5
-0+# Display the pack data contained in the bundle file, bypassing the +# header that contains the signature, prerequisites and references. +convert_bundle_to_pack () { + while read x && test -n "$x" + do
Quoted 8 lines
t/t6020-bundle-misc.sh line 60
Quoted 4 lines
t/t6020-bundle-misc.sh new file +4
-0+ :; + done + cat +}This looks somewhat familiar. Perhaps extract out necessary helpers including this one into t/test-bundle-lib or something similar in a preparatory step before this patch?
Will add a new helper "t/test-bundle-functions.sh". But as how to include this helper, there are several solutions, which is better?
1. For those scripts which use these shared functions include this helper like:
. ./test-lib.sh
# current directory changed, so add path prefix . "$TEST_DIRECTORY"/test-bundle-functions.sh
2. Include "test-bundle-functions.sh" in "test-lib.sh" like this:
. "$TEST_DIRECTORY/test-lib-functions.sh" . "$TEST_DIRECTORY/test-bundle-functions.sh"
3. Create a "t/test-lib.d/" directory, add add “test-bundle-functions.sh” inside "t/test-lib.d/" as an extension, just like chriscool introduced in sharness project.
----8<-----
# test_perf subshells can have them too . "$TEST_DIRECTORY/test-lib-functions.sh"
+if test -d "$TEST_DIRECTORY/test-lib.d"
+then
+ for file in "$TEST_DIRECTORY/test-lib.d/"*.sh
+ do
+ # Ensure glob was not an empty match:
+ test -e "${file}" || break
+
+ if test -n "$debug"
+ then
+ echo >&5 "test-lib: loading extensions from ${file}"
+ fi
+ . "${file}"
+ if test $? != 0
+ then+ echo >&5 "test-lib: Error loading ${file}. Aborting."
+ exit 1
+ fi
+ done
+fi
+
----->8-----
Re: [PATCH v4 1/2] bundle: lost objects when removing duplicate pendings
In reply to Jiang XinQuoted 14 lines · Jiang Xin
Quoted 14 lines
Quoted 14 lines
t/t6020-bundle-misc.sh new file +13
-0+# Create a commit or tag and set the variable with the object ID. +test_commit_setvar () { + notick= && + signoff= && + indir= && + merge= && + tag= && + var= && + while test $# != 0 + do + case "$1" in + -*) + echo >&2 "error: unknown option $1"
Quoted 8 lines
Quoted 5 lines
t/t6020-bundle-misc.sh line 95
Quoted 5 lines
t/t6020-bundle-misc.sh new file +5
-0+ return 1 + ;; + *) + test -n "$var" && break + var=$1The loop ends only if $var has been assigned a value, or no other args. Will report error if no other args later.
"We see an arg that is not an dashed option. We already saw an arg and taken it as the name of the variable, so break" was a misleading way to structure this loop.
It looked as if it was just refusing to parse the remainder of the command line, so that a check after the loop would complain if there is still remaining arguments (as if to warn "var given twice"). But that is not what the post-loop check does; it expects there still are some argument left to be processed in mode-specific code that follows, so it happens to work as intended.
That is brittle, though. The current code may always consume one or more extra arguments in $merge/$tag/other specific mode in the code after the loop, but a new mode that will get added in the future to sit next to --merge and --tag may learn all necessary info in the command line parsing loop above, without any need for extra args to be processed after the loop. And $#=0 may not always be an error at that point.
I forgot to notice / mention it, but now you made me to look at the loop again, I see this part
-C) indir="$2" shift ;;
does not ensure we are getting something sensible in -C; that potential bug by the caller also happens to be covered by the post-loop "we require at least one argument that we can use as an arg" check, but as I said already, that feels rather brittle.
Anyway, let's queue the patches as-is and see what others think.
Thanks.
Re: [PATCH v4 1/2] bundle: lost objects when removing duplicate pendings
In reply to Jiang XinQuoted 5 lines · Jiang Xin
1. For those scripts which use these shared functions include this helper like:
. ./test-lib.sh
# current directory changed, so add path prefix . "$TEST_DIRECTORY"/test-bundle-functions.sh
This one. But without double spaces after the dot.
v5 cover improvements for git-bundle
In reply to Junio C HamanoFrom: Jiang Xin <[email protected]>
Introduce two improvements for git-bundle
+ Commit "bundle: lost objects when removing duplicate pendings", which fixes command like:
$ git bundle create <file> 'master^!'
+ Commits "bundle: arguments can be read from stdin", which add "--stdin" option support for git-bundle, like:
$ git bundle create <file> <input
## Changes since v4
+ New patch 1: Add helper functions in 't/test-bundle-functions.sh' for git-bundle.
+ Move t/t6020 to patch 1.
--
Jiang Xin (3):
test: add helper functions for git-bundle
bundle: lost objects when removing duplicate pendings
bundle: arguments can be read from stdin
bundle.c | 109 +++++---- object.c | 10 +- t/t5510-fetch.sh | 26 +-- t/t5607-clone-bundle.sh | 4 +- t/t6020-bundle-misc.sh | 465 +++++++++++++++++++++++++++++++++++++ t/test-bundle-functions.sh | 47 ++++ 6 files changed, 583 insertions(+), 78 deletions(-) create mode 100755 t/t6020-bundle-misc.sh create mode 100644 t/test-bundle-functions.sh
Signature · 1 line
2.30.0.2.g06d2f50715
PATCH v5 1/3 test: add helper functions for git-bundle
In reply to Junio C HamanoFrom: Jiang Xin <[email protected]>
Move git-bundle related functions from t5510 to a library, and this lib will be shared with a new testcase t6020 which finds a known breakage of "git-bundle".
--- t/t5510-fetch.sh | 26 +-- t/t6020-bundle-misc.sh | 396 +++++++++++++++++++++++++++++++++++++ t/test-bundle-functions.sh | 47 +++++ 3 files changed, 447 insertions(+), 22 deletions(-) create mode 100755 t/t6020-bundle-misc.sh create mode 100644 t/test-bundle-functions.sh
t/t5510-fetch.sh +4 -22
@@ -6,22 +6,10 @@ test_description='Per branch config variables affects "git fetch". ' . ./test-lib.sh +. "$TEST_DIRECTORY"/test-bundle-functions.sh D=$(pwd) -test_bundle_object_count () { - git verify-pack -v "$1" >verify.out && - test "$2" = $(grep "^$OID_REGEX " verify.out | wc -l) -} - -convert_bundle_to_pack () { - while read x && test -n "$x" - do - :; - done - cat -} - test_expect_success setup ' echo >file original && git add file &&
@@ -312,9 +300,7 @@ test_expect_success 'unbundle 1' ' test_expect_success 'bundle 1 has only 3 files ' ' cd "$D" && - convert_bundle_to_pack <bundle1 >bundle.pack && - git index-pack bundle.pack && - test_bundle_object_count bundle.pack 3 + test_bundle_object_count bundle1 3 ' test_expect_success 'unbundle 2' '
@@ -329,9 +315,7 @@ test_expect_success 'bundle does not prerequisite objects' ' git add file2 && git commit -m add.file2 file2 && git bundle create bundle3 -1 HEAD && - convert_bundle_to_pack <bundle3 >bundle.pack && - git index-pack bundle.pack && - test_bundle_object_count bundle.pack 3 + test_bundle_object_count bundle3 3 ' test_expect_success 'bundle should be able to create a full history' '
@@ -884,9 +868,7 @@ test_expect_success 'all boundary commits are excluded' ' git merge otherside && ad=$(git log --no-walk --format=%ad HEAD) && git bundle create twoside-boundary.bdl main --since="$ad" && - convert_bundle_to_pack <twoside-boundary.bdl >twoside-boundary.pack && - pack=$(git index-pack --fix-thin --stdin <twoside-boundary.pack) && - test_bundle_object_count .git/objects/pack/pack-${pack##pack }.pack 3 + test_bundle_object_count --thin twoside-boundary.bdl 3 ' test_expect_success 'fetch --prune prints the remotes url' '
t/t6020-bundle-misc.sh new file +396 -0
396 lines not on this page · raw message
t/test-bundle-functions.sh new file +47 -0
47 lines not on this page · raw message
Signature · 1 line
2.30.0.2.g06d2f50715
PATCH v5 2/3 bundle: lost objects when removing duplicate pendings
In reply to Junio C HamanoFrom: Jiang Xin <[email protected]>
git rev-list will list one commit for the following command:
$ git rev-list 'main^!'
<tip-commit-of-main-branch>But providing the same rev-list args to git bundle, fail to create a bundle file.
$ git bundle create - 'main^!'
# v2 git bundle
-<OID> <one-line-message>fatal: Refusing to create empty bundle.
This is because when removing duplicate objects in function object_array_remove_duplicates(), one unique pending object which has the same name is deleted by mistake. The revision arg 'main^!' in the above example is parsed by handle_revision_arg(), and at lease two different objects will be appended to revs.pending, one points to the parent commit of the "main" branch, and the other points to the tip commit of the "main" branch. These two objects have the same name "main". Only one object is left with the name "main" after calling the function object_array_remove_duplicates().
And what's worse, when adding boundary commits into pending list, we use one-line commit message as names, and the arbitory names may surprise git-bundle.
Only comparing objects themselves (".item") is also not good enough, because user may want to create a bundle with two identical objects but with different reference names, such as: "HEAD" and "refs/heads/main".
Add new function contains_object() which compare both the address and the name of the object.
--- object.c | 10 ++++++---- t/t6020-bundle-misc.sh | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-)
object.c +6 -4
@@ -412,15 +412,16 @@ void object_array_clear(struct object_array *array) }/* -* Return true iff array already contains an entry with name. +* Return true if array already contains an entry. */ -static int contains_name (struct object_array *array,const char *name) +static int contains_object (struct object_array *array, +const struct object *item,const char *name) {unsigned nr = array->nr, i;struct object_array_entry *object = array->objects;for (i =0 ; i < nr; i++, object++) -if (!strcmp (object->name, name)) +if (item == object->item && !strcmp (object->name, name))return 1 ;return 0 ; }
@@ -432,7 +433,8 @@ void object_array_remove_duplicates(struct object_array *array) array->nr =0 ;for (src =0 ; src < nr; src++) { -if (!contains_name (array, objects[src].name)) { +if (!contains_object (array, objects[src].item, + objects[src].name)) {if (src != array->nr) objects[array->nr] = objects[src]; array->nr++;
t/t6020-bundle-misc.sh +1 -1
@@ -169,7 +169,7 @@ test_expect_success 'setup' ' test_commit_setvar P "Commit P" main.txt ' -test_expect_failure 'create bundle from special rev: main^!' ' +test_expect_success 'create bundle from special rev: main^!' ' git bundle create special-rev.bdl "main^!" && git bundle list-heads special-rev.bdl |
Signature · 1 line
2.30.0.2.g06d2f50715
PATCH v5 3/3 bundle: arguments can be read from stdin
In reply to Junio C HamanoFrom: Jiang Xin <[email protected]>
In order to create an incremental bundle, we need to pass many arguments to let git-bundle ignore some already packed commits. It will be more convenient to pass args via stdin. But the current implementation does not allow us to do this.
This is because args are parsed twice when creating bundle. The first time for parsing args is in compute_and_write_prerequisites() by running git-rev-list command to write prerequisites in bundle file, and stdin is consumed in this step if "--stdin" option is provided for git-bundle. Later nothing can be read from stdin when running setup_revisions() in create_bundle().
The solution is to parse args once by removing the entire function compute_and_write_prerequisites() and then calling function setup_revisions(). In order to write prerequisites for bundle, will call prepare_revision_walk() and traverse_commit_list(). But after calling prepare_revision_walk(), the object array revs.pending is left empty, and the following steps could not work properly with the empty object array (revs.pending). Therefore, make a copy of revs to revs_copy for later use right after calling setup_revisions().
The copy of revs_copy is not a deep copy, it shares the same objects with revs. The object array of revs has been cleared, but objects themselves are still kept. Flags of objects may change after calling prepare_revision_walk(), we can use these changed flags without calling the git rev-list command and parsing its output like the former implementation.
Also add testcases for git bundle in t6020, which read args from stdin.
--- bundle.c | 109 ++++++++++++++++++++++------------------ t/t5607-clone-bundle.sh | 4 +- t/t6020-bundle-misc.sh | 77 ++++++++++++++++++++++++++-- 3 files changed, 134 insertions(+), 56 deletions(-)
bundle.c +59 -50
@@ -338,48 +338,6 @@ static int write_pack_data(int bundle_fd, struct rev_info *revs, struct strvec *return 0 ; } -static int compute_and_write_prerequisites (int bundle_fd, -struct rev_info *revs, -int argc,const char **argv) -{ -struct child_process rls =CHILD_PROCESS_INIT ; -struct strbuf buf =STRBUF_INIT ; -FILE *rls_fout; -int i; - -strvec_pushl (&rls.args, -"rev-list" ,"--boundary" ,"--pretty=oneline" , -NULL ); -for (i =1 ; i < argc; i++) -strvec_push (&rls.args, argv[i]); - rls.out =-1 ; - rls.git_cmd =1 ; -if (start_command (&rls)) -return -1 ; - rls_fout =xfdopen (rls.out,"r" ); -while (strbuf_getwholeline (&buf, rls_fout,'\n' ) !=EOF ) { -struct object_id oid; -if (buf.len >0 && buf.buf[0 ] =='-' ) { -write_or_die (bundle_fd, buf.buf, buf.len); -if (!get_oid_hex (buf.buf +1 , &oid)) { -struct object *object =parse_object_or_die (&oid, - buf.buf); - object->flags |=UNINTERESTING ; -add_pending_object (revs, object, buf.buf); - } - }else if (!get_oid_hex (buf.buf, &oid)) { -struct object *object =parse_object_or_die (&oid, - buf.buf); - object->flags |=SHOWN ; - } - } -strbuf_release (&buf); -fclose (rls_fout); -if (finish_command (&rls)) -return error (_ ("rev-list died" )); -return 0 ; -} -/* * Write out bundle refs based on the tips already * parsed into revs.pending. As a side effect, may
@@ -474,6 +432,38 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs)return ref_count; } +struct bundle_prerequisites_info { +struct object_array *pending; +int fd; +}; + +static void write_bundle_prerequisites (struct commit *commit,void *data) +{ +struct bundle_prerequisites_info *bpi = data; +struct object *object; +struct pretty_print_context ctx = {0 }; +struct strbuf buf =STRBUF_INIT ; + +if (!(commit->object.flags &BOUNDARY )) +return ; +strbuf_addf (&buf,"-%s " ,oid_to_hex (&commit->object.oid)); +write_or_die (bpi->fd, buf.buf, buf.len); + + ctx.fmt =CMIT_FMT_ONELINE ; + ctx.output_encoding =get_log_output_encoding (); +strbuf_reset (&buf); +pretty_print_commit (&ctx, commit, &buf); +strbuf_trim (&buf); + + object = (struct object *)commit; + object->flags |=UNINTERESTING ; +add_object_array_with_path (object, buf.buf, bpi->pending,S_IFINVALID , +NULL ); +strbuf_addch (&buf,'\n' ); +write_or_die (bpi->fd, buf.buf, buf.len); +strbuf_release (&buf); +} +int create_bundle (struct repository *r,const char *path,int argc,const char **argv,struct strvec *pack_options,int version) {
@@ -481,8 +471,10 @@ int create_bundle(struct repository *r, const char *path,int bundle_fd =-1 ;int bundle_to_stdout;int ref_count =0 ; -struct rev_info revs; +struct rev_info revs, revs_copy;int min_version = the_hash_algo == &hash_algos[GIT_HASH_SHA1 ] ?2 :3 ; +struct bundle_prerequisites_info bpi; +int i; bundle_to_stdout = !strcmp (path,"-" );if (bundle_to_stdout)
@@ -512,10 +504,6 @@ int create_bundle(struct repository *r, const char *path, save_commit_buffer =0 ;repo_init_revisions (r, &revs,NULL ); -/* write prerequisites */ -if (compute_and_write_prerequisites (bundle_fd, &revs, argc, argv)) - goto err; - argc =setup_revisions (argc, argv, &revs,NULL );if (argc >1 ) {
@@ -523,16 +511,37 @@ int create_bundle(struct repository *r, const char *path, goto err; } -object_array_remove_duplicates (&revs.pending); +/* save revs.pending in revs_copy for later use */ +memcpy (&revs_copy, &revs,sizeof (revs)); + revs_copy.pending.nr =0 ; + revs_copy.pending.alloc =0 ; + revs_copy.pending.objects =NULL ; +for (i =0 ; i < revs.pending.nr; i++) { +struct object_array_entry *e = revs.pending.objects + i; +if (e) +add_object_array_with_path (e->item, e->name, + &revs_copy.pending, + e->mode, e->path); + } - ref_count =write_bundle_refs (bundle_fd, &revs); +/* write prerequisites */ + revs.boundary =1 ; +if (prepare_revision_walk (&revs)) +die ("revision walk setup failed" ); + bpi.fd = bundle_fd; + bpi.pending = &revs_copy.pending; +traverse_commit_list (&revs, write_bundle_prerequisites,NULL , &bpi); +object_array_remove_duplicates (&revs_copy.pending); + +/* write bundle refs */ + ref_count =write_bundle_refs (bundle_fd, &revs_copy);if (!ref_count)die (_ ("Refusing to create empty bundle." ));else if (ref_count <0 ) goto err;/* write pack */ -if (write_pack_data (bundle_fd, &revs, pack_options)) +if (write_pack_data (bundle_fd, &revs_copy, pack_options)) goto err;if (!bundle_to_stdout) {
t/t5607-clone-bundle.sh +2 -2
@@ -38,13 +38,13 @@ test_expect_success 'die if bundle file cannot be created' ' test_must_fail git bundle create adir --all ' -test_expect_failure 'bundle --stdin' ' +test_expect_success 'bundle --stdin' ' echo master | git bundle create stdin-bundle.bdl --stdin && git ls-remote stdin-bundle.bdl >output && grep master output ' -test_expect_failure 'bundle --stdin <rev-list options>' ' +test_expect_success 'bundle --stdin <rev-list options>' ' echo master | git bundle create hybrid-bundle.bdl --stdin tag && git ls-remote hybrid-bundle.bdl >output && grep master output
t/t6020-bundle-misc.sh +73 -4
143 lines not on this page · raw message
Signature · 1 line
2.30.0.2.g06d2f50715
Re: [PATCH v5 1/3] test: add helper functions for git-bundle
In reply to Jiang XinQuoted 36 lines · Jiang Xin
t/t6020-bundle-misc.sh new file +36
-0+# Create a commit or tag and set the variable with the object ID. +test_commit_setvar () { + notick= + signoff= + indir= + merge= + tag= + var= + + while test $# != 0 + do + case "$1" in + --merge) + merge=t + ;; + --tag) + tag=t + ;; + --notick) + notick=t + ;; + --signoff) + signoff="$1" + ;; + -C) + shift + indir="$1" + ;; + -*) + echo >&2 "error: unknown option $1" + return 1 + ;; + *) + break + ;; + esac
t/t6020-bundle-misc.sh line 59
Quoted 10 lines
t/t6020-bundle-misc.sh new file +10
-0+ shift + done + + var=$1 + shift + if test -z "$var" + then + echo >&2 "error: var is not defined" + return 1 + fi
We need to check $# immediately after the loop to ensure that we can carve out $var and at least another arg. [*Nit 1*]
The previous round required the command line to have at least one after the loop (including parsing of $var) parsed it, but now we fall through from here when a command line were:
test_commit_setvar --merge -C there VAR
and because "$1" does not exist, such an error is propagated down to "git merge" not getting the side branch, "git tag" not getting the object to tag, etc.
Quoted 7 lines
t/t6020-bundle-misc.sh new file +7
-0+ indir=${indir:+"$indir"/} + if test -z "$notick" + then + test_tick + fi && + if test -n "$merge" + then
t/t6020-bundle-misc.sh line 76
Quoted 10 lines
t/t6020-bundle-misc.sh new file +10
-0+ git ${indir:+ -C "$indir"} merge --no-edit --no-ff \ + ${2:+-m "$2"} "$1" && + oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD) + elif test -n "$tag" + then + git ${indir:+ -C "$indir"} tag -m "$1" "$1" && + oid=$(git ${indir:+ -C "$indir"} rev-parse "$1") + else + file=${2:-"$1.t"} && + echo "${3-$1}" > "$indir$file" &&
Style? [*Nit 2*]
Quoted 6 lines
t/t6020-bundle-misc.sh new file +6
-0+ git ${indir:+ -C "$indir"} add "$file" && + git ${indir:+ -C "$indir"} commit $signoff -m "$1" && + oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD) + fi && + eval $var=$oid +}
t/t6020-bundle-misc.sh line 90
Quoted 10 lines
t/t6020-bundle-misc.sh new file +8
-0+ + +# Format the output of git commands to make a user-friendly and stable +# text. We can easily prepare the expect text without having to worry +# about future changes of the commit ID and spaces of the output. +make_user_friendly_and_stable_output () { + sed \ + -e "s/$(echo $A | cut -c1-7)[0-9a-f]*/<COMMIT-A>/g" \
Is "$(echo $A | cut -c1-7)" the same as "${A%${A#???????}}"? If so, the latter may be a bit shorter.
Quoted 33 lines
t/test-bundle-functions.sh new file +27
-0@@ -0,0 +1,47 @@ +# Library of git-bundle related functions. + +# Display the pack data contained in the bundle file, bypassing the +# header that contains the signature, prerequisites and references. +convert_bundle_to_pack () { + while read x && test -n "$x" + do + :; + done + cat +} + +# Check count of objects in a bundle file. +# We can use "--thin" opiton to check thin pack, which must be fixed by +# command `git-index-pack --fix-thin --stdin`. +test_bundle_object_count () { + thin= + if test "$1" = "--thin" + then + thin=t + shift + fi + if test $# -ne 2 + then + echo >&2 "args should be: <bundle> <count>" + return 1 + fi
t/test-bundle-functions.sh line 37
Quoted 10 lines
t/test-bundle-functions.sh new file +10
-0+ bundle=$1 + pack=$bundle.pack + convert_bundle_to_pack <"$bundle" >"$pack" && + if test -n "$thin" + then + mv "$pack" "$bundle.thin.pack" && + git index-pack --stdin --fix-thin "$pack" <"$bundle.thin.pack" + else + git index-pack "$pack" + fi
I wonder why we shouldn't always do "--fix-thin", so that the caller does not even have to bother passing the "--thin" option.
Is this to protect us from "git bundle" creating a bundle that contains a thin pack when it should not? A caller that knows it is storing a fully connected history can deliberately omit "--thin" from the command line and make sure "index-pack" that is not asked to do "--fix-thin" indeed finds the pack data fully self-contained, so it may be a good idea to have these two separate codepaths after all. OK.
t/test-bundle-functions.sh line 42
Quoted 5 lines
t/test-bundle-functions.sh new file +5
-0+ if test $? -ne 0 + then + echo >&2 "error: fail to convert $bundle or index-pack" + return 1 + fi
Do we even need the "error" message? "git index-pack" would have already given some error message to its standard error stream, no? If so
if test -n "$thin" then ... fi || return 1
would be sufficient, I guess.
t/test-bundle-functions.sh line 47
Quoted 5 lines
t/test-bundle-functions.sh new file +5
-0+ count=$(git show-index <"${pack%pack}idx" | wc -l) && + test $2 = $count && return 0 + echo >&2 "error: object count for $bundle is $count, not $2" + return 1 +}
Looking good except for a few minor nits I mentioned above.
Thanks.
Re: [PATCH v5 2/3] bundle: lost objects when removing duplicate pendings
In reply to Jiang XinQuoted 4 lines · Jiang Xin
/*
- * Return true iff array already contains an entry with name.
+ * Return true if array already contains an entry. */
I think the original deliberately spells "if and only if", but a natural reading of "return true if X" would be "this would never return true if not X", so perhaps the above change is OK.
Quoted 28 lines
object.c +5
-3-static int contains_name (struct object_array *array,const char *name) +static int contains_object (struct object_array *array, +const struct object *item,const char *name) {unsigned nr = array->nr, i;struct object_array_entry *object = array->objects;for (i =0 ; i < nr; i++, object++) -if (!strcmp (object->name, name)) +if (item == object->item && !strcmp (object->name, name))return 1 ;return 0 ; }@@ -432,7 +433,8 @@ void object_array_remove_duplicates(struct object_array *array) array->nr =0 ;for (src =0 ; src < nr; src++) { -if (!contains_name (array, objects[src].name)) { +if (!contains_object (array, objects[src].item, + objects[src].name)) {if (src != array->nr) objects[array->nr] = objects[src]; array->nr++;t/t6020-bundle-misc.sh +0
-0
t/t6020-bundle-misc.sh test_expect_success 'setup' ' · line 175
Quoted 8 lines
t/t6020-bundle-misc.sh +1
-1@@ -169,7 +169,7 @@ test_expect_success 'setup' ' test_commit_setvar P "Commit P" main.txt ' -test_expect_failure 'create bundle from special rev: main^!' ' +test_expect_success 'create bundle from special rev: main^!' ' git bundle create special-rev.bdl "main^!" && git bundle list-heads special-rev.bdl |
n
v6 cover improvements for git-bundle
In reply to Junio C HamanoFrom: Jiang Xin <[email protected]>
Introduce two improvements for git-bundle
+ Commit "bundle: lost objects when removing duplicate pendings", which fixes command like:
$ git bundle create <file> 'master^!'
+ Commits "bundle: arguments can be read from stdin", which add "--stdin" option support for git-bundle, like:
$ git bundle create <file> <input
## Changes since v5:
Quoted 18 lines · Junio C Hamano
Quoted 7 lines · Jiang Xin
+ var=$1
+ shift
+ if test -z "$var"
+ then
+ echo >&2 "error: var is not defined"
+ return 1
+ fiWe need to check $# immediately after the loop to ensure that we can carve out $var and at least another arg. [*Nit 1*]
The previous round required the command line to have at least one after the loop (including parsing of $var) parsed it, but now we fall through from here when a command line were:
Quoted 5 lines
test_commit_setvar--merge -C there VARand because "$1" does not exist, such an error is propagated down to "git merge" not getting the side branch, "git tag" not getting the object to tag, etc.
range-diff v5...v6 (part-1):
@@ t/t6020-bundle-misc.sh (new)
+ esac
+ shift
+ done
-+
-+ var=$1
-+ shift
-+ if test -z "$var"
++ if test $# -lt 2
+ then
-+ echo >&2 "error: var is not defined"
++ echo >&2 "error: test_commit_setvar must have at least 2 arguments"
+ return 1
+ fi
++ var=$1
++ shift
+ indir=${indir:+"$indir"/}
+ if test -z "$notick"
+ thenQuoted 5 lines
Quoted 3 lines
+ else
+ file=${2:-"$1.t"} &&
+ echo "${3-$1}" > "$indir$file" &&Style? [*Nit 2*]
range-diff v5...v6 (part-2):
* Add new arg "${2:-HEAD}" for git tag.
* Fix [*Nit 2*].
@@ t/t6020-bundle-misc.sh (new)
+ oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD)
+ elif test -n "$tag"
+ then
-+ git ${indir:+ -C "$indir"} tag -m "$1" "$1" &&
++ git ${indir:+ -C "$indir"} tag -m "$1" "$1" "${2:-HEAD}" &&
+ oid=$(git ${indir:+ -C "$indir"} rev-parse "$1")
+ else
+ file=${2:-"$1.t"} &&
-+ echo "${3-$1}" > "$indir$file" &&
++ echo "${3-$1}" >"$indir$file" &&
+ git ${indir:+ -C "$indir"} add "$file" &&
+ git ${indir:+ -C "$indir"} commit $signoff -m "$1" &&
+ oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD)Quoted 9 lines
t/t6020-bundle-misc.sh line 90
Quoted 6 lines
t/t6020-bundle-misc.sh new file +6
-0+# Format the output of git commands to make a user-friendly and stable +# text. We can easily prepare the expect text without having to worry +# about future changes of the commit ID and spaces of the output. +make_user_friendly_and_stable_output () { + sed \ + -e "s/$(echo $A | cut -c1-7)[0-9a-f]*/<COMMIT-A>/g" \Is "$(echo $A | cut -c1-7)" the same as "${A%${A#???????}}"? If so, the latter may be a bit shorter.
range-diff v5...v6 (part-3):
@@ t/t6020-bundle-misc.sh (new)
+ eval $var=$oid
+}
+
-+
+# Format the output of git commands to make a user-friendly and stable
+# text. We can easily prepare the expect text without having to worry
+# about future changes of the commit ID and spaces of the output.
+make_user_friendly_and_stable_output () {
+ sed \
-+ -e "s/$(echo $A | cut -c1-7)[0-9a-f]*/<COMMIT-A>/g" \
-+ -e "s/$(echo $B | cut -c1-7)[0-9a-f]*/<COMMIT-B>/g" \
-+ -e "s/$(echo $C | cut -c1-7)[0-9a-f]*/<COMMIT-C>/g" \
-+ -e "s/$(echo $D | cut -c1-7)[0-9a-f]*/<COMMIT-D>/g" \
-+ -e "s/$(echo $E | cut -c1-7)[0-9a-f]*/<COMMIT-E>/g" \
-+ -e "s/$(echo $F | cut -c1-7)[0-9a-f]*/<COMMIT-F>/g" \
-+ -e "s/$(echo $G | cut -c1-7)[0-9a-f]*/<COMMIT-G>/g" \
-+ -e "s/$(echo $H | cut -c1-7)[0-9a-f]*/<COMMIT-H>/g" \
-+ -e "s/$(echo $I | cut -c1-7)[0-9a-f]*/<COMMIT-I>/g" \
-+ -e "s/$(echo $J | cut -c1-7)[0-9a-f]*/<COMMIT-J>/g" \
-+ -e "s/$(echo $K | cut -c1-7)[0-9a-f]*/<COMMIT-K>/g" \
-+ -e "s/$(echo $L | cut -c1-7)[0-9a-f]*/<COMMIT-L>/g" \
-+ -e "s/$(echo $M | cut -c1-7)[0-9a-f]*/<COMMIT-M>/g" \
-+ -e "s/$(echo $N | cut -c1-7)[0-9a-f]*/<COMMIT-N>/g" \
-+ -e "s/$(echo $O | cut -c1-7)[0-9a-f]*/<COMMIT-O>/g" \
-+ -e "s/$(echo $P | cut -c1-7)[0-9a-f]*/<COMMIT-P>/g" \
-+ -e "s/$(echo $TAG1 | cut -c1-7)[0-9a-f]*/<TAG-1>/g" \
-+ -e "s/$(echo $TAG2 | cut -c1-7)[0-9a-f]*/<TAG-2>/g" \
-+ -e "s/$(echo $TAG3 | cut -c1-7)[0-9a-f]*/<TAG-3>/g" \
++ -e "s/${A%${A#???????}}[0-9a-f]*/<COMMIT-A>/g" \
++ -e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" \
++ -e "s/${C%${C#???????}}[0-9a-f]*/<COMMIT-C>/g" \
++ -e "s/${D%${D#???????}}[0-9a-f]*/<COMMIT-D>/g" \
++ -e "s/${E%${E#???????}}[0-9a-f]*/<COMMIT-E>/g" \
++ -e "s/${F%${F#???????}}[0-9a-f]*/<COMMIT-F>/g" \
++ -e "s/${G%${G#???????}}[0-9a-f]*/<COMMIT-G>/g" \
++ -e "s/${H%${H#???????}}[0-9a-f]*/<COMMIT-H>/g" \
++ -e "s/${I%${I#???????}}[0-9a-f]*/<COMMIT-I>/g" \
++ -e "s/${J%${J#???????}}[0-9a-f]*/<COMMIT-J>/g" \
++ -e "s/${K%${K#???????}}[0-9a-f]*/<COMMIT-K>/g" \
++ -e "s/${L%${L#???????}}[0-9a-f]*/<COMMIT-L>/g" \
++ -e "s/${M%${M#???????}}[0-9a-f]*/<COMMIT-M>/g" \
++ -e "s/${N%${N#???????}}[0-9a-f]*/<COMMIT-N>/g" \
++ -e "s/${O%${O#???????}}[0-9a-f]*/<COMMIT-O>/g" \
++ -e "s/${P%${P#???????}}[0-9a-f]*/<COMMIT-P>/g" \
++ -e "s/${TAG1%${TAG1#???????}}[0-9a-f]*/<TAG-1>/g" \
++ -e "s/${TAG2%${TAG2#???????}}[0-9a-f]*/<TAG-2>/g" \
++ -e "s/${TAG3%${TAG3#???????}}[0-9a-f]*/<TAG-3>/g" \
+ -e "s/ *\$//"
+}
+Quoted 10 lines
Do we even need the "error" message? "git index-pack" would have already given some error message to its standard error stream, no? If so
if test -n "$thin"
then
...
fi || return 1would be sufficient, I guess.
range-diff v5...v6 (part-4):
1: fa7516b2ec ! 1: 900bb16178 test: add helper functions for git-bundle
@@ t/test-bundle-functions.sh (new)
+ git index-pack --stdin --fix-thin "$pack" <"$bundle.thin.pack"
+ else
+ git index-pack "$pack"
-+ fi
-+ if test $? -ne 0
-+ then
-+ echo >&2 "error: fail to convert $bundle or index-pack"
-+ return 1
-+ fi
++ fi || return 1
+ count=$(git show-index <"${pack%pack}idx" | wc -l) &&
+ test $2 = $count && return 0
+ echo >&2 "error: object count for $bundle is $count, not $2"
2: ea543de111 = 2: 1bbf0ab213 bundle: lost objects when removing duplicate pendings
3: 18f0d48814 = 3: 7ac0751821 bundle: arguments can be read from stdin--
Jiang Xin (3):
test: add helper functions for git-bundle
bundle: lost objects when removing duplicate pendings
bundle: arguments can be read from stdin
bundle.c | 109 +++++---- object.c | 10 +- t/t5510-fetch.sh | 26 +-- t/t5607-clone-bundle.sh | 4 +- t/t6020-bundle-misc.sh | 463 +++++++++++++++++++++++++++++++++++++ t/test-bundle-functions.sh | 42 ++++ 6 files changed, 576 insertions(+), 78 deletions(-) create mode 100755 t/t6020-bundle-misc.sh create mode 100644 t/test-bundle-functions.sh
Signature · 1 line
2.28.0.15.gba9e81f0bd
PATCH v6 1/3 test: add helper functions for git-bundle
In reply to Junio C HamanoFrom: Jiang Xin <[email protected]>
Move git-bundle related functions from t5510 to a library, and this lib will be shared with a new testcase t6020 which finds a known breakage of "git-bundle".
--- t/t5510-fetch.sh | 26 +-- t/t6020-bundle-misc.sh | 394 +++++++++++++++++++++++++++++++++++++ t/test-bundle-functions.sh | 42 ++++ 3 files changed, 440 insertions(+), 22 deletions(-) create mode 100755 t/t6020-bundle-misc.sh create mode 100644 t/test-bundle-functions.sh
t/t5510-fetch.sh +4 -22
@@ -6,22 +6,10 @@ test_description='Per branch config variables affects "git fetch". ' . ./test-lib.sh +. "$TEST_DIRECTORY"/test-bundle-functions.sh D=$(pwd) -test_bundle_object_count () { - git verify-pack -v "$1" >verify.out && - test "$2" = $(grep "^$OID_REGEX " verify.out | wc -l) -} - -convert_bundle_to_pack () { - while read x && test -n "$x" - do - :; - done - cat -} - test_expect_success setup ' echo >file original && git add file &&
@@ -312,9 +300,7 @@ test_expect_success 'unbundle 1' ' test_expect_success 'bundle 1 has only 3 files ' ' cd "$D" && - convert_bundle_to_pack <bundle1 >bundle.pack && - git index-pack bundle.pack && - test_bundle_object_count bundle.pack 3 + test_bundle_object_count bundle1 3 ' test_expect_success 'unbundle 2' '
@@ -329,9 +315,7 @@ test_expect_success 'bundle does not prerequisite objects' ' git add file2 && git commit -m add.file2 file2 && git bundle create bundle3 -1 HEAD && - convert_bundle_to_pack <bundle3 >bundle.pack && - git index-pack bundle.pack && - test_bundle_object_count bundle.pack 3 + test_bundle_object_count bundle3 3 ' test_expect_success 'bundle should be able to create a full history' '
@@ -884,9 +868,7 @@ test_expect_success 'all boundary commits are excluded' ' git merge otherside && ad=$(git log --no-walk --format=%ad HEAD) && git bundle create twoside-boundary.bdl main --since="$ad" && - convert_bundle_to_pack <twoside-boundary.bdl >twoside-boundary.pack && - pack=$(git index-pack --fix-thin --stdin <twoside-boundary.pack) && - test_bundle_object_count .git/objects/pack/pack-${pack##pack }.pack 3 + test_bundle_object_count --thin twoside-boundary.bdl 3 ' test_expect_success 'fetch --prune prints the remotes url' '
t/t6020-bundle-misc.sh new file +394 -0
394 lines not on this page · raw message
t/test-bundle-functions.sh new file +42 -0
42 lines not on this page · raw message
Signature · 1 line
2.28.0.15.gba9e81f0bd
PATCH v6 3/3 bundle: arguments can be read from stdin
In reply to Junio C HamanoFrom: Jiang Xin <[email protected]>
In order to create an incremental bundle, we need to pass many arguments to let git-bundle ignore some already packed commits. It will be more convenient to pass args via stdin. But the current implementation does not allow us to do this.
This is because args are parsed twice when creating bundle. The first time for parsing args is in compute_and_write_prerequisites() by running git-rev-list command to write prerequisites in bundle file, and stdin is consumed in this step if "--stdin" option is provided for git-bundle. Later nothing can be read from stdin when running setup_revisions() in create_bundle().
The solution is to parse args once by removing the entire function compute_and_write_prerequisites() and then calling function setup_revisions(). In order to write prerequisites for bundle, will call prepare_revision_walk() and traverse_commit_list(). But after calling prepare_revision_walk(), the object array revs.pending is left empty, and the following steps could not work properly with the empty object array (revs.pending). Therefore, make a copy of revs to revs_copy for later use right after calling setup_revisions().
The copy of revs_copy is not a deep copy, it shares the same objects with revs. The object array of revs has been cleared, but objects themselves are still kept. Flags of objects may change after calling prepare_revision_walk(), we can use these changed flags without calling the git rev-list command and parsing its output like the former implementation.
Also add testcases for git bundle in t6020, which read args from stdin.
--- bundle.c | 109 ++++++++++++++++++++++------------------ t/t5607-clone-bundle.sh | 4 +- t/t6020-bundle-misc.sh | 77 ++++++++++++++++++++++++++-- 3 files changed, 134 insertions(+), 56 deletions(-)
bundle.c +59 -50
@@ -338,48 +338,6 @@ static int write_pack_data(int bundle_fd, struct rev_info *revs, struct strvec *return 0 ; } -static int compute_and_write_prerequisites (int bundle_fd, -struct rev_info *revs, -int argc,const char **argv) -{ -struct child_process rls =CHILD_PROCESS_INIT ; -struct strbuf buf =STRBUF_INIT ; -FILE *rls_fout; -int i; - -strvec_pushl (&rls.args, -"rev-list" ,"--boundary" ,"--pretty=oneline" , -NULL ); -for (i =1 ; i < argc; i++) -strvec_push (&rls.args, argv[i]); - rls.out =-1 ; - rls.git_cmd =1 ; -if (start_command (&rls)) -return -1 ; - rls_fout =xfdopen (rls.out,"r" ); -while (strbuf_getwholeline (&buf, rls_fout,'\n' ) !=EOF ) { -struct object_id oid; -if (buf.len >0 && buf.buf[0 ] =='-' ) { -write_or_die (bundle_fd, buf.buf, buf.len); -if (!get_oid_hex (buf.buf +1 , &oid)) { -struct object *object =parse_object_or_die (&oid, - buf.buf); - object->flags |=UNINTERESTING ; -add_pending_object (revs, object, buf.buf); - } - }else if (!get_oid_hex (buf.buf, &oid)) { -struct object *object =parse_object_or_die (&oid, - buf.buf); - object->flags |=SHOWN ; - } - } -strbuf_release (&buf); -fclose (rls_fout); -if (finish_command (&rls)) -return error (_ ("rev-list died" )); -return 0 ; -} -/* * Write out bundle refs based on the tips already * parsed into revs.pending. As a side effect, may
@@ -474,6 +432,38 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs)return ref_count; } +struct bundle_prerequisites_info { +struct object_array *pending; +int fd; +}; + +static void write_bundle_prerequisites (struct commit *commit,void *data) +{ +struct bundle_prerequisites_info *bpi = data; +struct object *object; +struct pretty_print_context ctx = {0 }; +struct strbuf buf =STRBUF_INIT ; + +if (!(commit->object.flags &BOUNDARY )) +return ; +strbuf_addf (&buf,"-%s " ,oid_to_hex (&commit->object.oid)); +write_or_die (bpi->fd, buf.buf, buf.len); + + ctx.fmt =CMIT_FMT_ONELINE ; + ctx.output_encoding =get_log_output_encoding (); +strbuf_reset (&buf); +pretty_print_commit (&ctx, commit, &buf); +strbuf_trim (&buf); + + object = (struct object *)commit; + object->flags |=UNINTERESTING ; +add_object_array_with_path (object, buf.buf, bpi->pending,S_IFINVALID , +NULL ); +strbuf_addch (&buf,'\n' ); +write_or_die (bpi->fd, buf.buf, buf.len); +strbuf_release (&buf); +} +int create_bundle (struct repository *r,const char *path,int argc,const char **argv,struct strvec *pack_options,int version) {
@@ -481,8 +471,10 @@ int create_bundle(struct repository *r, const char *path,int bundle_fd =-1 ;int bundle_to_stdout;int ref_count =0 ; -struct rev_info revs; +struct rev_info revs, revs_copy;int min_version = the_hash_algo == &hash_algos[GIT_HASH_SHA1 ] ?2 :3 ; +struct bundle_prerequisites_info bpi; +int i; bundle_to_stdout = !strcmp (path,"-" );if (bundle_to_stdout)
@@ -512,10 +504,6 @@ int create_bundle(struct repository *r, const char *path, save_commit_buffer =0 ;repo_init_revisions (r, &revs,NULL ); -/* write prerequisites */ -if (compute_and_write_prerequisites (bundle_fd, &revs, argc, argv)) - goto err; - argc =setup_revisions (argc, argv, &revs,NULL );if (argc >1 ) {
@@ -523,16 +511,37 @@ int create_bundle(struct repository *r, const char *path, goto err; } -object_array_remove_duplicates (&revs.pending); +/* save revs.pending in revs_copy for later use */ +memcpy (&revs_copy, &revs,sizeof (revs)); + revs_copy.pending.nr =0 ; + revs_copy.pending.alloc =0 ; + revs_copy.pending.objects =NULL ; +for (i =0 ; i < revs.pending.nr; i++) { +struct object_array_entry *e = revs.pending.objects + i; +if (e) +add_object_array_with_path (e->item, e->name, + &revs_copy.pending, + e->mode, e->path); + } - ref_count =write_bundle_refs (bundle_fd, &revs); +/* write prerequisites */ + revs.boundary =1 ; +if (prepare_revision_walk (&revs)) +die ("revision walk setup failed" ); + bpi.fd = bundle_fd; + bpi.pending = &revs_copy.pending; +traverse_commit_list (&revs, write_bundle_prerequisites,NULL , &bpi); +object_array_remove_duplicates (&revs_copy.pending); + +/* write bundle refs */ + ref_count =write_bundle_refs (bundle_fd, &revs_copy);if (!ref_count)die (_ ("Refusing to create empty bundle." ));else if (ref_count <0 ) goto err;/* write pack */ -if (write_pack_data (bundle_fd, &revs, pack_options)) +if (write_pack_data (bundle_fd, &revs_copy, pack_options)) goto err;if (!bundle_to_stdout) {
t/t5607-clone-bundle.sh +2 -2
@@ -38,13 +38,13 @@ test_expect_success 'die if bundle file cannot be created' ' test_must_fail git bundle create adir --all ' -test_expect_failure 'bundle --stdin' ' +test_expect_success 'bundle --stdin' ' echo master | git bundle create stdin-bundle.bdl --stdin && git ls-remote stdin-bundle.bdl >output && grep master output ' -test_expect_failure 'bundle --stdin <rev-list options>' ' +test_expect_success 'bundle --stdin <rev-list options>' ' echo master | git bundle create hybrid-bundle.bdl --stdin tag && git ls-remote hybrid-bundle.bdl >output && grep master output
t/t6020-bundle-misc.sh +73 -4
143 lines not on this page · raw message
Signature · 1 line
2.28.0.15.gba9e81f0bd
PATCH v6 2/3 bundle: lost objects when removing duplicate pendings
In reply to Junio C HamanoFrom: Jiang Xin <[email protected]>
git rev-list will list one commit for the following command:
$ git rev-list 'main^!'
<tip-commit-of-main-branch>But providing the same rev-list args to git bundle, fail to create a bundle file.
$ git bundle create - 'main^!'
# v2 git bundle
-<OID> <one-line-message>fatal: Refusing to create empty bundle.
This is because when removing duplicate objects in function object_array_remove_duplicates(), one unique pending object which has the same name is deleted by mistake. The revision arg 'main^!' in the above example is parsed by handle_revision_arg(), and at lease two different objects will be appended to revs.pending, one points to the parent commit of the "main" branch, and the other points to the tip commit of the "main" branch. These two objects have the same name "main". Only one object is left with the name "main" after calling the function object_array_remove_duplicates().
And what's worse, when adding boundary commits into pending list, we use one-line commit message as names, and the arbitory names may surprise git-bundle.
Only comparing objects themselves (".item") is also not good enough, because user may want to create a bundle with two identical objects but with different reference names, such as: "HEAD" and "refs/heads/main".
Add new function contains_object() which compare both the address and the name of the object.
--- object.c | 10 ++++++---- t/t6020-bundle-misc.sh | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-)
object.c +6 -4
@@ -412,15 +412,16 @@ void object_array_clear(struct object_array *array) }/* -* Return true iff array already contains an entry with name. +* Return true if array already contains an entry. */ -static int contains_name (struct object_array *array,const char *name) +static int contains_object (struct object_array *array, +const struct object *item,const char *name) {unsigned nr = array->nr, i;struct object_array_entry *object = array->objects;for (i =0 ; i < nr; i++, object++) -if (!strcmp (object->name, name)) +if (item == object->item && !strcmp (object->name, name))return 1 ;return 0 ; }
@@ -432,7 +433,8 @@ void object_array_remove_duplicates(struct object_array *array) array->nr =0 ;for (src =0 ; src < nr; src++) { -if (!contains_name (array, objects[src].name)) { +if (!contains_object (array, objects[src].item, + objects[src].name)) {if (src != array->nr) objects[array->nr] = objects[src]; array->nr++;
t/t6020-bundle-misc.sh +1 -1
@@ -167,7 +167,7 @@ test_expect_success 'setup' ' test_commit_setvar P "Commit P" main.txt ' -test_expect_failure 'create bundle from special rev: main^!' ' +test_expect_success 'create bundle from special rev: main^!' ' git bundle create special-rev.bdl "main^!" && git bundle list-heads special-rev.bdl |
Signature · 1 line
2.28.0.15.gba9e81f0bd
Runaway sed memory use in test on older sed+glibc (was "Re: [PATCH v6 1/3] test: add helper functions for git-bundle")
In reply to Jiang XinQuoted 23 lines · Jiang Xin
From: Jiang Xin <[email protected]>
Move git-bundle related functions from t5510 to a library, and this lib will be shared with a new testcase t6020 which finds a known breakage of "git-bundle".
[...]t/t6020-bundle-misc.sh new file +17
-0+ +# Format the output of git commands to make a user-friendly and stable +# text. We can easily prepare the expect text without having to worry +# about future changes of the commit ID and spaces of the output. +make_user_friendly_and_stable_output () { + sed \ + -e "s/${A%${A#???????}}[0-9a-f]*/<COMMIT-A>/g" \ + -e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" \ + -e "s/${C%${C#???????}}[0-9a-f]*/<COMMIT-C>/g" \ + -e "s/${D%${D#???????}}[0-9a-f]*/<COMMIT-D>/g" \ + -e "s/${E%${E#???????}}[0-9a-f]*/<COMMIT-E>/g" \ + -e "s/${F%${F#???????}}[0-9a-f]*/<COMMIT-F>/g" \ + -e "s/${G%${G#???????}}[0-9a-f]*/<COMMIT-G>/g" \ + -e "s/${H%${H#???????}}[0-9a-f]*/<COMMIT-H>/g" \ + -e "s/${I%${I#???????}}[0-9a-f]*/<COMMIT-I>/g" \ + -e "s/${J%${J#???????}}[0-9a-f]*/<COMMIT-J>/g" \ + -e "s/${K%${K#???????}}[0-9a-f]*/<COMMIT-K>/g" \
t/t6020-bundle-misc.sh line 108
Quoted 10 lines
t/t6020-bundle-misc.sh new file +10
-0+ -e "s/${L%${L#???????}}[0-9a-f]*/<COMMIT-L>/g" \ + -e "s/${M%${M#???????}}[0-9a-f]*/<COMMIT-M>/g" \ + -e "s/${N%${N#???????}}[0-9a-f]*/<COMMIT-N>/g" \ + -e "s/${O%${O#???????}}[0-9a-f]*/<COMMIT-O>/g" \ + -e "s/${P%${P#???????}}[0-9a-f]*/<COMMIT-P>/g" \ + -e "s/${TAG1%${TAG1#???????}}[0-9a-f]*/<TAG-1>/g" \ + -e "s/${TAG2%${TAG2#???????}}[0-9a-f]*/<TAG-2>/g" \ + -e "s/${TAG3%${TAG3#???????}}[0-9a-f]*/<TAG-3>/g" \ + -e "s/ *\$//" +}
On one of the gcc farm boxes, a i386 box (gcc45) this fails because sed gets killed after >500MB of memory use (I was just eyeballing it in htop) on the "reate bundle from special rev: main^!" test. This with GNU sed 4.2.2.
I suspect this regex pattern creates some runaway behavior in sed that's since been fixed (or maybe it's the glibc regex engine?). The glibc is 2.19-18+deb8u10:
+ git bundle list-heads special-rev.bdl
+ make_user_friendly_and_stable_output
+ sed -e s/[0-9a-f]*/<COMMIT-A>/g -e s/[0-9a-f]*/<COMMIT-B>/g -e s/[0-9a-f]*/<COMMIT-C>/g -e s/[0-9a-f]*/<COMMIT-D>/g -e s/[0-9a-f]*/<COMMIT-E>/g -e s/[0-9a-f]*/<COMMIT-F>/g -e s/[0-9a-f]*/<COMMIT-G>/g -e s/[0-9a-f]*/<COMMIT-H>/g -e s/[0-9a-f]*/<COMMIT-I>/g -e s/[0-9a-f]*/<COMMIT-J>/g -e s/[0-9a-f]*/<COMMIT-K>/g -e s/[0-9a-f]*/<COMMIT-L>/g -e s/[0-9a-f]*/<COMMIT-M>/g -e s/[0-9a-f]*/<COMMIT-N>/g -e s/[0-9a-f]*/<COMMIT-O>/g -e s/[0-9a-f]*/<COMMIT-P>/g -e s/[0-9a-f]*/<TAG-1>/g -e s/[0-9a-f]*/<TAG-2>/g -e s/[0-9a-f]*/<TAG-3>/g -e s/ *$//
sed: couldn't re-allocate memory
Re: Runaway sed memory use in test on older sed+glibc (was "Re: [PATCH v6 1/3] test: add helper functions for git-bundle")
Ævar Arnfjörð Bjarmason <[email protected]> 于2021年5月27日周四
Quoted 54 lines · 上午2:51写道:
Quoted 27 lines · Jiang Xin
From: Jiang Xin <[email protected]>
Move git-bundle related functions from t5510 to a library, and this lib
will be shared with a new testcase t6020 which finds a known breakage of
"git-bundle".
[...](quoted diff) +2
-0+ +# Format the output of git commands to make a user-friendly andstable
+# text. We can easily prepare the expect text without having to worry(quoted diff) +14
-0+# about future changes of the commit ID and spaces of the output. +make_user_friendly_and_stable_output () { + sed \ + -e "s/${A%${A#???????}}[0-9a-f]*/<COMMIT-A>/g" \ + -e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" \ + -e "s/${C%${C#???????}}[0-9a-f]*/<COMMIT-C>/g" \ + -e "s/${D%${D#???????}}[0-9a-f]*/<COMMIT-D>/g" \ + -e "s/${E%${E#???????}}[0-9a-f]*/<COMMIT-E>/g" \ + -e "s/${F%${F#???????}}[0-9a-f]*/<COMMIT-F>/g" \ + -e "s/${G%${G#???????}}[0-9a-f]*/<COMMIT-G>/g" \ + -e "s/${H%${H#???????}}[0-9a-f]*/<COMMIT-H>/g" \ + -e "s/${I%${I#???????}}[0-9a-f]*/<COMMIT-I>/g" \ + -e "s/${J%${J#???????}}[0-9a-f]*/<COMMIT-J>/g" \ + -e "s/${K%${K#???????}}[0-9a-f]*/<COMMIT-K>/g" \Review
Quoted 10 lines
(quoted diff) +10
-0+ -e "s/${L%${L#???????}}[0-9a-f]*/<COMMIT-L>/g" \ + -e "s/${M%${M#???????}}[0-9a-f]*/<COMMIT-M>/g" \ + -e "s/${N%${N#???????}}[0-9a-f]*/<COMMIT-N>/g" \ + -e "s/${O%${O#???????}}[0-9a-f]*/<COMMIT-O>/g" \ + -e "s/${P%${P#???????}}[0-9a-f]*/<COMMIT-P>/g" \ + -e "s/${TAG1%${TAG1#???????}}[0-9a-f]*/<TAG-1>/g" \ + -e "s/${TAG2%${TAG2#???????}}[0-9a-f]*/<TAG-2>/g" \ + -e "s/${TAG3%${TAG3#???????}}[0-9a-f]*/<TAG-3>/g" \ + -e "s/ *\$//" +}On one of the gcc farm boxes, a i386 box (gcc45) this fails because sed
gets killed after >500MB of memory use (I was just eyeballing it in htop) on the "reate bundle from special rev: main^!" test. This with GNU
sed 4.2.2.I suspect this regex pattern creates some runaway behavior in sed that's
since been fixed (or maybe it's the glibc regex engine?). The glibc is 2.19-18+deb8u10:
Quoted 13 lines
+ git bundle list-heads special-rev.bdl
+make_user_friendly_and_stable_output
+ sed -e s/[0-9a-f]*/<COMMIT-A>/g -e s/[0-9a-f]*/<COMMIT-B>/g -e s/[0-9a-f]*/<COMMIT-C>/g -e s/[0-9a-f]*/<COMMIT-D>/g -e s/[0-9a-f]*/<COMMIT-E>/g -e s/[0-9a-f]*/<COMMIT-F>/g -e s/[0-9a-f]*/<COMMIT-G>/g -e s/[0-9a-f]*/<COMMIT-H>/g -e s/[0-9a-f]*/<COMMIT-I>/g -e s/[0-9a-f]*/<COMMIT-J>/g -e s/[0-9a-f]*/<COMMIT-K>/g -e s/[0-9a-f]*/<COMMIT-L>/g -e s/[0-9a-f]*/<COMMIT-M>/g -e s/[0-9a-f]*/<COMMIT-N>/g -e s/[0-9a-f]*/<COMMIT-O>/g -e s/[0-9a-f]*/<COMMIT-P>/g -e s/[0-9a-f]*/<TAG-1>/g -e s/[0-9a-f]*/<TAG-2>/g -e s/[0-9a-f]*/<TAG-3>/g -e s/ *$//
sed: couldn't re-allocate memory
I wrote a program on macOS to check memory footprint for sed and perl. See:
https://github.com/jiangxin/compare-sed-perl
Test result:
$ go build && ./compare-sed-perl Command: sed ..., MaxRSS: 901120 Command: gsed ..., MaxRSS: 2056192 Command: perl ..., MaxRSS: 2269184
It seems that sed (both the builtin version on macOS and GNU sed v4.8) has less memory consumed than perl.
Can you run this program on the i386 box (gcc45) to check memory consumed by sed and perl?
If this issue can be resolved by replacing sed with perl, the following patch may help:
--- >8 ---
From: Jiang Xin <[email protected]>
Date: Thu, 27 May 2021 14:31:49 +0800
Subject: [PATCH] test: use perl for complex text replacement
Ævar reported that the function make_user_friendly_and_stable_output() failed on a i386 box (gcc45) in the gcc farm boxes with error:
sed: couldn't re-allocate memory
It seems that sed (GNU sed 4.2.2) gets killed after >500MB of memory use on the "create bundle from special rev: main^!" test.
Call perl instead of sed for complex text replacement.
--- t/t5411/common-functions.sh | 27 ++++++++++++------------ t/t5548-push-porcelain.sh | 20 +++++++++--------- t/t6020-bundle-misc.sh | 42 ++++++++++++++++++------------------- 3 files changed, 45 insertions(+), 44 deletions(-)
t/t5411/common-functions.sh +14 -13
@@ -39,19 +39,20 @@ create_commits_in () { # remove some locale error messages. The emitted human-readable errors are # redundant to the more machine-readable output the tests already assert. make_user_friendly_and_stable_output () { - sed \ - -e "s/ *\$//" \ - -e "s/ */ /g" \ - -e "s/'/\"/g" \ - -e "s/ / /g" \ - -e "s/$A/<COMMIT-A>/g" \ - -e "s/$B/<COMMIT-B>/g" \ - -e "s/$TAG/<TAG-v123>/g" \ - -e "s/$ZERO_OID/<ZERO-OID>/g" \ - -e "s/$(echo $A | cut -c1-7)[0-9a-f]*/<OID-A>/g" \ - -e "s/$(echo $B | cut -c1-7)[0-9a-f]*/<OID-B>/g" \ - -e "s#To $URL_PREFIX/upstream.git#To <URL/of/upstream.git>#" \ - -e "/^error: / d" + perl -ne " + s/ *\$//; + s/ */ /g; + s/'/\"/g; + s/ / /g; + s/$A/<COMMIT-A>/g; + s/$B/<COMMIT-B>/g; + s/$TAG/<TAG-v123>/g; + s/$ZERO_OID/<ZERO-OID>/g; + s/$(echo $A | cut -c1-7)[0-9a-f]*/<OID-A>/g; + s/$(echo $B | cut -c1-7)[0-9a-f]*/<OID-B>/g; + s#To $URL_PREFIX/upstream.git#To <URL/of/upstream.git>#; + next if /^error: .*$/; + print" } filter_out_user_friendly_and_stable_output () {
t/t5548-push-porcelain.sh +10 -10
@@ -44,16 +44,16 @@ create_commits_in () { # without having to worry about future changes of the commit ID and spaces # of the output. make_user_friendly_and_stable_output () { - sed \ - -e "s/ *\$//" \ - -e "s/ */ /g" \ - -e "s/ / /g" \ - -e "s/$A/<COMMIT-A>/g" \ - -e "s/$B/<COMMIT-B>/g" \ - -e "s/$ZERO_OID/<ZERO-OID>/g" \ - -e "s/$(echo $A | cut -c1-7)[0-9a-f]*/<OID-A>/g" \ - -e "s/$(echo $B | cut -c1-7)[0-9a-f]*/<OID-B>/g" \ - -e "s#To $URL_PREFIX/upstream.git#To <URL/of/upstream.git>#" + perl -pe " + s/ *\$//; + s/ */ /g; + s/ / /g; + s/$A/<COMMIT-A>/g; + s/$B/<COMMIT-B>/g; + s/$ZERO_OID/<ZERO-OID>/g; + s/$(echo $A | cut -c1-7)[0-9a-f]*/<OID-A>/g; + s/$(echo $B | cut -c1-7)[0-9a-f]*/<OID-B>/g; + s#To $URL_PREFIX/upstream.git#To <URL/of/upstream.git>#" } setup_upstream_and_workbench () {
t/t6020-bundle-misc.sh +21 -21
48 lines not on this page · raw message
Signature · 1 line
2.32.0.rc0
Re: Runaway sed memory use in test on older sed+glibc (was "Re: [PATCH v6 1/3] test: add helper functions for git-bundle")
Quoted 70 lines · Jiang Xin
Ævar Arnfjörð Bjarmason <[email protected]> 于2021年5月27日周四
Quoted 54 lines · 上午2:51写道:
Quoted 27 lines · Jiang Xin
From: Jiang Xin <[email protected]>
Move git-bundle related functions from t5510 to a library, and this lib
will be shared with a new testcase t6020 which finds a known breakage of
"git-bundle".
[...](quoted diff) +2
-0+ +# Format the output of git commands to make a user-friendly andstable
+# text. We can easily prepare the expect text without having to worry(quoted diff) +14
-0+# about future changes of the commit ID and spaces of the output. +make_user_friendly_and_stable_output () { + sed \ + -e "s/${A%${A#???????}}[0-9a-f]*/<COMMIT-A>/g" \ + -e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" \ + -e "s/${C%${C#???????}}[0-9a-f]*/<COMMIT-C>/g" \ + -e "s/${D%${D#???????}}[0-9a-f]*/<COMMIT-D>/g" \ + -e "s/${E%${E#???????}}[0-9a-f]*/<COMMIT-E>/g" \ + -e "s/${F%${F#???????}}[0-9a-f]*/<COMMIT-F>/g" \ + -e "s/${G%${G#???????}}[0-9a-f]*/<COMMIT-G>/g" \ + -e "s/${H%${H#???????}}[0-9a-f]*/<COMMIT-H>/g" \ + -e "s/${I%${I#???????}}[0-9a-f]*/<COMMIT-I>/g" \ + -e "s/${J%${J#???????}}[0-9a-f]*/<COMMIT-J>/g" \ + -e "s/${K%${K#???????}}[0-9a-f]*/<COMMIT-K>/g" \Review
Quoted 10 lines
(quoted diff) +10
-0+ -e "s/${L%${L#???????}}[0-9a-f]*/<COMMIT-L>/g" \ + -e "s/${M%${M#???????}}[0-9a-f]*/<COMMIT-M>/g" \ + -e "s/${N%${N#???????}}[0-9a-f]*/<COMMIT-N>/g" \ + -e "s/${O%${O#???????}}[0-9a-f]*/<COMMIT-O>/g" \ + -e "s/${P%${P#???????}}[0-9a-f]*/<COMMIT-P>/g" \ + -e "s/${TAG1%${TAG1#???????}}[0-9a-f]*/<TAG-1>/g" \ + -e "s/${TAG2%${TAG2#???????}}[0-9a-f]*/<TAG-2>/g" \ + -e "s/${TAG3%${TAG3#???????}}[0-9a-f]*/<TAG-3>/g" \ + -e "s/ *\$//" +}On one of the gcc farm boxes, a i386 box (gcc45) this fails because sed
gets killed after >500MB of memory use (I was just eyeballing it in htop) on the "reate bundle from special rev: main^!" test. This with GNU
sed 4.2.2.I suspect this regex pattern creates some runaway behavior in sed that's
since been fixed (or maybe it's the glibc regex engine?). The glibc is 2.19-18+deb8u10:Quoted 13 lines
+ git bundle list-heads special-rev.bdl
+make_user_friendly_and_stable_output
+ sed -e s/[0-9a-f]*/<COMMIT-A>/g -e s/[0-9a-f]*/<COMMIT-B>/g -e s/[0-9a-f]*/<COMMIT-C>/g -e s/[0-9a-f]*/<COMMIT-D>/g -e s/[0-9a-f]*/<COMMIT-E>/g -e s/[0-9a-f]*/<COMMIT-F>/g -e s/[0-9a-f]*/<COMMIT-G>/g -e s/[0-9a-f]*/<COMMIT-H>/g -e s/[0-9a-f]*/<COMMIT-I>/g -e s/[0-9a-f]*/<COMMIT-J>/g -e s/[0-9a-f]*/<COMMIT-K>/g -e s/[0-9a-f]*/<COMMIT-L>/g -e s/[0-9a-f]*/<COMMIT-M>/g -e s/[0-9a-f]*/<COMMIT-N>/g -e s/[0-9a-f]*/<COMMIT-O>/g -e s/[0-9a-f]*/<COMMIT-P>/g -e s/[0-9a-f]*/<TAG-1>/g -e s/[0-9a-f]*/<TAG-2>/g -e s/[0-9a-f]*/<TAG-3>/g -e s/ *$//
sed: couldn't re-allocate memory
Quoted 4 lines
I wrote a program on macOS to check memory footprint for sed and perl. See:
Interesting use of Go for as a /usr/bin/time -v replacement :)
After changing your int64 to int32 and digging up how to cross-compile Go I get similar results, it's because your test has actual short SHA-1s in the "-e 's///g'"'s, but notice how in the trace I have it's e.g. "s/[0-9a-f]*/<COMMIT-A>/g".
That's the problem, so that Go command won't reproduce it. Anyway, changing the test to emit to "input" first and running this shows it:
avar@gcc45:/run/user/1632/git/t/trash directory.t6020-bundle-misc$ /usr/bin/time -v sed -e 's/[0-9a-f]*/<COMMIT-A>/g' -e 's/[0-9a-f]*/<COMMIT-B>/g' -e 's/[0-9a-f]*/<COMMIT-C>/g' -e 's/[0-9a-f]*/<COMMIT-D>/g' -e 's/[0-9a-f]*/<COMMIT-E>/g' -e 's/[0-9a-f]*/<COMMIT-F>/g' -e 's/[0-9a-f]*/<COMMIT-G>/g' -e 's/[0-9a-f]*/<COMMIT-H>/g' -e 's/[0-9a-f]*/<COMMIT-I>/g' -e 's/[0-9a-f]*/<COMMIT-J>/g' -e 's/[0-9a-f]*/<COMMIT-K>/g' -e 's/[0-9a-f]*/<COMMIT-L>/g' -e 's/[0-9a-f]*/<COMMIT-M>/g' -e 's/[0-9a-f]*/<COMMIT-N>/g' -e 's/[0-9a-f]*/<COMMIT-O>/g' -e 's/[0-9a-f]*/<COMMIT-P>/g' -e 's/[0-9a-f]*/<TAG-1>/g' -e 's/[0-9a-f]*/<TAG-2>/g' -e 's/[0-9a-f]*/<TAG-3>/g' -e 's/ *$//' <input
sed: couldn't re-allocate memory
Command exited with non-zero status 4
Command being timed: "sed -e s/[0-9a-f]*/<COMMIT-A>/g -e s/[0-9a-f]*/<COMMIT-B>/g -e s/[0-9a-f]*/<COMMIT-C>/g -e s/[0-9a-f]*/<COMMIT-D>/g -e s/[0-9a-f]*/<COMMIT-E>/g -e s/[0-9a-f]*/<COMMIT-F>/g -e s/[0-9a-f]*/<COMMIT-G>/g -e s/[0-9a-f]*/<COMMIT-H>/g -e s/[0-9a-f]*/<COMMIT-I>/g -e s/[0-9a-f]*/<COMMIT-J>/g -e s/[0-9a-f]*/<COMMIT-K>/g -e s/[0-9a-f]*/<COMMIT-L>/g -e s/[0-9a-f]*/<COMMIT-M>/g -e s/[0-9a-f]*/<COMMIT-N>/g -e s/[0-9a-f]*/<COMMIT-O>/g -e s/[0-9a-f]*/<COMMIT-P>/g -e s/[0-9a-f]*/<TAG-1>/g -e s/[0-9a-f]*/<TAG-2>/g -e s/[0-9a-f]*/<TAG-3>/g -e s/ *$//" User time (seconds): 130.00
System time (seconds): 2.42
Percent of CPU this job got: 100%
Elapsed (wall clock) time (h:mm:ss or m:ss): 2:12.41
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 1030968
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 257333
Voluntary context switches: 1
Involuntary context switches: 12578
Swaps: 0
File system inputs: 0
File system outputs: 0
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 4
But no, the issue as it turns out is not Perl v.s. Sed, it's that there's some bug in the shellscript / tooling version (happens with both dash 0.5.7-4 and bash 4.3-11+deb8u2 on that box) where those expansions like ${A%${A#??????0?}} resolve to nothing.
So if we make that:
cat >input &&
cat input >&2 &&
sed -e "s/${A%${A#??????0?}}[0-9a-f]*/<COMMIT-A>/g" <input >input.tmp && mv input.tmp input &&
cat input >&2 &&
sed -e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" <input >input.tmp && mv input.tmp input &&
cat input >&2 &&We get things like:
+ sed -e s/[0-9a-f]*/<COMMIT-A>/g
+ mv input.tmp input
+ cat input
<COMMIT-A> <COMMIT-A>r<COMMIT-A>s<COMMIT-A>/<COMMIT-A>h<COMMIT-A>s<COMMIT-A>/<COMMIT-A>m<COMMIT-A>i<COMMIT-A>n<COMMIT-A>
+ sed -e s/[0-9a-f]*/<COMMIT-B>/g
+ mv input.tmp input
+ cat input
<COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B> <COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B>r<COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B>s<COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B>/<COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B>h<COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B>s<COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B>/<COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B>m<COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B>i<COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B>n<COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B> [...]
etc. I.e. it's the sed expression itself that's the issue. I.e. you should be able to reproduce this locally with something like:
echo 0 | sed -e 's/[0-9]*/<BEGIN>0<END>/g' -e 's/[0-9]*/<BEGIN>0<END>/g' -e 's/[0-9]*/<BEGIN>0<END>/g' -e 's/[0-9]*/<BEGIN>0<END>/g' -e 's/[0-9]*/<BEGIN>0<END>/g' -e 's/[0-9]*/<BEGIN>0<END>/g' -e 's/[0-9]*/<BEGIN>0<END>/g' -e 's/[0-9]*/<BEGIN>0<END>/g'
If not just copy the -e a few more times.
Anyway, looking at this whole test file with fresh eyes this pattern seems very strange. You duplicated most of test_commit with this test_commit_setvar. It's a bit more verbosity but why not just use:
test_commit ...
A=$(git rev-parse HEAD)Or teach test_commit a --rev-parse option or something and:
A=$(test_commit ...)
This make_user_friendly_and_stable_output then actually loses information, e.g. sometimes the bundle output you're testing emits trailing spaces, but the normalization function overzelously trims that.
I think this whole thing would be much simpler with the above and then something like:
@@ -146,7 +126,8 @@ test_expect_success 'setup' '
# branch main: merge commit I & J
git checkout main &&
- test_commit_setvar --merge I topic/1 "Merge commit I" &&
+ git merge --no-edit --no-ff -m"Merge commit I" topic/1 &&
+ I=$(git rev-parse HEAD) &&
test_commit_setvar --merge J refs/pull/2/head "Merge commit J" &&
# branch main: commit K
@@ -172,18 +153,18 @@ test_expect_success 'create bundle from special rev: main^!' '
git bundle list-heads special-rev.bdl |
make_user_friendly_and_stable_output >actual &&
- cat >expect <<-\EOF &&
- <COMMIT-P> refs/heads/main
+ cat >expect <<-EOF &&
+ $P refs/heads/main
EOF
test_cmp expect actual &&
Or just add a --merge option to test_commit itself.
Re: Runaway sed memory use in test on older sed+glibc (was "Re: [PATCH v6 1/3] test: add helper functions for git-bundle")
Quoted 10 lines · Ævar Arnfjörð Bjarmason
Anyway, looking at this whole test file with fresh eyes this pattern seems very strange. You duplicated most of
test_commitwith thistest_commit_setvar. It's a bit more verbosity but why not just use:test_commit ... A=$(git rev-parse HEAD)Or teach
test_commita --rev-parse option or something and:A=$(test_commit ...)
The latter would be nice (and maybe even could be the default, as the stdout is usually just going to --verbose output anyway). But I don't think it works, because the $() is a subshell, so we lose the side effect of test_tick incrementing the timestamp counter.
I wasn't following this thread carefully, but your suggestion to just do "A=$(git rev-parse HEAD)" seems quite readable to me.
-Peff
Re: Runaway sed memory use in test on older sed+glibc (was "Re: [PATCH v6 1/3] test: add helper functions for git-bundle")
In reply to Ævar Arnfjörð BjarmasonQuoted 73 lines · Ævar Arnfjörð Bjarmason
Quoted 70 lines · Jiang Xin
Ævar Arnfjörð Bjarmason <[email protected]> 于2021年5月27日周四
Quoted 67 lines · 上午2:51写道:
Quoted 27 lines · Jiang Xin
From: Jiang Xin <[email protected]>
Move git-bundle related functions from t5510 to a library, and this lib
will be shared with a new testcase t6020 which finds a known breakage of
"git-bundle".
[...](quoted diff) +2
-0+ +# Format the output of git commands to make a user-friendly andstable
+# text. We can easily prepare the expect text without having to worry(quoted diff) +14
-0+# about future changes of the commit ID and spaces of the output. +make_user_friendly_and_stable_output () { + sed \ + -e "s/${A%${A#???????}}[0-9a-f]*/<COMMIT-A>/g" \ + -e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" \ + -e "s/${C%${C#???????}}[0-9a-f]*/<COMMIT-C>/g" \ + -e "s/${D%${D#???????}}[0-9a-f]*/<COMMIT-D>/g" \ + -e "s/${E%${E#???????}}[0-9a-f]*/<COMMIT-E>/g" \ + -e "s/${F%${F#???????}}[0-9a-f]*/<COMMIT-F>/g" \ + -e "s/${G%${G#???????}}[0-9a-f]*/<COMMIT-G>/g" \ + -e "s/${H%${H#???????}}[0-9a-f]*/<COMMIT-H>/g" \ + -e "s/${I%${I#???????}}[0-9a-f]*/<COMMIT-I>/g" \ + -e "s/${J%${J#???????}}[0-9a-f]*/<COMMIT-J>/g" \ + -e "s/${K%${K#???????}}[0-9a-f]*/<COMMIT-K>/g" \Review
Quoted 10 lines
(quoted diff) +10
-0+ -e "s/${L%${L#???????}}[0-9a-f]*/<COMMIT-L>/g" \ + -e "s/${M%${M#???????}}[0-9a-f]*/<COMMIT-M>/g" \ + -e "s/${N%${N#???????}}[0-9a-f]*/<COMMIT-N>/g" \ + -e "s/${O%${O#???????}}[0-9a-f]*/<COMMIT-O>/g" \ + -e "s/${P%${P#???????}}[0-9a-f]*/<COMMIT-P>/g" \ + -e "s/${TAG1%${TAG1#???????}}[0-9a-f]*/<TAG-1>/g" \ + -e "s/${TAG2%${TAG2#???????}}[0-9a-f]*/<TAG-2>/g" \ + -e "s/${TAG3%${TAG3#???????}}[0-9a-f]*/<TAG-3>/g" \ + -e "s/ *\$//" +}On one of the gcc farm boxes, a i386 box (gcc45) this fails because sed
gets killed after >500MB of memory use (I was just eyeballing it in htop) on the "reate bundle from special rev: main^!" test. This with GNU
sed 4.2.2.I suspect this regex pattern creates some runaway behavior in sed that's
since been fixed (or maybe it's the glibc regex engine?). The glibc is 2.19-18+deb8u10:+ git bundle list-heads special-rev.bdl
+make_user_friendly_and_stable_output
+ sed -e s/[0-9a-f]*/<COMMIT-A>/g -e s/[0-9a-f]*/<COMMIT-B>/g -e s/[0-9a-f]*/<COMMIT-C>/g -e s/[0-9a-f]*/<COMMIT-D>/g -e s/[0-9a-f]*/<COMMIT-E>/g -e s/[0-9a-f]*/<COMMIT-F>/g -e s/[0-9a-f]*/<COMMIT-G>/g -e s/[0-9a-f]*/<COMMIT-H>/g -e s/[0-9a-f]*/<COMMIT-I>/g -e s/[0-9a-f]*/<COMMIT-J>/g -e s/[0-9a-f]*/<COMMIT-K>/g -e s/[0-9a-f]*/<COMMIT-L>/g -e s/[0-9a-f]*/<COMMIT-M>/g -e s/[0-9a-f]*/<COMMIT-N>/g -e s/[0-9a-f]*/<COMMIT-O>/g -e s/[0-9a-f]*/<COMMIT-P>/g -e s/[0-9a-f]*/<TAG-1>/g -e s/[0-9a-f]*/<TAG-2>/g -e s/[0-9a-f]*/<TAG-3>/g -e s/ *$//
sed: couldn't re-allocate memory
Quoted 6 lines
Quoted 4 lines
I wrote a program on macOS to check memory footprint for sed and perl. See:
Interesting use of Go for as a
/usr/bin/time-v replacement :)
Here's a Ruby version:
https://dpaste.com/FYT2QKHJE
I'm not sure if will be useful in this particular case, but Ruby code always ends up simpler ;)
Signature · 1 line
Felipe Contreras
Re: Runaway sed memory use in test on older sed+glibc (was "Re: [PATCH v6 1/3] test: add helper functions for git-bundle")
In reply to Ævar Arnfjörð BjarmasonQuoted 5 lines · Ævar Arnfjörð Bjarmason <[email protected]> 于2021年5月27日周四 下午8:49写道:
But no, the issue as it turns out is not Perl v.s. Sed, it's that there's some bug in the shellscript / tooling version (happens with both dash 0.5.7-4 and bash 4.3-11+deb8u2 on that box) where those expansions like ${A%${A#??????0?}} resolve to nothing.
That's the root cause. It can be reproduced by running the following test script:
```
#!/bin/sh
# test script: test.sh
test_commit_setvar () {
var=$1 &&
oid=1234567890123456789012345678901234567890 &&
eval $var=$oid
}
test_commit_setvar A
echo "A: $A"
echo "Abbrev of A: ${A%${A#???????}}"
```By running different version of dash, we can see that dash 0.5.7 fail the test:
``` $ /opt/dash/0.5.11/bin/dash test.sh A: 1234567890123456789012345678901234567890 Abbrev of A: 1234567 $ /opt/dash/0.5.7/bin/dash test.sh A: 1234567890123456789012345678901234567890 Abbrev of A: ```
This issue can be fixed using the following example:
```
#!/bin/sh
test_commit_setvar () {
var=$1 &&
oid=1234567890123456789012345678901234567890 &&
suffix=${oid#???????} &&
oid=${oid%$suffix} &&
eval $var=$oid
}
test_commit_setvar A
echo "Abbrev of A: $A"
```Quoted 6 lines
Anyway, looking at this whole test file with fresh eyes this pattern seems very strange. You duplicated most of
test_commitwith thistest_commit_setvar. It's a bit more verbosity but why not just use:test_commit ... A=$(git rev-parse HEAD)
The function "test_commit()" in "test-lib-function.sh" always creates tags and it cannot make merge commit. So I rewrite a new function which reuse the scaffold of "test_commit".
BTW, sorry for the late reply, will send patch later.
--
Jiang Xin
Re: Runaway sed memory use in test on older sed+glibc (was "Re: [PATCH v6 1/3] test: add helper functions for git-bundle")
In reply to Felipe ContrerasQuoted 7 lines · Felipe Contreras <[email protected]> 于2021年5月28日周五 上午3:19写道:
Quoted 5 lines · Ævar Arnfjörð Bjarmason
Quoted 3 lines · Jiang Xin
I wrote a program on macOS to check memory footprint for sed and perl. See:
Quoted 6 lines
Quoted 3 lines
Quoted 1 line
Interesting use of Go for as a
/usr/bin/time-v replacement :)Here's a Ruby version:
https://dpaste.com/FYT2QKHJE
Nice, it's much simpler.
Quoted 5 lines
I'm not sure if will be useful in this particular case, but Ruby code always ends up simpler ;)
--
Felipe Contreras
Re: Runaway sed memory use in test on older sed+glibc (was "Re: [PATCH v6 1/3] test: add helper functions for git-bundle")
In reply to Jiang XinQuoted 39 lines · Jiang Xin
Quoted 5 lines · Ævar Arnfjörð Bjarmason <[email protected]> 于2021年5月27日周四 下午8:49写道:
But no, the issue as it turns out is not Perl v.s. Sed, it's that there's some bug in the shellscript / tooling version (happens with both dash 0.5.7-4 and bash 4.3-11+deb8u2 on that box) where those expansions like ${A%${A#??????0?}} resolve to nothing.
That's the root cause. It can be reproduced by running the following test script:
``` #!/bin/sh # test script: test.sh test_commit_setvar () { var=$1 && oid=1234567890123456789012345678901234567890 && eval $var=$oid } test_commit_setvar A echo "A: $A" echo "Abbrev of A: ${A%${A#???????}}" ```By running different version of dash, we can see that dash 0.5.7 fail the test:
``` $ /opt/dash/0.5.11/bin/dash test.sh A: 1234567890123456789012345678901234567890 Abbrev of A: 1234567 $ /opt/dash/0.5.7/bin/dash test.sh A: 1234567890123456789012345678901234567890 Abbrev of A: ```This issue can be fixed using the following example:
Quoted 14 lines
```
#!/bin/sh
test_commit_setvar () {
var=$1 &&
oid=1234567890123456789012345678901234567890 &&
suffix=${oid#???????} &&
oid=${oid%$suffix} &&
eval $var=$oid
}
test_commit_setvar A
echo "Abbrev of A: $A"
```*nod*
Quoted 10 lines
Quoted 6 lines
Anyway, looking at this whole test file with fresh eyes this pattern seems very strange. You duplicated most of
test_commitwith thistest_commit_setvar. It's a bit more verbosity but why not just use:test_commit ... A=$(git rev-parse HEAD)The function "
test_commit()" in "test-lib-function.sh" always creates tags and it cannot make merge commit. So I rewrite a new function which reuse the scaffold of "test_commit".
It's had a --no-tag since 3803a3a099 (t: add --no-tag option to test_commit, 2021-02-09). I also have patches in "next" to add more options, you can just add more, having a --merge and maybe a way to tell it to eval the rev-parse into a given variable seem like sensible additions.
Quoted 1 line
BTW, sorry for the late reply, will send patch later.
My main point was that looking at this I think it's very much over the complexity v.s. benefit line on the "complexity" side.
Even if there wasn't a --no-tag just using "test_commit" with a "git tag -d" and "commit_X=$(git rev-parse HEAD)" is less magical and more readable.
I.e. the mostly copy/pasted from test-lib-functions.sh function is ~70 lines, the whole setup function is 50 lines.
And as I noted with the whitespace getting lost in the munging the end result is actually less reliable than just doing a test_cmp with $(git rev-parse ...) instead of <COMMIT-XYZ>.
If you were trying to avoid the whitespace warnings then see the "'s/Z$//'" pattern in t0000-basic.sh for how we've usually tested that, i.e. had a "Z" at the end mark intentional whitespace for test_cmp-alike.
There's a big value in the test suite being mostly consistent (which it somewhat isn't, but we're hopefully getting there). I.e. the goal isn't to optimize each test file to be as small as possible, but to e.g. have the next person maintaining it not wondering where <COMMIT-P> comes from, understanding some test_commit-alike that eval's variables into existence, how it's subtly different (if at all) from test_commit etc.
Re: Runaway sed memory use in test on older sed+glibc (was "Re: [PATCH v6 1/3] test: add helper functions for git-bundle")
Quoted 98 lines · Ævar Arnfjörð Bjarmason <[email protected]> 于2021年6月1日周二 下午8:04写道:
Quoted 39 lines · Jiang Xin
Quoted 5 lines · Ævar Arnfjörð Bjarmason <[email protected]> 于2021年5月27日周四 下午8:49写道:
But no, the issue as it turns out is not Perl v.s. Sed, it's that there's some bug in the shellscript / tooling version (happens with both dash 0.5.7-4 and bash 4.3-11+deb8u2 on that box) where those expansions like ${A%${A#??????0?}} resolve to nothing.
That's the root cause. It can be reproduced by running the following test script:
``` #!/bin/sh # test script: test.sh test_commit_setvar () { var=$1 && oid=1234567890123456789012345678901234567890 && eval $var=$oid } test_commit_setvar A echo "A: $A" echo "Abbrev of A: ${A%${A#???????}}" ```By running different version of dash, we can see that dash 0.5.7 fail the test:
``` $ /opt/dash/0.5.11/bin/dash test.sh A: 1234567890123456789012345678901234567890 Abbrev of A: 1234567 $ /opt/dash/0.5.7/bin/dash test.sh A: 1234567890123456789012345678901234567890 Abbrev of A: ```This issue can be fixed using the following example:
Quoted 14 lines
``` #!/bin/sh test_commit_setvar () { var=$1 && oid=1234567890123456789012345678901234567890 && suffix=${oid#???????} && oid=${oid%$suffix} && eval $var=$oid } test_commit_setvar A echo "Abbrev of A: $A" ```*nod*
Quoted 10 lines
Quoted 6 lines
Anyway, looking at this whole test file with fresh eyes this pattern seems very strange. You duplicated most of
test_commitwith thistest_commit_setvar. It's a bit more verbosity but why not just use:test_commit ... A=$(git rev-parse HEAD)The function "
test_commit()" in "test-lib-function.sh" always creates tags and it cannot make merge commit. So I rewrite a new function which reuse the scaffold of "test_commit".It's had a --no-tag since
3803a3a099(t: add --no-tag option totest_commit, 2021-02-09). I also have patches in "next" to add more options, you can just add more, having a --merge and maybe a way to tell it to eval the rev-parse into a given variable seem like sensible additions.Quoted 1 line
BTW, sorry for the late reply, will send patch later.
My main point was that looking at this I think it's very much over the complexity v.s. benefit line on the "complexity" side.
Even if there wasn't a --no-tag just using "
test_commit" with a "git tag -d" and "commit_X=$(git rev-parse HEAD)" is less magical and more readable.I.e. the mostly copy/pasted from
test-lib-functions.shfunction is ~70 lines, the whole setup function is 50 lines.And as I noted with the whitespace getting lost in the munging the end result is actually less reliable than just doing a
test_cmpwith $(git rev-parse ...) instead of <COMMIT-XYZ>.If you were trying to avoid the whitespace warnings then see the "'s/Z$//'" pattern in
t0000-basic.shfor how we've usually tested that, i.e. had a "Z" at the end mark intentional whitespace for test_cmp-alike.
Quoted 6 lines
There's a big value in the test suite being mostly consistent (which it somewhat isn't, but we're hopefully getting there). I.e. the goal isn't to optimize each test file to be as small as possible, but to e.g. have the next person maintaining it not wondering where <COMMIT-P> comes from, understanding some test_commit-alike that eval's variables into existence, how it's subtly different (if at all) from
test_commitetc.
Will send a patch for quick fix for t6020 which is broken on older version of bash.
After changes on "test_commit()" of "test-lib-function.sh" has been merge to master branch, I will try to refactor t6020 again to remove test_commit_setvar() and reuse test_commit().
PATCH v4 1/2 t6020: fix bash incompatible issue
From: Jiang Xin <[email protected]>
Ævar reported that the function make_user_friendly_and_stable_output() failed on a i386 box (gcc45) in the gcc farm boxes with error:
sed: couldn't re-allocate memory
It turns out that older versions of bash (4.3) or dash (0.5.7) cannot evaluate expression like ${A%${A#???????}} used to get the leading 7 characters of variable A.
Replace the complex expressions so that t6020 works on older version of bash or dash.
--- t/t6020-bundle-misc.sh | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-)
t/t6020-bundle-misc.sh +21 -20
@@ -77,7 +77,8 @@ test_commit_setvar () { git ${indir:+ -C "$indir"} commit $signoff -m "$1" && oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD) fi && - eval $var=$oid + suffix=${oid#???????} && + eval $var=${oid%$suffix} } # Format the output of git commands to make a user-friendly and stable
@@ -85,25 +86,25 @@ test_commit_setvar () { # about future changes of the commit ID and spaces of the output. make_user_friendly_and_stable_output () { sed \ - -e "s/${A%${A#???????}}[0-9a-f]*/<COMMIT-A>/g" \ - -e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" \ - -e "s/${C%${C#???????}}[0-9a-f]*/<COMMIT-C>/g" \ - -e "s/${D%${D#???????}}[0-9a-f]*/<COMMIT-D>/g" \ - -e "s/${E%${E#???????}}[0-9a-f]*/<COMMIT-E>/g" \ - -e "s/${F%${F#???????}}[0-9a-f]*/<COMMIT-F>/g" \ - -e "s/${G%${G#???????}}[0-9a-f]*/<COMMIT-G>/g" \ - -e "s/${H%${H#???????}}[0-9a-f]*/<COMMIT-H>/g" \ - -e "s/${I%${I#???????}}[0-9a-f]*/<COMMIT-I>/g" \ - -e "s/${J%${J#???????}}[0-9a-f]*/<COMMIT-J>/g" \ - -e "s/${K%${K#???????}}[0-9a-f]*/<COMMIT-K>/g" \ - -e "s/${L%${L#???????}}[0-9a-f]*/<COMMIT-L>/g" \ - -e "s/${M%${M#???????}}[0-9a-f]*/<COMMIT-M>/g" \ - -e "s/${N%${N#???????}}[0-9a-f]*/<COMMIT-N>/g" \ - -e "s/${O%${O#???????}}[0-9a-f]*/<COMMIT-O>/g" \ - -e "s/${P%${P#???????}}[0-9a-f]*/<COMMIT-P>/g" \ - -e "s/${TAG1%${TAG1#???????}}[0-9a-f]*/<TAG-1>/g" \ - -e "s/${TAG2%${TAG2#???????}}[0-9a-f]*/<TAG-2>/g" \ - -e "s/${TAG3%${TAG3#???????}}[0-9a-f]*/<TAG-3>/g" \ + -e "s/$A[0-9a-f]*/<COMMIT-A>/g" \ + -e "s/$B[0-9a-f]*/<COMMIT-B>/g" \ + -e "s/$C[0-9a-f]*/<COMMIT-C>/g" \ + -e "s/$D[0-9a-f]*/<COMMIT-D>/g" \ + -e "s/$E[0-9a-f]*/<COMMIT-E>/g" \ + -e "s/$F[0-9a-f]*/<COMMIT-F>/g" \ + -e "s/$G[0-9a-f]*/<COMMIT-G>/g" \ + -e "s/$H[0-9a-f]*/<COMMIT-H>/g" \ + -e "s/$I[0-9a-f]*/<COMMIT-I>/g" \ + -e "s/$J[0-9a-f]*/<COMMIT-J>/g" \ + -e "s/$K[0-9a-f]*/<COMMIT-K>/g" \ + -e "s/$L[0-9a-f]*/<COMMIT-L>/g" \ + -e "s/$M[0-9a-f]*/<COMMIT-M>/g" \ + -e "s/$N[0-9a-f]*/<COMMIT-N>/g" \ + -e "s/$O[0-9a-f]*/<COMMIT-O>/g" \ + -e "s/$P[0-9a-f]*/<COMMIT-P>/g" \ + -e "s/$TAG1[0-9a-f]*/<TAG-1>/g" \ + -e "s/$TAG2[0-9a-f]*/<TAG-2>/g" \ + -e "s/$TAG3[0-9a-f]*/<TAG-3>/g" \ -e "s/ *\$//" }
Signature · 1 line
2.32.0.rc0
PATCH v4 2/2 t6020: do not mangle trailing spaces in output
In reply to Jiang XinFrom: Jiang Xin <[email protected]>
We used to call make_user_friendly_and_stable_output to mangle trailing spaces in output before comparing with the expect file. Ævar recommends generating expect file using pattern "'s/Z$//'" to compare expect file with raw output.
--- t/t6020-bundle-misc.sh | 89 ++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 43 deletions(-)
t/t6020-bundle-misc.sh +46 -43
@@ -83,7 +83,7 @@ test_commit_setvar () { # Format the output of git commands to make a user-friendly and stable # text. We can easily prepare the expect text without having to worry -# about future changes of the commit ID and spaces of the output. +# about future changes of the commit ID. make_user_friendly_and_stable_output () { sed \ -e "s/$A[0-9a-f]*/<COMMIT-A>/g" \
@@ -104,8 +104,11 @@ make_user_friendly_and_stable_output () { -e "s/$P[0-9a-f]*/<COMMIT-P>/g" \ -e "s/$TAG1[0-9a-f]*/<TAG-1>/g" \ -e "s/$TAG2[0-9a-f]*/<TAG-2>/g" \ - -e "s/$TAG3[0-9a-f]*/<TAG-3>/g" \ - -e "s/ *\$//" + -e "s/$TAG3[0-9a-f]*/<TAG-3>/g" +} + +format_and_save_expect () { + sed -e 's/^> //' -e 's/Z$//' >expect } # (C) (D, pull/1/head, topic/1)
@@ -180,11 +183,11 @@ test_expect_success 'create bundle from special rev: main^!' ' git bundle verify special-rev.bdl | make_user_friendly_and_stable_output >actual && - cat >expect <<-\EOF && - The bundle contains this ref: - <COMMIT-P> refs/heads/main - The bundle requires this ref: - <COMMIT-O> + format_and_save_expect <<-\EOF && + > The bundle contains this ref: + > <COMMIT-P> refs/heads/main + > The bundle requires this ref: + > <COMMIT-O> Z EOF test_cmp expect actual &&
@@ -201,12 +204,12 @@ test_expect_success 'create bundle with --max-count option' ' git bundle verify max-count.bdl | make_user_friendly_and_stable_output >actual && - cat >expect <<-\EOF && - The bundle contains these 2 refs: - <COMMIT-P> refs/heads/main - <TAG-1> refs/tags/v1 - The bundle requires this ref: - <COMMIT-O> + format_and_save_expect <<-\EOF && + > The bundle contains these 2 refs: + > <COMMIT-P> refs/heads/main + > <TAG-1> refs/tags/v1 + > The bundle requires this ref: + > <COMMIT-O> Z EOF test_cmp expect actual &&
@@ -226,16 +229,16 @@ test_expect_success 'create bundle with --since option' ' git bundle verify since.bdl | make_user_friendly_and_stable_output >actual && - cat >expect <<-\EOF && - The bundle contains these 5 refs: - <COMMIT-P> refs/heads/main - <COMMIT-N> refs/heads/release - <TAG-2> refs/tags/v2 - <TAG-3> refs/tags/v3 - <COMMIT-P> HEAD - The bundle requires these 2 refs: - <COMMIT-M> - <COMMIT-K> + format_and_save_expect <<-\EOF && + > The bundle contains these 5 refs: + > <COMMIT-P> refs/heads/main + > <COMMIT-N> refs/heads/release + > <TAG-2> refs/tags/v2 + > <TAG-3> refs/tags/v3 + > <COMMIT-P> HEAD + > The bundle requires these 2 refs: + > <COMMIT-M> Z + > <COMMIT-K> Z EOF test_cmp expect actual &&
@@ -294,13 +297,13 @@ test_expect_success 'create bundle 2 - has prerequisites' ' --stdin \ release <input && - cat >expect <<-\EOF && - The bundle contains this ref: - <COMMIT-N> refs/heads/release - The bundle requires these 3 refs: - <COMMIT-D> - <COMMIT-E> - <COMMIT-G> + format_and_save_expect <<-\EOF && + > The bundle contains this ref: + > <COMMIT-N> refs/heads/release + > The bundle requires these 3 refs: + > <COMMIT-D> Z + > <COMMIT-E> Z + > <COMMIT-G> Z EOF git bundle verify 2.bdl |
@@ -318,11 +321,11 @@ test_expect_success 'create bundle 2 - has prerequisites' ' test_expect_success 'fail to verify bundle without prerequisites' ' git init --bare test1.git && - cat >expect <<-\EOF && - error: Repository lacks these prerequisite commits: - error: <COMMIT-D> - error: <COMMIT-E> - error: <COMMIT-G> + format_and_save_expect <<-\EOF && + > error: Repository lacks these prerequisite commits: + > error: <COMMIT-D> Z + > error: <COMMIT-E> Z + > error: <COMMIT-G> Z EOF test_must_fail git -C test1.git bundle verify ../2.bdl 2>&1 |
@@ -353,13 +356,13 @@ test_expect_success 'create bundle 3 - two refs, same object' ' --stdin \ main HEAD <input && - cat >expect <<-\EOF && - The bundle contains these 2 refs: - <COMMIT-P> refs/heads/main - <COMMIT-P> HEAD - The bundle requires these 2 refs: - <COMMIT-M> - <COMMIT-K> + format_and_save_expect <<-\EOF && + > The bundle contains these 2 refs: + > <COMMIT-P> refs/heads/main + > <COMMIT-P> HEAD + > The bundle requires these 2 refs: + > <COMMIT-M> Z + > <COMMIT-K> Z EOF git bundle verify 3.bdl |
Signature · 1 line
2.32.0.rc0
Re: [PATCH 2/2] t6020: do not mangle trailing spaces in output
Quoted 6 lines · Jiang Xin
From: Jiang Xin <[email protected]>
We used to call
make_user_friendly_and_stable_outputto mangle trailing spaces in output before comparing with the expect file. Ævar recommends generating expect file using pattern "'s/Z$//'" to compare expect file with raw output.
I've tested both of these patches and they fix the reported issue on that gcc45 machine.
Quoted 39 lines
Suggested-by Ævar Arnfjörð Bjarmason <[email protected]>Signed-off-by Jiang Xin <[email protected]>--- t/t6020-bundle-misc.sh | 89 ++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 43 deletions(-)t/t6020-bundle-misc.sh +6
-5@@ -83,7 +83,7 @@ test_commit_setvar () { # Format the output of git commands to make a user-friendly and stable # text. We can easily prepare the expect text without having to worry -# about future changes of the commit ID and spaces of the output. +# about future changes of the commit ID. make_user_friendly_and_stable_output () { sed \ -e "s/$A[0-9a-f]*/<COMMIT-A>/g" \@@ -104,8 +104,11 @@ make_user_friendly_and_stable_output () { -e "s/$P[0-9a-f]*/<COMMIT-P>/g" \ -e "s/$TAG1[0-9a-f]*/<TAG-1>/g" \ -e "s/$TAG2[0-9a-f]*/<TAG-2>/g" \ - -e "s/$TAG3[0-9a-f]*/<TAG-3>/g" \ - -e "s/ *\$//" + -e "s/$TAG3[0-9a-f]*/<TAG-3>/g" +} + +format_and_save_expect () { + sed -e 's/^> //' -e 's/Z$//' >expect } # (C) (D, pull/1/head, topic/1)@@ -180,11 +183,11 @@ test_expect_success 'create bundle from special rev: main^!' ' git bundle verify special-rev.bdl | make_user_friendly_and_stable_output >actual && - cat >expect <<-\EOF && - The bundle contains this ref:
t/t6020-bundle-misc.sh test_expect_success 'create bundle from special rev: main^!' ' · line 192
Quoted 10 lines
t/t6020-bundle-misc.sh +5
-3- <COMMIT-P> refs/heads/main - The bundle requires this ref: - <COMMIT-O> + format_and_save_expect <<-\EOF && + > The bundle contains this ref: + > <COMMIT-P> refs/heads/main + > The bundle requires this ref: + > <COMMIT-O> Z EOF test_cmp expect actual &&
I think for this test it would be better to just have the "Z" suffix and leave off prefixing all the lines with ">". That's done in t0000.sh because we have various empty lines, but here that's not the case.
I don't think that nit is worth a re-roll, and as you noted you're doing some larger changes to these tests in follow-up patches.
It's just convenient to have the test_cmp expect input as close to copy/paste-able as possible.
v2 cover Fixed t6020 bash compatible issue and fixed wrong sideband suffix issue
From: Jiang Xin <[email protected]>
In addition to fix the bash incompatible issue of t6020, find another issue when try to rewrite t5411 to compare raw command output.
Function "demultiplex_sideband()" will try to split the sideband-2 message by line breaks, and append a suffix to each nonempty line to clear the end of the screen line. But in the following example, there will be no suffix (8 spaces) for "<message-3>":
PKT-LINE(\2 <message-1> CR <message-2> CR <message-3>)
PKT-LINE(\2 CR <message-4> CR <message-5> CR)This is because the line break of "<message-3>" is placed in the next pktline message.
With this fix, we could rewrite t5411 to test raw output of "git push" which has a stable suffix for each remote sideband-2 message.
Jiang Xin (4):
t6020: fix bash incompatible issue
test: refactor create_commits_in() for t5411 and t5548
sideband: append suffix for message whose CR in next pktline
test: compare raw output, not mangle tabs and spaces
sideband.c | 4 + t/t5411/common-functions.sh | 54 +++-- t/t5411/test-0000-standard-git-push.sh | 82 +++---- .../test-0001-standard-git-push--porcelain.sh | 90 ++++---- ...st-0003-pre-receive-declined--porcelain.sh | 8 +- t/t5411/test-0011-no-hook-error.sh | 40 ++-- t/t5411/test-0012-no-hook-error--porcelain.sh | 42 ++-- t/t5411/test-0013-bad-protocol.sh | 62 +++--- t/t5411/test-0014-bad-protocol--porcelain.sh | 80 +++---- t/t5411/test-0020-report-ng.sh | 32 +-- t/t5411/test-0021-report-ng--porcelain.sh | 36 ++-- t/t5411/test-0022-report-unexpect-ref.sh | 26 +-- ...est-0023-report-unexpect-ref--porcelain.sh | 28 +-- t/t5411/test-0024-report-unknown-ref.sh | 18 +- ...test-0025-report-unknown-ref--porcelain.sh | 20 +- t/t5411/test-0026-push-options.sh | 58 ++--- t/t5411/test-0027-push-options--porcelain.sh | 62 +++--- t/t5411/test-0030-report-ok.sh | 20 +- t/t5411/test-0031-report-ok--porcelain.sh | 22 +- t/t5411/test-0032-report-with-options.sh | 186 ++++++++-------- ...est-0033-report-with-options--porcelain.sh | 200 +++++++++--------- t/t5411/test-0034-report-ft.sh | 22 +- t/t5411/test-0035-report-ft--porcelain.sh | 24 +-- ...t-0036-report-multi-rewrite-for-one-ref.sh | 132 ++++++------ ...rt-multi-rewrite-for-one-ref--porcelain.sh | 138 ++++++------ t/t5411/test-0038-report-mixed-refs.sh | 74 +++---- .../test-0039-report-mixed-refs--porcelain.sh | 76 +++---- t/t5411/test-0040-process-all-refs.sh | 80 +++---- .../test-0041-process-all-refs--porcelain.sh | 82 +++---- ...t-0050-proc-receive-refs-with-modifiers.sh | 90 ++++---- t/t5548-push-porcelain.sh | 97 +++++---- t/t6020-bundle-misc.sh | 93 ++++---- 32 files changed, 1047 insertions(+), 1031 deletions(-)
Signature · 1 line
2.32.0.rc0.27.g7b1e85181b
PATCH v2 3/4 sideband: append suffix for message whose CR in next pktline
In reply to Ævar Arnfjörð BjarmasonFrom: Jiang Xin <[email protected]>
When calling "demultiplex_sideband" on a sideband-2 message, will try to split the message by line breaks, and append a suffix to each nonempty line to clear the end of the screen line. But in the following example, there will be no suffix (8 spaces) for "<message-3>":
PKT-LINE(\2 <message-1> CR <message-2> CR <message-3>)
PKT-LINE(\2 CR <message-4> CR <message-5> CR)This is because the line break of "<message-3>" is placed in the next pktline message.
Without this fix, t5411 must remove trailing spaces of the actual output of "git-push" command before comparing.
--- sideband.c | 4 ++++ 1 file changed, 4 insertions(+)
sideband.c +4 -0
@@ -185,6 +185,10 @@ int demultiplex_sideband(const char *me, int status,if (!scratch->len)strbuf_addstr (scratch,DISPLAY_PREFIX ); +else if (!linelen) +/* buf has a leading CR which ends the remaining +* scratch of last round of "demultiplex_sideband" */ +strbuf_addstr (scratch, suffix);if (linelen >0 ) {maybe_colorize_sideband (scratch, b, linelen);strbuf_addstr (scratch, suffix);
Signature · 1 line
2.32.0.rc0.27.g7b1e85181b
PATCH v2 1/4 t6020: fix bash incompatible issue
In reply to Ævar Arnfjörð BjarmasonFrom: Jiang Xin <[email protected]>
Ævar reported that the function make_user_friendly_and_stable_output() failed on a i386 box (gcc45) in the gcc farm boxes with error:
sed: couldn't re-allocate memory
It turns out that older versions of bash (4.3) or dash (0.5.7) cannot evaluate expression like ${A%${A#???????}} used to get the leading 7 characters of variable A.
Replace the complex expressions so that t6020 works on older version of bash or dash.
--- t/t6020-bundle-misc.sh | 50 ++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 19 deletions(-)
t/t6020-bundle-misc.sh +31 -19
@@ -80,30 +80,42 @@ test_commit_setvar () { eval $var=$oid } +get_abbrev_oid () { + oid=$1 && + suffix=${oid#???????} && + oid=${oid%$suffix} && + if test -n "$oid" + then + echo "$oid" + else + echo "undefined-oid" + fi +} + # Format the output of git commands to make a user-friendly and stable # text. We can easily prepare the expect text without having to worry # about future changes of the commit ID and spaces of the output. make_user_friendly_and_stable_output () { sed \ - -e "s/${A%${A#???????}}[0-9a-f]*/<COMMIT-A>/g" \ - -e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" \ - -e "s/${C%${C#???????}}[0-9a-f]*/<COMMIT-C>/g" \ - -e "s/${D%${D#???????}}[0-9a-f]*/<COMMIT-D>/g" \ - -e "s/${E%${E#???????}}[0-9a-f]*/<COMMIT-E>/g" \ - -e "s/${F%${F#???????}}[0-9a-f]*/<COMMIT-F>/g" \ - -e "s/${G%${G#???????}}[0-9a-f]*/<COMMIT-G>/g" \ - -e "s/${H%${H#???????}}[0-9a-f]*/<COMMIT-H>/g" \ - -e "s/${I%${I#???????}}[0-9a-f]*/<COMMIT-I>/g" \ - -e "s/${J%${J#???????}}[0-9a-f]*/<COMMIT-J>/g" \ - -e "s/${K%${K#???????}}[0-9a-f]*/<COMMIT-K>/g" \ - -e "s/${L%${L#???????}}[0-9a-f]*/<COMMIT-L>/g" \ - -e "s/${M%${M#???????}}[0-9a-f]*/<COMMIT-M>/g" \ - -e "s/${N%${N#???????}}[0-9a-f]*/<COMMIT-N>/g" \ - -e "s/${O%${O#???????}}[0-9a-f]*/<COMMIT-O>/g" \ - -e "s/${P%${P#???????}}[0-9a-f]*/<COMMIT-P>/g" \ - -e "s/${TAG1%${TAG1#???????}}[0-9a-f]*/<TAG-1>/g" \ - -e "s/${TAG2%${TAG2#???????}}[0-9a-f]*/<TAG-2>/g" \ - -e "s/${TAG3%${TAG3#???????}}[0-9a-f]*/<TAG-3>/g" \ + -e "s/$(get_abbrev_oid $A)[0-9a-f]*/<COMMIT-A>/g" \ + -e "s/$(get_abbrev_oid $B)[0-9a-f]*/<COMMIT-B>/g" \ + -e "s/$(get_abbrev_oid $C)[0-9a-f]*/<COMMIT-C>/g" \ + -e "s/$(get_abbrev_oid $D)[0-9a-f]*/<COMMIT-D>/g" \ + -e "s/$(get_abbrev_oid $E)[0-9a-f]*/<COMMIT-E>/g" \ + -e "s/$(get_abbrev_oid $F)[0-9a-f]*/<COMMIT-F>/g" \ + -e "s/$(get_abbrev_oid $G)[0-9a-f]*/<COMMIT-G>/g" \ + -e "s/$(get_abbrev_oid $H)[0-9a-f]*/<COMMIT-H>/g" \ + -e "s/$(get_abbrev_oid $I)[0-9a-f]*/<COMMIT-I>/g" \ + -e "s/$(get_abbrev_oid $J)[0-9a-f]*/<COMMIT-J>/g" \ + -e "s/$(get_abbrev_oid $K)[0-9a-f]*/<COMMIT-K>/g" \ + -e "s/$(get_abbrev_oid $L)[0-9a-f]*/<COMMIT-L>/g" \ + -e "s/$(get_abbrev_oid $M)[0-9a-f]*/<COMMIT-M>/g" \ + -e "s/$(get_abbrev_oid $N)[0-9a-f]*/<COMMIT-N>/g" \ + -e "s/$(get_abbrev_oid $O)[0-9a-f]*/<COMMIT-O>/g" \ + -e "s/$(get_abbrev_oid $P)[0-9a-f]*/<COMMIT-P>/g" \ + -e "s/$(get_abbrev_oid $TAG1)[0-9a-f]*/<TAG-1>/g" \ + -e "s/$(get_abbrev_oid $TAG2)[0-9a-f]*/<TAG-2>/g" \ + -e "s/$(get_abbrev_oid $TAG3)[0-9a-f]*/<TAG-3>/g" \ -e "s/ *\$//" }
Signature · 1 line
2.32.0.rc0.27.g7b1e85181b
PATCH v2 2/4 test: refactor create_commits_in() for t5411 and t5548
In reply to Ævar Arnfjörð BjarmasonFrom: Jiang Xin <[email protected]>
Use "test_commit" in "create_commits_in" to create commit and get abbrev object ID by using function "get_abbrev_oid".
--- t/t5411/common-functions.sh | 45 ++++++++--------- t/t5411/test-0000-standard-git-push.sh | 6 +-- .../test-0001-standard-git-push--porcelain.sh | 6 +-- t/t5411/test-0022-report-unexpect-ref.sh | 2 +- ...est-0023-report-unexpect-ref--porcelain.sh | 2 +- t/t5411/test-0032-report-with-options.sh | 8 +-- ...est-0033-report-with-options--porcelain.sh | 8 +-- ...t-0036-report-multi-rewrite-for-one-ref.sh | 10 ++-- ...rt-multi-rewrite-for-one-ref--porcelain.sh | 10 ++-- t/t5411/test-0038-report-mixed-refs.sh | 4 +- .../test-0039-report-mixed-refs--porcelain.sh | 4 +- t/t5411/test-0040-process-all-refs.sh | 8 +-- .../test-0041-process-all-refs--porcelain.sh | 8 +-- ...t-0050-proc-receive-refs-with-modifiers.sh | 4 +- t/t5548-push-porcelain.sh | 49 +++++++++---------- 15 files changed, 84 insertions(+), 90 deletions(-)
t/t5411/common-functions.sh +21 -24
@@ -6,29 +6,28 @@ # NOTE: Never calling this function from a subshell since variable # assignments will disappear when subshell exits. create_commits_in () { - repo="$1" && - if ! parent=$(git -C "$repo" rev-parse HEAD^{} --) - then - parent= - fi && - T=$(git -C "$repo" write-tree) && + repo="$1" && test -d "$repo" || + error "Repository $repo does not exist." shift && while test $# -gt 0 do name=$1 && - test_tick && - if test -z "$parent" - then - oid=$(echo $name | git -C "$repo" commit-tree $T) - else - oid=$(echo $name | git -C "$repo" commit-tree -p $parent $T) - fi && - eval $name=$oid && - parent=$oid && - shift || - return 1 - done && - git -C "$repo" update-ref refs/heads/main $oid + shift && + test_commit -C "$repo" --no-tag "$name" && + eval $name=$(git -C "$repo" rev-parse HEAD) + done +} + +get_abbrev_oid () { + oid=$1 && + suffix=${oid#???????} && + oid=${oid%$suffix} && + if test -n "$oid" + then + echo "$oid" + else + echo "undefined-oid" + fi } # Format the output of git-push, git-show-ref and other commands to make a
@@ -44,12 +43,10 @@ make_user_friendly_and_stable_output () { -e "s/ */ /g" \ -e "s/'/\"/g" \ -e "s/ / /g" \ - -e "s/$A/<COMMIT-A>/g" \ - -e "s/$B/<COMMIT-B>/g" \ - -e "s/$TAG/<TAG-v123>/g" \ + -e "s/$(get_abbrev_oid $A)[0-9a-f]*/<COMMIT-A>/g" \ + -e "s/$(get_abbrev_oid $B)[0-9a-f]*/<COMMIT-B>/g" \ + -e "s/$(get_abbrev_oid $TAG)[0-9a-f]*/<TAG-v123>/g" \ -e "s/$ZERO_OID/<ZERO-OID>/g" \ - -e "s/$(echo $A | cut -c1-7)[0-9a-f]*/<OID-A>/g" \ - -e "s/$(echo $B | cut -c1-7)[0-9a-f]*/<OID-B>/g" \ -e "s#To $URL_PREFIX/upstream.git#To <URL/of/upstream.git>#" \ -e "/^error: / d" }
t/t5411/test-0000-standard-git-push.sh +3 -3
@@ -15,7 +15,7 @@ test_expect_success "git-push ($PROTOCOL)" ' remote: post-receive< <COMMIT-A> <COMMIT-B> refs/heads/main remote: post-receive< <ZERO-OID> <COMMIT-A> refs/heads/next To <URL/of/upstream.git> - <OID-A>..<OID-B> <COMMIT-B> -> main + <COMMIT-A>..<COMMIT-B> <COMMIT-B> -> main * [new branch] HEAD -> next EOF test_cmp expect actual &&
@@ -69,7 +69,7 @@ test_expect_success "non-fast-forward git-push ($PROTOCOL)" ' remote: # post-receive hook remote: post-receive< <COMMIT-A> <COMMIT-B> refs/heads/next To <URL/of/upstream.git> - <OID-A>..<OID-B> <COMMIT-B> -> next + <COMMIT-A>..<COMMIT-B> <COMMIT-B> -> next ! [rejected] main -> main (non-fast-forward) EOF test_cmp expect actual &&
@@ -106,7 +106,7 @@ test_expect_success "git-push -f ($PROTOCOL)" ' remote: post-receive< <ZERO-OID> <COMMIT-A> refs/review/main/topic remote: post-receive< <ZERO-OID> <COMMIT-A> refs/heads/a/b/c To <URL/of/upstream.git> - + <OID-B>...<OID-A> main -> main (forced update) + + <COMMIT-B>...<COMMIT-A> main -> main (forced update) - [deleted] next * [new tag] v123 -> v123 * [new reference] main -> refs/review/main/topic
t/t5411/test-0001-standard-git-push--porcelain.sh +3 -3
@@ -15,7 +15,7 @@ test_expect_success "git-push ($PROTOCOL/porcelain)" ' remote: post-receive< <COMMIT-A> <COMMIT-B> refs/heads/main remote: post-receive< <ZERO-OID> <COMMIT-A> refs/heads/next To <URL/of/upstream.git> - <COMMIT-B>:refs/heads/main <OID-A>..<OID-B> + <COMMIT-B>:refs/heads/main <COMMIT-A>..<COMMIT-B> * HEAD:refs/heads/next [new branch] Done EOF
@@ -71,7 +71,7 @@ test_expect_success "non-fast-forward git-push ($PROTOCOL/porcelain)" ' remote: # post-receive hook remote: post-receive< <COMMIT-A> <COMMIT-B> refs/heads/next To <URL/of/upstream.git> - <COMMIT-B>:refs/heads/next <OID-A>..<OID-B> + <COMMIT-B>:refs/heads/next <COMMIT-A>..<COMMIT-B> ! refs/heads/main:refs/heads/main [rejected] (non-fast-forward) Done EOF
@@ -109,7 +109,7 @@ test_expect_success "git-push -f ($PROTOCOL/porcelain)" ' remote: post-receive< <ZERO-OID> <COMMIT-A> refs/review/main/topic remote: post-receive< <ZERO-OID> <COMMIT-A> refs/heads/a/b/c To <URL/of/upstream.git> - + refs/heads/main:refs/heads/main <OID-B>...<OID-A> (forced update) + + refs/heads/main:refs/heads/main <COMMIT-B>...<COMMIT-A> (forced update) - :refs/heads/next [deleted] * refs/tags/v123:refs/tags/v123 [new tag] * refs/heads/main:refs/review/main/topic [new reference]
t/t5411/test-0022-report-unexpect-ref.sh +1 -1
@@ -26,7 +26,7 @@ test_expect_success "proc-receive: report unexpected ref ($PROTOCOL)" ' remote: # post-receive hook remote: post-receive< <COMMIT-A> <COMMIT-B> refs/heads/main To <URL/of/upstream.git> - <OID-A>..<OID-B> <COMMIT-B> -> main + <COMMIT-A>..<COMMIT-B> <COMMIT-B> -> main ! [remote rejected] HEAD -> refs/for/main/topic (proc-receive failed to report status) EOF test_cmp expect actual &&
t/t5411/test-0023-report-unexpect-ref--porcelain.sh +1 -1
@@ -26,7 +26,7 @@ test_expect_success "proc-receive: report unexpected ref ($PROTOCOL/porcelain)" remote: # post-receive hook remote: post-receive< <COMMIT-A> <COMMIT-B> refs/heads/main To <URL/of/upstream.git> - <COMMIT-B>:refs/heads/main <OID-A>..<OID-B> + <COMMIT-B>:refs/heads/main <COMMIT-A>..<COMMIT-B> ! HEAD:refs/for/main/topic [remote rejected] (proc-receive failed to report status) Done EOF
t/t5411/test-0032-report-with-options.sh +4 -4
@@ -123,7 +123,7 @@ test_expect_success "proc-receive: report option refname and old-oid ($PROTOCOL) remote: # post-receive hook remote: post-receive< <COMMIT-B> <COMMIT-A> refs/pull/123/head To <URL/of/upstream.git> - <OID-B>..<OID-A> HEAD -> refs/pull/123/head + <COMMIT-B>..<COMMIT-A> HEAD -> refs/pull/123/head EOF test_cmp expect actual '
@@ -155,7 +155,7 @@ test_expect_success "proc-receive: report option old-oid ($PROTOCOL)" ' remote: # post-receive hook remote: post-receive< <COMMIT-B> <COMMIT-A> refs/for/main/topic To <URL/of/upstream.git> - <OID-B>..<OID-A> HEAD -> refs/for/main/topic + <COMMIT-B>..<COMMIT-A> HEAD -> refs/for/main/topic EOF test_cmp expect actual '
@@ -189,7 +189,7 @@ test_expect_success "proc-receive: report option old-oid and new-oid ($PROTOCOL) remote: # post-receive hook remote: post-receive< <COMMIT-A> <COMMIT-B> refs/for/main/topic To <URL/of/upstream.git> - <OID-A>..<OID-B> HEAD -> refs/for/main/topic + <COMMIT-A>..<COMMIT-B> HEAD -> refs/for/main/topic EOF test_cmp expect actual '
@@ -243,7 +243,7 @@ test_expect_success "proc-receive: report with multiple rewrites ($PROTOCOL)" ' To <URL/of/upstream.git> * [new reference] HEAD -> refs/pull/123/head * [new reference] HEAD -> refs/for/a/b/c/topic - + <OID-B>...<OID-A> HEAD -> refs/pull/124/head (forced update) + + <COMMIT-B>...<COMMIT-A> HEAD -> refs/pull/124/head (forced update) EOF test_cmp expect actual &&
t/t5411/test-0033-report-with-options--porcelain.sh +4 -4
@@ -127,7 +127,7 @@ test_expect_success "proc-receive: report option refname and old-oid ($PROTOCOL/ remote: # post-receive hook remote: post-receive< <COMMIT-B> <COMMIT-A> refs/pull/123/head To <URL/of/upstream.git> - HEAD:refs/pull/123/head <OID-B>..<OID-A> + HEAD:refs/pull/123/head <COMMIT-B>..<COMMIT-A> Done EOF test_cmp expect actual
@@ -160,7 +160,7 @@ test_expect_success "proc-receive: report option old-oid ($PROTOCOL/porcelain)" remote: # post-receive hook remote: post-receive< <COMMIT-B> <COMMIT-A> refs/for/main/topic To <URL/of/upstream.git> - HEAD:refs/for/main/topic <OID-B>..<OID-A> + HEAD:refs/for/main/topic <COMMIT-B>..<COMMIT-A> Done EOF test_cmp expect actual
@@ -195,7 +195,7 @@ test_expect_success "proc-receive: report option old-oid and new-oid ($PROTOCOL/ remote: # post-receive hook remote: post-receive< <COMMIT-A> <COMMIT-B> refs/for/main/topic To <URL/of/upstream.git> - HEAD:refs/for/main/topic <OID-A>..<OID-B> + HEAD:refs/for/main/topic <COMMIT-A>..<COMMIT-B> Done EOF test_cmp expect actual
@@ -251,7 +251,7 @@ test_expect_success "proc-receive: report with multiple rewrites ($PROTOCOL/porc To <URL/of/upstream.git> * HEAD:refs/pull/123/head [new reference] * HEAD:refs/for/a/b/c/topic [new reference] - + HEAD:refs/pull/124/head <OID-B>...<OID-A> (forced update) + + HEAD:refs/pull/124/head <COMMIT-B>...<COMMIT-A> (forced update) Done EOF test_cmp expect actual &&
t/t5411/test-0036-report-multi-rewrite-for-one-ref.sh +5 -5
@@ -60,9 +60,9 @@ test_expect_success "proc-receive: multiple rewrite for one ref, no refname for remote: post-receive< <ZERO-OID> <COMMIT-A> refs/changes/24/124/1 remote: post-receive< <COMMIT-A> <COMMIT-B> refs/changes/25/125/1 To <URL/of/upstream.git> - <OID-A>..<OID-B> HEAD -> refs/for/main/topic + <COMMIT-A>..<COMMIT-B> HEAD -> refs/for/main/topic * [new reference] HEAD -> refs/changes/24/124/1 - <OID-A>..<OID-B> HEAD -> refs/changes/25/125/1 + <COMMIT-A>..<COMMIT-B> HEAD -> refs/changes/25/125/1 EOF test_cmp expect actual &&
@@ -136,8 +136,8 @@ test_expect_success "proc-receive: multiple rewrites for one ref, no refname for remote: post-receive< <COMMIT-B> <COMMIT-A> refs/changes/25/125/1 To <URL/of/upstream.git> * [new reference] HEAD -> refs/changes/24/124/1 - <OID-A>..<OID-B> HEAD -> refs/for/main/topic - + <OID-B>...<OID-A> HEAD -> refs/changes/25/125/1 (forced update) + <COMMIT-A>..<COMMIT-B> HEAD -> refs/for/main/topic + + <COMMIT-B>...<COMMIT-A> HEAD -> refs/changes/25/125/1 (forced update) EOF test_cmp expect actual &&
@@ -198,7 +198,7 @@ test_expect_success "proc-receive: multiple rewrites for one ref ($PROTOCOL)" ' remote: post-receive< <COMMIT-A> <COMMIT-B> refs/changes/24/124/2 To <URL/of/upstream.git> * [new reference] HEAD -> refs/changes/23/123/1 - <OID-A>..<OID-B> HEAD -> refs/changes/24/124/2 + <COMMIT-A>..<COMMIT-B> HEAD -> refs/changes/24/124/2 EOF test_cmp expect actual &&
t/t5411/test-0037-report-multi-rewrite-for-one-ref--porcelain.sh +5 -5
@@ -45,9 +45,9 @@ test_expect_success "proc-receive: multiple rewrite for one ref, no refname for remote: post-receive< <ZERO-OID> <COMMIT-A> refs/changes/24/124/1 remote: post-receive< <COMMIT-A> <COMMIT-B> refs/changes/25/125/1 To <URL/of/upstream.git> - HEAD:refs/for/main/topic <OID-A>..<OID-B> + HEAD:refs/for/main/topic <COMMIT-A>..<COMMIT-B> * HEAD:refs/changes/24/124/1 [new reference] - HEAD:refs/changes/25/125/1 <OID-A>..<OID-B> + HEAD:refs/changes/25/125/1 <COMMIT-A>..<COMMIT-B> Done EOF test_cmp expect actual &&
@@ -107,8 +107,8 @@ test_expect_success "proc-receive: multiple rewrites for one ref, no refname for remote: post-receive< <COMMIT-B> <COMMIT-A> refs/changes/25/125/1 To <URL/of/upstream.git> * HEAD:refs/changes/24/124/1 [new reference] - HEAD:refs/for/main/topic <OID-A>..<OID-B> - + HEAD:refs/changes/25/125/1 <OID-B>...<OID-A> (forced update) + HEAD:refs/for/main/topic <COMMIT-A>..<COMMIT-B> + + HEAD:refs/changes/25/125/1 <COMMIT-B>...<COMMIT-A> (forced update) Done EOF test_cmp expect actual &&
@@ -155,7 +155,7 @@ test_expect_success "proc-receive: multiple rewrites for one ref ($PROTOCOL/porc remote: post-receive< <COMMIT-A> <COMMIT-B> refs/changes/24/124/2 To <URL/of/upstream.git> * HEAD:refs/changes/23/123/1 [new reference] - HEAD:refs/changes/24/124/2 <OID-A>..<OID-B> + HEAD:refs/changes/24/124/2 <COMMIT-A>..<COMMIT-B> Done EOF test_cmp expect actual &&
t/t5411/test-0038-report-mixed-refs.sh +2 -2
@@ -55,12 +55,12 @@ test_expect_success "proc-receive: report update of mixed refs ($PROTOCOL)" ' remote: post-receive< <ZERO-OID> <COMMIT-A> refs/heads/foo remote: post-receive< <COMMIT-A> <COMMIT-B> refs/for/main/topic To <URL/of/upstream.git> - <OID-A>..<OID-B> <COMMIT-B> -> main + <COMMIT-A>..<COMMIT-B> <COMMIT-B> -> main * [new branch] HEAD -> bar * [new branch] HEAD -> baz * [new reference] HEAD -> refs/for/next/topic2 * [new branch] HEAD -> foo - <OID-A>..<OID-B> HEAD -> refs/for/main/topic + <COMMIT-A>..<COMMIT-B> HEAD -> refs/for/main/topic ! [remote rejected] HEAD -> refs/for/next/topic1 (fail to call Web API) ! [remote rejected] HEAD -> refs/for/next/topic3 (proc-receive failed to report status) EOF
t/t5411/test-0039-report-mixed-refs--porcelain.sh +2 -2
@@ -55,12 +55,12 @@ test_expect_success "proc-receive: report update of mixed refs ($PROTOCOL/porcel remote: post-receive< <ZERO-OID> <COMMIT-A> refs/heads/foo remote: post-receive< <COMMIT-A> <COMMIT-B> refs/for/main/topic To <URL/of/upstream.git> - <COMMIT-B>:refs/heads/main <OID-A>..<OID-B> + <COMMIT-B>:refs/heads/main <COMMIT-A>..<COMMIT-B> * HEAD:refs/heads/bar [new branch] * HEAD:refs/heads/baz [new branch] * HEAD:refs/for/next/topic2 [new reference] * HEAD:refs/heads/foo [new branch] - HEAD:refs/for/main/topic <OID-A>..<OID-B> + HEAD:refs/for/main/topic <COMMIT-A>..<COMMIT-B> ! HEAD:refs/for/next/topic1 [remote rejected] (fail to call Web API) ! HEAD:refs/for/next/topic3 [remote rejected] (proc-receive failed to report status) Done
t/t5411/test-0040-process-all-refs.sh +4 -4
@@ -85,11 +85,11 @@ test_expect_success "proc-receive: process all refs ($PROTOCOL)" ' remote: post-receive< <COMMIT-A> <COMMIT-B> refs/pull/123/head remote: post-receive< <COMMIT-B> <COMMIT-A> refs/pull/124/head To <URL/of/upstream.git> - <OID-A>..<OID-B> <COMMIT-B> -> bar + <COMMIT-A>..<COMMIT-B> <COMMIT-B> -> bar - [deleted] foo - + <OID-B>...<OID-A> HEAD -> main (forced update) - <OID-A>..<OID-B> HEAD -> refs/pull/123/head - + <OID-B>...<OID-A> HEAD -> refs/pull/124/head (forced update) + + <COMMIT-B>...<COMMIT-A> HEAD -> main (forced update) + <COMMIT-A>..<COMMIT-B> HEAD -> refs/pull/123/head + + <COMMIT-B>...<COMMIT-A> HEAD -> refs/pull/124/head (forced update) EOF test_cmp expect actual &&
t/t5411/test-0041-process-all-refs--porcelain.sh +4 -4
@@ -85,11 +85,11 @@ test_expect_success "proc-receive: process all refs ($PROTOCOL/porcelain)" ' remote: post-receive< <COMMIT-A> <COMMIT-B> refs/pull/123/head remote: post-receive< <COMMIT-B> <COMMIT-A> refs/pull/124/head To <URL/of/upstream.git> - <COMMIT-B>:refs/heads/bar <OID-A>..<OID-B> + <COMMIT-B>:refs/heads/bar <COMMIT-A>..<COMMIT-B> - :refs/heads/foo [deleted] - + HEAD:refs/heads/main <OID-B>...<OID-A> (forced update) - HEAD:refs/pull/123/head <OID-A>..<OID-B> - + HEAD:refs/pull/124/head <OID-B>...<OID-A> (forced update) + + HEAD:refs/heads/main <COMMIT-B>...<COMMIT-A> (forced update) + HEAD:refs/pull/123/head <COMMIT-A>..<COMMIT-B> + + HEAD:refs/pull/124/head <COMMIT-B>...<COMMIT-A> (forced update) Done EOF test_cmp expect actual &&
t/t5411/test-0050-proc-receive-refs-with-modifiers.sh +2 -2
@@ -46,7 +46,7 @@ test_expect_success "proc-receive: update branch and new tag ($PROTOCOL)" ' remote: post-receive< <COMMIT-A> <COMMIT-B> refs/pull/123/head remote: post-receive< <ZERO-OID> <TAG-v123> refs/pull/124/head To <URL/of/upstream.git> - <OID-A>..<OID-B> <COMMIT-B> -> refs/pull/123/head + <COMMIT-A>..<COMMIT-B> <COMMIT-B> -> refs/pull/123/head * [new reference] v123 -> refs/pull/124/head EOF test_cmp expect actual &&
@@ -116,7 +116,7 @@ test_expect_success "proc-receive: create/delete branch, and delete tag ($PROTOC remote: post-receive< <ZERO-OID> <COMMIT-A> refs/pull/124/head To <URL/of/upstream.git> - [deleted] refs/pull/123/head - <OID-A>..<OID-B> <COMMIT-B> -> topic + <COMMIT-A>..<COMMIT-B> <COMMIT-B> -> topic - [deleted] v123 * [new reference] <COMMIT-A> -> refs/pull/124/head EOF
t/t5548-push-porcelain.sh +23 -26
79 lines not on this page · raw message
Signature · 1 line
2.32.0.rc0.27.g7b1e85181b
PATCH v2 4/4 test: compare raw output, not mangle tabs and spaces
In reply to Ævar Arnfjörð BjarmasonFrom: Jiang Xin <[email protected]>
We used to call make_user_friendly_and_stable_output to mangle trailing spaces in output before comparing with the expect file. Ævar recommends generating expect file using pattern "'s/Z$//'" to compare expect file with raw output.
--- t/t5411/common-functions.sh | 9 +- t/t5411/test-0000-standard-git-push.sh | 82 +++---- .../test-0001-standard-git-push--porcelain.sh | 90 ++++---- ...st-0003-pre-receive-declined--porcelain.sh | 8 +- t/t5411/test-0011-no-hook-error.sh | 40 ++-- t/t5411/test-0012-no-hook-error--porcelain.sh | 42 ++-- t/t5411/test-0013-bad-protocol.sh | 62 +++--- t/t5411/test-0014-bad-protocol--porcelain.sh | 80 +++---- t/t5411/test-0020-report-ng.sh | 32 +-- t/t5411/test-0021-report-ng--porcelain.sh | 36 ++-- t/t5411/test-0022-report-unexpect-ref.sh | 26 +-- ...est-0023-report-unexpect-ref--porcelain.sh | 28 +-- t/t5411/test-0024-report-unknown-ref.sh | 18 +- ...test-0025-report-unknown-ref--porcelain.sh | 20 +- t/t5411/test-0026-push-options.sh | 58 ++--- t/t5411/test-0027-push-options--porcelain.sh | 62 +++--- t/t5411/test-0030-report-ok.sh | 20 +- t/t5411/test-0031-report-ok--porcelain.sh | 22 +- t/t5411/test-0032-report-with-options.sh | 186 ++++++++-------- ...est-0033-report-with-options--porcelain.sh | 200 +++++++++--------- t/t5411/test-0034-report-ft.sh | 22 +- t/t5411/test-0035-report-ft--porcelain.sh | 24 +-- ...t-0036-report-multi-rewrite-for-one-ref.sh | 132 ++++++------ ...rt-multi-rewrite-for-one-ref--porcelain.sh | 138 ++++++------ t/t5411/test-0038-report-mixed-refs.sh | 74 +++---- .../test-0039-report-mixed-refs--porcelain.sh | 76 +++---- t/t5411/test-0040-process-all-refs.sh | 80 +++---- .../test-0041-process-all-refs--porcelain.sh | 82 +++---- ...t-0050-proc-receive-refs-with-modifiers.sh | 90 ++++---- t/t5548-push-porcelain.sh | 54 ++--- t/t6020-bundle-misc.sh | 45 ++-- 31 files changed, 972 insertions(+), 966 deletions(-)
t/t5411/common-functions.sh +5 -4
@@ -32,17 +32,14 @@ get_abbrev_oid () { # Format the output of git-push, git-show-ref and other commands to make a # user-friendly and stable text. We can easily prepare the expect text -# without having to worry about future changes of the commit ID and spaces +# without having to worry about changes of the commit ID (full or abbrev.) # of the output. Single quotes are replaced with double quotes, because # it is boring to prepare unquoted single quotes in expect text. We also # remove some locale error messages. The emitted human-readable errors are # redundant to the more machine-readable output the tests already assert. make_user_friendly_and_stable_output () { sed \ - -e "s/ *\$//" \ - -e "s/ */ /g" \ -e "s/'/\"/g" \ - -e "s/ / /g" \ -e "s/$(get_abbrev_oid $A)[0-9a-f]*/<COMMIT-A>/g" \ -e "s/$(get_abbrev_oid $B)[0-9a-f]*/<COMMIT-B>/g" \ -e "s/$(get_abbrev_oid $TAG)[0-9a-f]*/<TAG-v123>/g" \
@@ -56,6 +53,10 @@ filter_out_user_friendly_and_stable_output () { sed -n ${1+"$@"} } +format_and_save_expect () { + sed -e 's/^> //' -e 's/Z$//' >expect +} + test_cmp_refs () { indir= if test "$1" = "-C"
t/t5411/test-0000-standard-git-push.sh +41 -41
106 lines not on this page · raw message
t/t5411/test-0001-standard-git-push--porcelain.sh +45 -45
115 lines not on this page · raw message
t/t5411/test-0003-pre-receive-declined--porcelain.sh +4 -4
@@ -14,10 +14,10 @@ test_expect_success "git-push is declined ($PROTOCOL/porcelain)" ' HEAD:refs/heads/next \ >out-$test_count 2>&1 && make_user_friendly_and_stable_output <out-$test_count >actual && - cat >expect <<-EOF && - To <URL/of/upstream.git> - ! <COMMIT-B>:refs/heads/main [remote rejected] (pre-receive hook declined) - ! HEAD:refs/heads/next [remote rejected] (pre-receive hook declined) + format_and_save_expect <<-EOF && + > To <URL/of/upstream.git> + > ! <COMMIT-B>:refs/heads/main [remote rejected] (pre-receive hook declined) + > ! HEAD:refs/heads/next [remote rejected] (pre-receive hook declined) Done EOF test_cmp expect actual &&
t/t5411/test-0011-no-hook-error.sh +20 -20
@@ -7,16 +7,16 @@ test_expect_success "proc-receive: no hook, fail to push special ref ($PROTOCOL) HEAD:refs/for/main/topic \ >out-$test_count 2>&1 && make_user_friendly_and_stable_output <out-$test_count >actual && - cat >expect <<-EOF && - remote: # pre-receive hook - remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/heads/next - remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic - remote: error: cannot find hook "proc-receive" - remote: # post-receive hook - remote: post-receive< <ZERO-OID> <COMMIT-A> refs/heads/next - To <URL/of/upstream.git> - * [new branch] HEAD -> next - ! [remote rejected] HEAD -> refs/for/main/topic (fail to run proc-receive hook) + format_and_save_expect <<-EOF && + > remote: # pre-receive hook Z + > remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/heads/next Z + > remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic Z + > remote: error: cannot find hook "proc-receive" Z + > remote: # post-receive hook Z + > remote: post-receive< <ZERO-OID> <COMMIT-A> refs/heads/next Z + > To <URL/of/upstream.git> + > * [new branch] HEAD -> next + > ! [remote rejected] HEAD -> refs/for/main/topic (fail to run proc-receive hook) EOF test_cmp expect actual &&
@@ -41,16 +41,16 @@ test_expect_success "proc-receive: no hook, all failed for atomic push ($PROTOCO HEAD:next \ HEAD:refs/for/main/topic >out-$test_count 2>&1 && make_user_friendly_and_stable_output <out-$test_count >actual && - cat >expect <<-EOF && - remote: # pre-receive hook - remote: pre-receive< <COMMIT-A> <COMMIT-B> refs/heads/main - remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/heads/next - remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic - remote: error: cannot find hook "proc-receive" - To <URL/of/upstream.git> - ! [remote rejected] <COMMIT-B> -> main (fail to run proc-receive hook) - ! [remote rejected] HEAD -> next (fail to run proc-receive hook) - ! [remote rejected] HEAD -> refs/for/main/topic (fail to run proc-receive hook) + format_and_save_expect <<-EOF && + > remote: # pre-receive hook Z + > remote: pre-receive< <COMMIT-A> <COMMIT-B> refs/heads/main Z + > remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/heads/next Z + > remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic Z + > remote: error: cannot find hook "proc-receive" Z + > To <URL/of/upstream.git> + > ! [remote rejected] <COMMIT-B> -> main (fail to run proc-receive hook) + > ! [remote rejected] HEAD -> next (fail to run proc-receive hook) + > ! [remote rejected] HEAD -> refs/for/main/topic (fail to run proc-receive hook) EOF test_cmp expect actual &&
t/t5411/test-0012-no-hook-error--porcelain.sh +21 -21
54 lines not on this page · raw message
t/t5411/test-0013-bad-protocol.sh +31 -31
86 lines not on this page · raw message
t/t5411/test-0014-bad-protocol--porcelain.sh +40 -40
140 lines not on this page · raw message
t/t5411/test-0020-report-ng.sh +16 -16
@@ -14,14 +14,14 @@ test_expect_success "proc-receive: fail to update (ng, no message, $PROTOCOL)" ' HEAD:refs/for/main/topic \ >out-$test_count 2>&1 && make_user_friendly_and_stable_output <out-$test_count >actual && - cat >expect <<-EOF && - remote: # pre-receive hook - remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic - remote: # proc-receive hook - remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic - remote: proc-receive> ng refs/for/main/topic - To <URL/of/upstream.git> - ! [remote rejected] HEAD -> refs/for/main/topic (failed) + format_and_save_expect <<-EOF && + > remote: # pre-receive hook Z + > remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic Z + > remote: # proc-receive hook Z + > remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic Z + > remote: proc-receive> ng refs/for/main/topic Z + > To <URL/of/upstream.git> + > ! [remote rejected] HEAD -> refs/for/main/topic (failed) EOF test_cmp expect actual &&
@@ -46,14 +46,14 @@ test_expect_success "proc-receive: fail to update (ng, with message, $PROTOCOL)" HEAD:refs/for/main/topic \ >out-$test_count 2>&1 && make_user_friendly_and_stable_output <out-$test_count >actual && - cat >expect <<-EOF && - remote: # pre-receive hook - remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic - remote: # proc-receive hook - remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic - remote: proc-receive> ng refs/for/main/topic error msg - To <URL/of/upstream.git> - ! [remote rejected] HEAD -> refs/for/main/topic (error msg) + format_and_save_expect <<-EOF && + > remote: # pre-receive hook Z + > remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic Z + > remote: # proc-receive hook Z + > remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic Z + > remote: proc-receive> ng refs/for/main/topic error msg Z + > To <URL/of/upstream.git> + > ! [remote rejected] HEAD -> refs/for/main/topic (error msg) EOF test_cmp expect actual &&
t/t5411/test-0021-report-ng--porcelain.sh +18 -18
@@ -14,15 +14,15 @@ test_expect_success "proc-receive: fail to update (ng, no message, $PROTOCOL/por HEAD:refs/for/main/topic \ >out-$test_count 2>&1 && make_user_friendly_and_stable_output <out-$test_count >actual && - cat >expect <<-EOF && - remote: # pre-receive hook - remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic - remote: # proc-receive hook - remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic - remote: proc-receive> ng refs/for/main/topic - To <URL/of/upstream.git> - ! HEAD:refs/for/main/topic [remote rejected] (failed) - Done + format_and_save_expect <<-EOF && + > remote: # pre-receive hook Z + > remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic Z + > remote: # proc-receive hook Z + > remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic Z + > remote: proc-receive> ng refs/for/main/topic Z + > To <URL/of/upstream.git> + > ! HEAD:refs/for/main/topic [remote rejected] (failed) + > Done EOF test_cmp expect actual &&
@@ -47,15 +47,15 @@ test_expect_success "proc-receive: fail to update (ng, with message, $PROTOCOL/p HEAD:refs/for/main/topic \ >out-$test_count 2>&1 && make_user_friendly_and_stable_output <out-$test_count >actual && - cat >expect <<-EOF && - remote: # pre-receive hook - remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic - remote: # proc-receive hook - remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic - remote: proc-receive> ng refs/for/main/topic error msg - To <URL/of/upstream.git> - ! HEAD:refs/for/main/topic [remote rejected] (error msg) - Done + format_and_save_expect <<-EOF && + > remote: # pre-receive hook Z + > remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic Z + > remote: # proc-receive hook Z + > remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic Z + > remote: proc-receive> ng refs/for/main/topic error msg Z + > To <URL/of/upstream.git> + > ! HEAD:refs/for/main/topic [remote rejected] (error msg) + > Done EOF test_cmp expect actual &&
t/t5411/test-0022-report-unexpect-ref.sh +13 -13
@@ -15,19 +15,19 @@ test_expect_success "proc-receive: report unexpected ref ($PROTOCOL)" ' HEAD:refs/for/main/topic \ >out-$test_count 2>&1 && make_user_friendly_and_stable_output <out-$test_count >actual && - cat >expect <<-EOF && - remote: # pre-receive hook - remote: pre-receive< <COMMIT-A> <COMMIT-B> refs/heads/main - remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic - remote: # proc-receive hook - remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic - remote: proc-receive> ok refs/heads/main - remote: error: proc-receive reported status on unexpected ref: refs/heads/main - remote: # post-receive hook - remote: post-receive< <COMMIT-A> <COMMIT-B> refs/heads/main - To <URL/of/upstream.git> - <COMMIT-A>..<COMMIT-B> <COMMIT-B> -> main - ! [remote rejected] HEAD -> refs/for/main/topic (proc-receive failed to report status) + format_and_save_expect <<-EOF && + > remote: # pre-receive hook Z + > remote: pre-receive< <COMMIT-A> <COMMIT-B> refs/heads/main Z + > remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic Z + > remote: # proc-receive hook Z + > remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic Z + > remote: proc-receive> ok refs/heads/main Z + > remote: error: proc-receive reported status on unexpected ref: refs/heads/main Z + > remote: # post-receive hook Z + > remote: post-receive< <COMMIT-A> <COMMIT-B> refs/heads/main Z + > To <URL/of/upstream.git> + > <COMMIT-A>..<COMMIT-B> <COMMIT-B> -> main + > ! [remote rejected] HEAD -> refs/for/main/topic (proc-receive failed to report status) EOF test_cmp expect actual &&
t/t5411/test-0023-report-unexpect-ref--porcelain.sh +14 -14
@@ -15,20 +15,20 @@ test_expect_success "proc-receive: report unexpected ref ($PROTOCOL/porcelain)" HEAD:refs/for/main/topic \ >out-$test_count 2>&1 && make_user_friendly_and_stable_output <out-$test_count >actual && - cat >expect <<-EOF && - remote: # pre-receive hook - remote: pre-receive< <COMMIT-A> <COMMIT-B> refs/heads/main - remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic - remote: # proc-receive hook - remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic - remote: proc-receive> ok refs/heads/main - remote: error: proc-receive reported status on unexpected ref: refs/heads/main - remote: # post-receive hook - remote: post-receive< <COMMIT-A> <COMMIT-B> refs/heads/main - To <URL/of/upstream.git> - <COMMIT-B>:refs/heads/main <COMMIT-A>..<COMMIT-B> - ! HEAD:refs/for/main/topic [remote rejected] (proc-receive failed to report status) - Done + format_and_save_expect <<-EOF && + > remote: # pre-receive hook Z + > remote: pre-receive< <COMMIT-A> <COMMIT-B> refs/heads/main Z + > remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic Z + > remote: # proc-receive hook Z + > remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic Z + > remote: proc-receive> ok refs/heads/main Z + > remote: error: proc-receive reported status on unexpected ref: refs/heads/main Z + > remote: # post-receive hook Z + > remote: post-receive< <COMMIT-A> <COMMIT-B> refs/heads/main Z + > To <URL/of/upstream.git> + > <COMMIT-B>:refs/heads/main <COMMIT-A>..<COMMIT-B> + > ! HEAD:refs/for/main/topic [remote rejected] (proc-receive failed to report status) + > Done EOF test_cmp expect actual &&
t/t5411/test-0024-report-unknown-ref.sh +9 -9
@@ -14,15 +14,15 @@ test_expect_success "proc-receive: report unknown reference ($PROTOCOL)" ' HEAD:refs/for/a/b/c/my/topic \ >out-$test_count 2>&1 && make_user_friendly_and_stable_output <out-$test_count >actual && - cat >expect <<-EOF && - remote: # pre-receive hook - remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/a/b/c/my/topic - remote: # proc-receive hook - remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/a/b/c/my/topic - remote: proc-receive> ok refs/for/main/topic - remote: error: proc-receive reported status on unknown ref: refs/for/main/topic - To <URL/of/upstream.git> - ! [remote rejected] HEAD -> refs/for/a/b/c/my/topic (proc-receive failed to report status) + format_and_save_expect <<-EOF && + > remote: # pre-receive hook Z + > remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/a/b/c/my/topic Z + > remote: # proc-receive hook Z + > remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/a/b/c/my/topic Z + > remote: proc-receive> ok refs/for/main/topic Z + > remote: error: proc-receive reported status on unknown ref: refs/for/main/topic Z + > To <URL/of/upstream.git> + > ! [remote rejected] HEAD -> refs/for/a/b/c/my/topic (proc-receive failed to report status) EOF test_cmp expect actual &&
t/t5411/test-0025-report-unknown-ref--porcelain.sh +10 -10
@@ -14,16 +14,16 @@ test_expect_success "proc-receive: report unknown reference ($PROTOCOL/porcelain HEAD:refs/for/a/b/c/my/topic \ >out-$test_count 2>&1 && make_user_friendly_and_stable_output <out-$test_count >actual && - cat >expect <<-EOF && - remote: # pre-receive hook - remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/a/b/c/my/topic - remote: # proc-receive hook - remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/a/b/c/my/topic - remote: proc-receive> ok refs/for/main/topic - remote: error: proc-receive reported status on unknown ref: refs/for/main/topic - To <URL/of/upstream.git> - ! HEAD:refs/for/a/b/c/my/topic [remote rejected] (proc-receive failed to report status) - Done + format_and_save_expect <<-EOF && + > remote: # pre-receive hook Z + > remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/a/b/c/my/topic Z + > remote: # proc-receive hook Z + > remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/a/b/c/my/topic Z + > remote: proc-receive> ok refs/for/main/topic Z + > remote: error: proc-receive reported status on unknown ref: refs/for/main/topic Z + > To <URL/of/upstream.git> + > ! HEAD:refs/for/a/b/c/my/topic [remote rejected] (proc-receive failed to report status) + > Done EOF test_cmp expect actual &&
t/t5411/test-0026-push-options.sh +29 -29
70 lines not on this page · raw message
t/t5411/test-0027-push-options--porcelain.sh +31 -31
74 lines not on this page · raw message
t/t5411/test-0030-report-ok.sh +10 -10
@@ -14,16 +14,16 @@ test_expect_success "proc-receive: ok ($PROTOCOL)" ' HEAD:refs/for/main/topic \ >out 2>&1 && make_user_friendly_and_stable_output <out >actual && - cat >expect <<-EOF && - remote: # pre-receive hook - remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic - remote: # proc-receive hook - remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic - remote: proc-receive> ok refs/for/main/topic - remote: # post-receive hook - remote: post-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic - To <URL/of/upstream.git> - * [new reference] HEAD -> refs/for/main/topic + format_and_save_expect <<-EOF && + > remote: # pre-receive hook Z + > remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic Z + > remote: # proc-receive hook Z + > remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic Z + > remote: proc-receive> ok refs/for/main/topic Z + > remote: # post-receive hook Z + > remote: post-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic Z + > To <URL/of/upstream.git> + > * [new reference] HEAD -> refs/for/main/topic EOF test_cmp expect actual &&
t/t5411/test-0031-report-ok--porcelain.sh +11 -11
@@ -14,17 +14,17 @@ test_expect_success "proc-receive: ok ($PROTOCOL/porcelain)" ' HEAD:refs/for/main/topic \ >out 2>&1 && make_user_friendly_and_stable_output <out >actual && - cat >expect <<-EOF && - remote: # pre-receive hook - remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic - remote: # proc-receive hook - remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic - remote: proc-receive> ok refs/for/main/topic - remote: # post-receive hook - remote: post-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic - To <URL/of/upstream.git> - * HEAD:refs/for/main/topic [new reference] - Done + format_and_save_expect <<-EOF && + > remote: # pre-receive hook Z + > remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic Z + > remote: # proc-receive hook Z + > remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic Z + > remote: proc-receive> ok refs/for/main/topic Z + > remote: # post-receive hook Z + > remote: post-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic Z + > To <URL/of/upstream.git> + > * HEAD:refs/for/main/topic [new reference] + > Done EOF test_cmp expect actual &&
t/t5411/test-0032-report-with-options.sh +93 -93
228 lines not on this page · raw message
t/t5411/test-0033-report-with-options--porcelain.sh +100 -100
242 lines not on this page · raw message
t/t5411/test-0034-report-ft.sh +11 -11
@@ -15,17 +15,17 @@ test_expect_success "proc-receive: fall throught, let receive-pack to execute ($ $B:refs/for/main/topic \ >out 2>&1 && make_user_friendly_and_stable_output <out >actual && - cat >expect <<-EOF && - remote: # pre-receive hook - remote: pre-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic - remote: # proc-receive hook - remote: proc-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic - remote: proc-receive> ok refs/for/main/topic - remote: proc-receive> option fall-through - remote: # post-receive hook - remote: post-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic - To <URL/of/upstream.git> - * [new reference] <COMMIT-B> -> refs/for/main/topic + format_and_save_expect <<-EOF && + > remote: # pre-receive hook Z + > remote: pre-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic Z + > remote: # proc-receive hook Z + > remote: proc-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic Z + > remote: proc-receive> ok refs/for/main/topic Z + > remote: proc-receive> option fall-through Z + > remote: # post-receive hook Z + > remote: post-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic Z + > To <URL/of/upstream.git> + > * [new reference] <COMMIT-B> -> refs/for/main/topic EOF test_cmp expect actual &&
t/t5411/test-0035-report-ft--porcelain.sh +12 -12
@@ -15,18 +15,18 @@ test_expect_success "proc-receive: fall throught, let receive-pack to execute ($ $B:refs/for/main/topic \ >out 2>&1 && make_user_friendly_and_stable_output <out >actual && - cat >expect <<-EOF && - remote: # pre-receive hook - remote: pre-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic - remote: # proc-receive hook - remote: proc-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic - remote: proc-receive> ok refs/for/main/topic - remote: proc-receive> option fall-through - remote: # post-receive hook - remote: post-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic - To <URL/of/upstream.git> - * <COMMIT-B>:refs/for/main/topic [new reference] - Done + format_and_save_expect <<-EOF && + > remote: # pre-receive hook Z + > remote: pre-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic Z + > remote: # proc-receive hook Z + > remote: proc-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic Z + > remote: proc-receive> ok refs/for/main/topic Z + > remote: proc-receive> option fall-through Z + > remote: # post-receive hook Z + > remote: post-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic Z + > To <URL/of/upstream.git> + > * <COMMIT-B>:refs/for/main/topic [new reference] + > Done EOF test_cmp expect actual &&
t/t5411/test-0036-report-multi-rewrite-for-one-ref.sh +66 -66
150 lines not on this page · raw message
t/t5411/test-0037-report-multi-rewrite-for-one-ref--porcelain.sh +69 -69
156 lines not on this page · raw message
t/t5411/test-0038-report-mixed-refs.sh +37 -37
80 lines not on this page · raw message
t/t5411/test-0039-report-mixed-refs--porcelain.sh +38 -38
82 lines not on this page · raw message
t/t5411/test-0040-process-all-refs.sh +40 -40
86 lines not on this page · raw message
t/t5411/test-0041-process-all-refs--porcelain.sh +41 -41
88 lines not on this page · raw message
t/t5411/test-0050-proc-receive-refs-with-modifiers.sh +45 -45
102 lines not on this page · raw message
t/t5548-push-porcelain.sh +28 -26
99 lines not on this page · raw message
t/t6020-bundle-misc.sh +24 -21
114 lines not on this page · raw message
Signature · 1 line
2.32.0.rc0.27.g7b1e85181b
Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline
In reply to Jiang XinQuoted 7 lines · Jiang Xin
From: Jiang Xin <[email protected]>
When calling "
demultiplex_sideband" on a sideband-2 message, will try to split the message by line breaks, and append a suffix to each nonempty line to clear the end of the screen line. But in the following example, there will be no suffix (8 spaces) for "<message-3>":
Quoted 8 lines
PKT-LINE(\2 <message-1> CR <message-2> CR <message-3>) PKT-LINE(\2 CR <message-4> CR <message-5> CR)This is because the line break of "<message-3>" is placed in the next pktline message.
Without this fix, t5411 must remove trailing spaces of the actual output of "git-push" command before comparing.
Nice, i.e. let's generally fix the output instead.
Quoted 10 lines
Signed-off-by Jiang Xin <[email protected]>--- sideband.c | 4 ++++ 1 file changed, 4 insertions(+)sideband.c +0
-0
sideband.c int demultiplex_sideband(const char *me, int status, · line 192
Quoted 8 lines
sideband.c +4
-0@@ -185,6 +185,10 @@ int demultiplex_sideband(const char *me, int status,if (!scratch->len)strbuf_addstr (scratch,DISPLAY_PREFIX ); +else if (!linelen) +/* buf has a leading CR which ends the remaining +* scratch of last round of "demultiplex_sideband" */ +strbuf_addstr (scratch, suffix);if (linelen >0 ) {
I haven't thought about this carefully but isn't there some way to combine these if/else if/if statementsn that's clearer?
I.e. here we're doing an "if" check for a !lineline and then an "if" that can't be true if !linelen.
Isn't this the same as:
if (!scratch->len) {
...
} else {
if (!linelen)
...
else if (linelen > 0)
...
}Or are there cases where we take that "linelen > 0" arm if !scratch->len?
Quoted 2 lines
sideband.c +0
-0maybe_colorize_sideband (scratch, b, linelen);strbuf_addstr (scratch, suffix);
Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline
In reply to Jiang XinQuoted 5 lines · Jiang Xin
From: Jiang Xin <[email protected]>
When calling "
demultiplex_sideband" on a sideband-2 message, will try to split the message by line breaks, and append a suffix to each nonempty line to clear the end of the screen line.
Subject of "will try" and "append" is missing. Do you mean that the helper function in question does these two things? I.e.
demultiplex_sideband() used on a sideband #2 will try to... and appends ...
Quoted 5 lines
But in the following example,
there will be no suffix (8 spaces) for "<message-3>":PKT-LINE(\2 <message-1> CR <message-2> CR <message-3>) PKT-LINE(\2 CR <message-4> CR <message-5> CR)
That description may mechanically correct, but
after <message-3>, we fail to clear to the end of line
may make it easier to understand what the problem we are trying to solve for those who do not remember what these suffix games are about.
Quoted 16 lines
This is because the line break of "<message-3>" is placed in the next pktline message.
Without this fix, t5411 must remove trailing spaces of the actual output of "git-push" command before comparing.
Signed-off-by Jiang Xin <[email protected]>--- sideband.c | 4 ++++ 1 file changed, 4 insertions(+)sideband.c +0
-0
sideband.c int demultiplex_sideband(const char *me, int status, · line 191
Quoted 7 lines
sideband.c +4
-0@@ -185,6 +185,10 @@ int demultiplex_sideband(const char *me, int status,if (!scratch->len)strbuf_addstr (scratch,DISPLAY_PREFIX ); +else if (!linelen) +/* buf has a leading CR which ends the remaining +* scratch of last round of "demultiplex_sideband" */ +strbuf_addstr (scratch, suffix);
The style of multi-line comment needs fixing, but the contents of the comment is a bit hard to grok.
sideband.c int demultiplex_sideband(const char *me, int status, · line 194
Quoted 3 lines
sideband.c +0
-0if (linelen >0 ) {maybe_colorize_sideband (scratch, b, linelen);strbuf_addstr (scratch, suffix);
I wonder if the following is simpler to read, though.
-- >8 --
Subject: [PATCH] sideband: don't lose clear-to-eol at packet boundary
When demultiplex_sideband() sees a CR or LF on the sideband #2, it adds "suffix" string to clear to the end of the current line, which helps when relaying a progress display whose records are terminated with CRs.
The code however forgot that depending on the length of the payload line, such a CR may fall exactly at the packet boundary and the number of bytes before the CR from the beginning of the packet could be zero. In such a case, the message that was terminated by the CR were leftover in the "scratch" buffer in the previous call to the function and we still need to clear to the end of the current line.
Just remove the unnecessary check on linelen; maybe_colorize_sideband() on 0-byte payload turns into a no-op, and we should be adding clear-to-eol for each and every CR/LF anyway.
sideband.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-)
sideband.c +3 -4
@@ -185,10 +185,9 @@ int demultiplex_sideband(const char *me, int status,if (!scratch->len)strbuf_addstr (scratch,DISPLAY_PREFIX ); -if (linelen >0 ) { -maybe_colorize_sideband (scratch, b, linelen); -strbuf_addstr (scratch, suffix); - } + +maybe_colorize_sideband (scratch, b, linelen); +strbuf_addstr (scratch, suffix);strbuf_addch (scratch, *brk);xwrite (2 , scratch->buf, scratch->len);
Re: [PATCH v2 0/4] Fixed t6020 bash compatible issue and fixed wrong sideband suffix issue
In reply to Jiang XinQuoted 4 lines · Jiang Xin
From: Jiang Xin <[email protected]>
In addition to fix the bash incompatible issue of t6020, find another issue when try to rewrite t5411 to compare raw command output.
Do the three later patches depend on the t6020 fix, or is this made a 4-patch series only for the convenience of sending them out?
It's not like get_abbrev_oid() used in t6020 is defined in a common part of the test library and later used by other tests (instead, the patches duplicate this helper function into yet two more files).
Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline
In reply to Junio C HamanoQuoted 92 lines · Junio C Hamano <[email protected]> 于2021年6月14日周一 上午11:50写道:
Quoted 5 lines · Jiang Xin
From: Jiang Xin <[email protected]>
When calling "
demultiplex_sideband" on a sideband-2 message, will try to split the message by line breaks, and append a suffix to each nonempty line to clear the end of the screen line.Subject of "will try" and "append" is missing. Do you mean that the helper function in question does these two things? I.e.
demultiplex_sideband()used on a sideband #2 will try to... and appends ...Quoted 5 lines
But in the following example,
there will be no suffix (8 spaces) for "<message-3>":PKT-LINE(\2 <message-1> CR <message-2> CR <message-3>) PKT-LINE(\2 CR <message-4> CR <message-5> CR)That description may mechanically correct, but
after <message-3>, we fail to clear to the end of line
may make it easier to understand what the problem we are trying to solve for those who do not remember what these suffix games are about.
Quoted 16 lines
This is because the line break of "<message-3>" is placed in the next pktline message.
Without this fix, t5411 must remove trailing spaces of the actual output of "git-push" command before comparing.
Signed-off-by Jiang Xin <[email protected]>--- sideband.c | 4 ++++ 1 file changed, 4 insertions(+)sideband.c +0
-0sideband.c int demultiplex_sideband(const char *me, int status, · line 191
Quoted 7 lines
sideband.c +4
-0@@ -185,6 +185,10 @@ int demultiplex_sideband(const char *me, int status,if (!scratch->len)strbuf_addstr (scratch,DISPLAY_PREFIX ); +else if (!linelen) +/* buf has a leading CR which ends the remaining +* scratch of last round of "demultiplex_sideband" */ +strbuf_addstr (scratch, suffix);The style of multi-line comment needs fixing, but the contents of the comment is a bit hard to grok.
sideband.c int demultiplex_sideband(const char *me, int status, · line 194
Quoted 3 lines
sideband.c +0
-0if (linelen >0 ) {maybe_colorize_sideband (scratch, b, linelen);strbuf_addstr (scratch, suffix);I wonder if the following is simpler to read, though.
-- >8 --
Subject: [PATCH] sideband: don't lose clear-to-eol at packet boundaryWhen
demultiplex_sideband()sees a CR or LF on the sideband #2, it adds "suffix" string to clear to the end of the current line, which helps when relaying a progress display whose records are terminated with CRs.The code however forgot that depending on the length of the payload line, such a CR may fall exactly at the packet boundary and the number of bytes before the CR from the beginning of the packet could be zero. In such a case, the message that was terminated by the CR were leftover in the "scratch" buffer in the previous call to the function and we still need to clear to the end of the current line.
Just remove the unnecessary check on linelen;
maybe_colorize_sideband()on 0-byte payload turns into a no-op, and we should be adding clear-to-eol for each and every CR/LF anyway.sideband.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-)sideband.c +0
-0@@ -185,10 +185,9 @@ int demultiplex_sideband(const char *me, int status,if (!scratch->len)strbuf_addstr (scratch,DISPLAY_PREFIX );
sideband.c int demultiplex_sideband(const char *me, int status, · line 193
Quoted 10 lines
sideband.c +3
-4-if (linelen >0 ) { -maybe_colorize_sideband (scratch, b, linelen); -strbuf_addstr (scratch, suffix); - } + +maybe_colorize_sideband (scratch, b, linelen); +strbuf_addstr (scratch, suffix);strbuf_addch (scratch, *brk);xwrite (2 , scratch->buf, scratch->len);
The above changes will add suffix to the end of each line, and even an empty lines. However, according to the comment in commit ebe8fa738d (fix display overlap between remote and local progress, 2007-11-04) which introduced the suffix implementation for the first time, no suffix should be appended for empty lines.
/*
* Let's insert a suffix to clear the end
* of the screen line, but only if current
* line data actually contains something.
*/
So my implementation is to try not to break the original implementation, and keep the linelen unchanged.
The strbuf "scratch" will be reset at line 18th in the while block, so the nonempty scratch at line 7 indicates the parameter scratch of demultiplex_sideband() is not empty. With the following patch, additional suffix is only added before a leading CR in a packet which is seperated with its message by packet boundary.
```
01 while ((brk = strpbrk(b, "\n\r"))) {
02 int linelen = brk - b;
03
04 + /* Has no empty scratch from last call of "demultiplex_sideband"
05 + * and has a leading CR in buf.
06 + */
07 + if (scratch->len && !linelen)
08 + strbuf_addstr(scratch, suffix);
09 if (!scratch->len)
10 strbuf_addstr(scratch, DISPLAY_PREFIX);
11 if (linelen > 0) {
12 maybe_colorize_sideband(scratch, b, linelen);
13 strbuf_addstr(scratch, suffix);
14 }
15
16 strbuf_addch(scratch, *brk);
17 xwrite(2, scratch->buf, scratch->len);
18 strbuf_reset(scratch);
19
20 b = brk + 1;
21 }
```Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline
In reply to Jiang XinQuoted 13 lines · Jiang Xin
Quoted 12 lines · Junio C Hamano <[email protected]> 于2021年6月15日周二 上午9:17写道:
Quoted 8 lines · Jiang Xin
/*
* Let's insert a suffix to clear the end
* of the screen line, but only if current
* line data actually contains something.
*/So my implementation is to try not to break the original implementation, and keep the linelen unchanged.
Quoted 7 lines
Quoted 4 lines
I knew what you wanted to do from your code---I am questioning if that "only when something is there" was really sensible, or if it was just attracting bugs.
@Nicolas, what's your opinion? Is it ok to add clear-to-eol suffix to each line even empty ones?
That would be the simplest thing to do.
But there must have been a reason for doing it otherwise. I just don't remember anymore.
Maybe it had to do with progress reporting that does a bunch of percentage updates followed by '\r' to remain on the same line, and at the end a single '\n' to move to the next line without erasing the final status report line. That would be a case for not clearing empty lines.
Nicolas
Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline
In reply to Jiang XinQuoted 8 lines · Jiang Xin
/*
* Let's insert a suffix to clear the end
* of the screen line, but only if current
* line data actually contains something.
*/So my implementation is to try not to break the original implementation, and keep the linelen unchanged.
I knew what you wanted to do from your code---I am questioning if that "only when something is there" was really sensible, or if it was just attracting bugs.
Thanks.
Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline
Quoted 9 lines · Junio C Hamano <[email protected]> 于2021年6月15日周二 上午9:17写道:
Quoted 6 lines · Jiang Xin
/*
* Let's insert a suffix to clear the end
* of the screen line, but only if current
* line data actually contains something.
*/
Quoted 7 lines
Quoted 2 lines
So my implementation is to try not to break the original implementation, and keep the linelen unchanged.
I knew what you wanted to do from your code---I am questioning if that "only when something is there" was really sensible, or if it was just attracting bugs.
@Nicolas, what's your opinion? Is it ok to add clear-to-eol suffix to each line even empty ones?
--
Jiang Xin
Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline
In reply to Nicolas PitreQuoted 26 lines · Nicolas Pitre <[email protected]> 于2021年6月15日周二 上午10:11写道:
Quoted 13 lines · Jiang Xin
Quoted 12 lines · Junio C Hamano <[email protected]> 于2021年6月15日周二 上午9:17写道:
Quoted 8 lines · Jiang Xin
/*
* Let's insert a suffix to clear the end
* of the screen line, but only if current
* line data actually contains something.
*/So my implementation is to try not to break the original implementation, and keep the linelen unchanged.
Quoted 7 lines
Quoted 4 lines
I knew what you wanted to do from your code---I am questioning if that "only when something is there" was really sensible, or if it was just attracting bugs.
@Nicolas, what's your opinion? Is it ok to add clear-to-eol suffix to each line even empty ones?
That would be the simplest thing to do.
Quoted 8 lines
But there must have been a reason for doing it otherwise. I just don't remember anymore.
Maybe it had to do with progress reporting that does a bunch of percentage updates followed by '\r' to remain on the same line, and at the end a single '\n' to move to the next line without erasing the final status report line. That would be a case for not clearing empty lines.
Thank @Nicolas for helping me understand the story behinds the code.
If there are two sideband #2 packets like this:
PKTLINE(\2 "<progress-1>" CR "<progress-2>" CR)
PKTLINE(\2 "<message-3>" LF "<message-4>" LF)We should append clear-to-eol suffix to "<progress-1>", "<progess-2>" and "<message-3>" to erase the last message displayed on the same line. Even though there is no need to add the clear-to-eol suffix to "<message-4>", always adding suffix before line breaks (CR or LF) of nonempty message make it simple to program.
If there are empty messages in sideband #2 packets like this:
PKTLINE(\2 "<progress-1>" CR LF "<message-2>" LF)
PKTLINE(\2 "<message-3>" LF)For the empty message between "<progress-1>" and "<message-2>", nothing to display and no need to add clear-to-eol suffix.
The issue this patch try to fix is like the following example:
PKTLINE(\2 "<progress-1>" CR "<progress-2>")
PKTLINE(\2 CR "<message-3>" LF)The message "<progress-2>" is displayed without a proper clear-to-eol suffix, because it's eol (CR) is in another pktline.
Since we can distinguished this case by checking the size of "scratch", IMHO, it better not add suffix before all line breaks.
--
Jiang Xin
Re: [PATCH v2 0/4] Fixed t6020 bash compatible issue and fixed wrong sideband suffix issue
In reply to Junio C HamanoQuoted 8 lines · Junio C Hamano <[email protected]> 于2021年6月14日周一 下午12:10写道:
Quoted 4 lines · Jiang Xin
From: Jiang Xin <[email protected]>
In addition to fix the bash incompatible issue of t6020, find another issue when try to rewrite t5411 to compare raw command output.
Quoted 7 lines
Do the three later patches depend on the t6020 fix, or is this made a 4-patch series only for the convenience of sending them out?
It's not like
get_abbrev_oid()used in t6020 is defined in a common part of the test library and later used by other tests (instead, the patches duplicate this helper function into yet two more files).
Will split it into two patch series. One will fix bash incompatible parameter expansion in t6020, another will fix clear-to-eol at packet boundary issue of sideband and try to test raw output in t6020, t5548 and t5411.
I also queue another patch series, which add "--bare" support to "test_create_repo", and replace "git init" command in test cases to adapt to variable GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME.
--
Jiang Xin
Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline
In reply to Jiang XinQuoted 7 lines · Jiang Xin
The issue this patch try to fix is like the following example:
PKTLINE(\2 "<progress-1>" CR "<progress-2>") PKTLINE(\2 CR "<message-3>" LF)The message "<progress-2>" is displayed without a proper clear-to-eol suffix, because it's eol (CR) is in another pktline.
I'd fix this issue with the following logic:
bool pending_clear_to_eol;
my_putchar(c) {
switch (c) {
case '\r':
case '\n':
pending_clear_to_eol = true;
break;
default:
if (pending_clear_to_eol) {
clear_to_eol();
pending_clear_to_eol = false;
}
break;
}
putchar(c);
}In other words, you clear the line after printing "remote:" but only if there is a non \n or \r coming next.
Nicolas
Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline
Quoted 30 lines · Nicolas Pitre
Quoted 7 lines · Jiang Xin
The issue this patch try to fix is like the following example:
PKTLINE(\2 "<progress-1>" CR "<progress-2>") PKTLINE(\2 CR "<message-3>" LF)The message "<progress-2>" is displayed without a proper clear-to-eol suffix, because it's eol (CR) is in another pktline.
I'd fix this issue with the following logic:
bool
pending_clear_to_eol;my_putchar(c) { switch (c) { case '\r': case '\n': pending_clear_to_eol = true; break; default: if (pending_clear_to_eol) { clear_to_eol(); pending_clear_to_eol = false; } break; } putchar(c); }
Quoted 2 lines
In other words, you clear the line after printing "remote:" but only if there is a non \n or \r coming next.
What puzzles me the most in this discussion is why we do this for LF. I do understand why we need it for CR---the line we are going to show message on after emitting CR would be full of leftover letters we previously have written before emitting CR, so we'd show the message (to overwrite the initial part enough to show our own message) and then clear to the end with either ANSI sequence of sufficient number of whitespaces. But line feed would take us to a fresh and blank line---there is nothing to clear, no?
Thanks.
Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline
Quoted 36 lines · Junio C Hamano <[email protected]> 于2021年6月15日周二 下午12:46写道:
Quoted 30 lines · Nicolas Pitre
Quoted 7 lines · Jiang Xin
The issue this patch try to fix is like the following example:
PKTLINE(\2 "<progress-1>" CR "<progress-2>") PKTLINE(\2 CR "<message-3>" LF)The message "<progress-2>" is displayed without a proper clear-to-eol suffix, because it's eol (CR) is in another pktline.
I'd fix this issue with the following logic:
bool
pending_clear_to_eol;my_putchar(c) { switch (c) { case '\r': case '\n': pending_clear_to_eol = true; break; default: if (pending_clear_to_eol) { clear_to_eol(); pending_clear_to_eol = false; } break; } putchar(c); }Quoted 2 lines
In other words, you clear the line after printing "remote:" but only if there is a non \n or \r coming next.
Quoted 8 lines
What puzzles me the most in this discussion is why we do this for LF. I do understand why we need it for CR---the line we are going to show message on after emitting CR would be full of leftover letters we previously have written before emitting CR, so we'd show the message (to overwrite the initial part enough to show our own message) and then clear to the end with either ANSI sequence of sufficient number of whitespaces. But line feed would take us to a fresh and blank line---there is nothing to clear, no?
I guess this may because sideband #2 messages are printed on the screen in a background process, it never know a line where it starts to print has characters on the right. So it is safe to write an additional clear-to-eol suffix no matter the message ends with CR or LF.
--
Jiang Xin
Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline
In reply to Junio C HamanoQuoted 35 lines · Junio C Hamano
Quoted 30 lines · Nicolas Pitre
Quoted 7 lines · Jiang Xin
The issue this patch try to fix is like the following example:
PKTLINE(\2 "<progress-1>" CR "<progress-2>") PKTLINE(\2 CR "<message-3>" LF)The message "<progress-2>" is displayed without a proper clear-to-eol suffix, because it's eol (CR) is in another pktline.
I'd fix this issue with the following logic:
bool
pending_clear_to_eol;my_putchar(c) { switch (c) { case '\r': case '\n': pending_clear_to_eol = true; break; default: if (pending_clear_to_eol) { clear_to_eol(); pending_clear_to_eol = false; } break; } putchar(c); }Quoted 2 lines
In other words, you clear the line after printing "remote:" but only if there is a non \n or \r coming next.
Quoted 8 lines
What puzzles me the most in this discussion is why we do this for LF. I do understand why we need it for CR---the line we are going to show message on after emitting CR would be full of leftover letters we previously have written before emitting CR, so we'd show the message (to overwrite the initial part enough to show our own message) and then clear to the end with either ANSI sequence of sufficient number of whitespaces. But line feed would take us to a fresh and blank line---there is nothing to clear, no?
Depends. Suppose the local process is doing the progress report with CR.
Then the remote sends a single line with LF.
You expects the remote line to be displayed over the local progress report and the local progress report to be resumed on the following line. Without the line clearing you might have leftover garbage on the remote message line.
Nicolas
PATCH v3 1/1 t6020: fix incompatible parameter expansion
In reply to Jiang XinÆvar reported that the function make_user_friendly_and_stable_output() failed on a i386 box (gcc45) in the gcc farm boxes with error:
sed: couldn't re-allocate memory
It turns out that older versions of bash (4.3) or dash (0.5.7) cannot evaluate expression like ${A%${A#???????}} used to get the leading 7 characters of variable A.
Replace the incompatible parameter expansion so that t6020 works on older version of bash or dash.
--- t/t6020-bundle-misc.sh | 50 ++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 19 deletions(-)
t/t6020-bundle-misc.sh +31 -19
@@ -80,30 +80,42 @@ test_commit_setvar () { eval $var=$oid } +get_abbrev_oid () { + oid=$1 && + suffix=${oid#???????} && + oid=${oid%$suffix} && + if test -n "$oid" + then + echo "$oid" + else + echo "undefined-oid" + fi +} + # Format the output of git commands to make a user-friendly and stable # text. We can easily prepare the expect text without having to worry # about future changes of the commit ID and spaces of the output. make_user_friendly_and_stable_output () { sed \ - -e "s/${A%${A#???????}}[0-9a-f]*/<COMMIT-A>/g" \ - -e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" \ - -e "s/${C%${C#???????}}[0-9a-f]*/<COMMIT-C>/g" \ - -e "s/${D%${D#???????}}[0-9a-f]*/<COMMIT-D>/g" \ - -e "s/${E%${E#???????}}[0-9a-f]*/<COMMIT-E>/g" \ - -e "s/${F%${F#???????}}[0-9a-f]*/<COMMIT-F>/g" \ - -e "s/${G%${G#???????}}[0-9a-f]*/<COMMIT-G>/g" \ - -e "s/${H%${H#???????}}[0-9a-f]*/<COMMIT-H>/g" \ - -e "s/${I%${I#???????}}[0-9a-f]*/<COMMIT-I>/g" \ - -e "s/${J%${J#???????}}[0-9a-f]*/<COMMIT-J>/g" \ - -e "s/${K%${K#???????}}[0-9a-f]*/<COMMIT-K>/g" \ - -e "s/${L%${L#???????}}[0-9a-f]*/<COMMIT-L>/g" \ - -e "s/${M%${M#???????}}[0-9a-f]*/<COMMIT-M>/g" \ - -e "s/${N%${N#???????}}[0-9a-f]*/<COMMIT-N>/g" \ - -e "s/${O%${O#???????}}[0-9a-f]*/<COMMIT-O>/g" \ - -e "s/${P%${P#???????}}[0-9a-f]*/<COMMIT-P>/g" \ - -e "s/${TAG1%${TAG1#???????}}[0-9a-f]*/<TAG-1>/g" \ - -e "s/${TAG2%${TAG2#???????}}[0-9a-f]*/<TAG-2>/g" \ - -e "s/${TAG3%${TAG3#???????}}[0-9a-f]*/<TAG-3>/g" \ + -e "s/$(get_abbrev_oid $A)[0-9a-f]*/<COMMIT-A>/g" \ + -e "s/$(get_abbrev_oid $B)[0-9a-f]*/<COMMIT-B>/g" \ + -e "s/$(get_abbrev_oid $C)[0-9a-f]*/<COMMIT-C>/g" \ + -e "s/$(get_abbrev_oid $D)[0-9a-f]*/<COMMIT-D>/g" \ + -e "s/$(get_abbrev_oid $E)[0-9a-f]*/<COMMIT-E>/g" \ + -e "s/$(get_abbrev_oid $F)[0-9a-f]*/<COMMIT-F>/g" \ + -e "s/$(get_abbrev_oid $G)[0-9a-f]*/<COMMIT-G>/g" \ + -e "s/$(get_abbrev_oid $H)[0-9a-f]*/<COMMIT-H>/g" \ + -e "s/$(get_abbrev_oid $I)[0-9a-f]*/<COMMIT-I>/g" \ + -e "s/$(get_abbrev_oid $J)[0-9a-f]*/<COMMIT-J>/g" \ + -e "s/$(get_abbrev_oid $K)[0-9a-f]*/<COMMIT-K>/g" \ + -e "s/$(get_abbrev_oid $L)[0-9a-f]*/<COMMIT-L>/g" \ + -e "s/$(get_abbrev_oid $M)[0-9a-f]*/<COMMIT-M>/g" \ + -e "s/$(get_abbrev_oid $N)[0-9a-f]*/<COMMIT-N>/g" \ + -e "s/$(get_abbrev_oid $O)[0-9a-f]*/<COMMIT-O>/g" \ + -e "s/$(get_abbrev_oid $P)[0-9a-f]*/<COMMIT-P>/g" \ + -e "s/$(get_abbrev_oid $TAG1)[0-9a-f]*/<TAG-1>/g" \ + -e "s/$(get_abbrev_oid $TAG2)[0-9a-f]*/<TAG-2>/g" \ + -e "s/$(get_abbrev_oid $TAG3)[0-9a-f]*/<TAG-3>/g" \ -e "s/ *\$//" }
Signature · 1 line
2.32.0.rc0.27.g7b1e85181b
Re: [PATCH v3] t6020: fix incompatible parameter expansion
Quoted 9 lines · Jiang Xin
Ævar reported that the function
make_user_friendly_and_stable_output()failed on a i386 box (gcc45) in the gcc farm boxes with error:sed: couldn't re-allocate memory
It turns out that older versions of bash (4.3) or dash (0.5.7) cannot evaluate expression like
${A%${A#???????}}used to get the leading 7 characters of variable A.
Quoted 6 lines
Replace the incompatible parameter expansion so that t6020 works on older version of bash or dash.
Reported-by Ævar Arnfjörð Bjarmason <[email protected]>Signed-off-by Jiang Xin <[email protected]>---
For what it's worth I've also tested this v3 on gcc45, it works too.