Skip to contentSkip to conversation

bundle: arguments can be read from stdin

PATCH In review Jiang Xin 7 participants 60 messages Jan 3, 2021 – Jun 21, 2021

lore
Share & tools
lore

Jiang Xin

author
#0 lore raw

PATCH 1/1 bundle: arguments can be read from stdin

From: Jiang Xin <[email protected]>

In order to create an incremental bundle, we need to pass many arguments to let git-bundle ignore some already packed commits. It will be more convenient to pass args via stdin. But the current implementation does not allow us to do this.

This is because args are parsed twice when creating bundle. The first time for parsing args is in compute_and_write_prerequisites() by running git-rev-list command to write prerequisites in bundle file, and stdin is consumed in this step if "--stdin" option is provided for git-bundle. Later nothing can be read from stdin when running setup_revisions() in create_bundle().

Remove the entire compute_and_write_prerequisites() function, and parse the args once by setup_revisions(). The first step for creating a bundle is to write prerequisites ("-" obj-id SP comment LF), but after calling prepare_revision_walk(), the revs.pending is left empty. Following steps could not work properly without data in revs.pending. Therefore, before calling prepare_revision_walk() function, make a copy on revs.pending for later use.

---
 bundle.c                | 109 ++++++++++++++++++++++------------------
 t/t5607-clone-bundle.sh |   4 +-
 2 files changed, 61 insertions(+), 52 deletions(-)
bundle.c +59 -50
index cb0e5931ac..693d619551 100644
@@ -338,48 +338,6 @@ static int write_pack_data(int bundle_fd, struct rev_info *revs, struct strvec *
 	return 0;
 }
 
-static int compute_and_write_prerequisites(int bundle_fd,
-					   struct rev_info *revs,
-					   int argc, const char **argv)
-{
-	struct child_process rls = CHILD_PROCESS_INIT;
-	struct strbuf buf = STRBUF_INIT;
-	FILE *rls_fout;
-	int i;
-
-	strvec_pushl(&rls.args,
-		     "rev-list", "--boundary", "--pretty=oneline",
-		     NULL);
-	for (i = 1; i < argc; i++)
-		strvec_push(&rls.args, argv[i]);
-	rls.out = -1;
-	rls.git_cmd = 1;
-	if (start_command(&rls))
-		return -1;
-	rls_fout = xfdopen(rls.out, "r");
-	while (strbuf_getwholeline(&buf, rls_fout, '\n') != EOF) {
-		struct object_id oid;
-		if (buf.len > 0 && buf.buf[0] == '-') {
-			write_or_die(bundle_fd, buf.buf, buf.len);
-			if (!get_oid_hex(buf.buf + 1, &oid)) {
-				struct object *object = parse_object_or_die(&oid,
-									    buf.buf);
-				object->flags |= UNINTERESTING;
-				add_pending_object(revs, object, buf.buf);
-			}
-		} else if (!get_oid_hex(buf.buf, &oid)) {
-			struct object *object = parse_object_or_die(&oid,
-								    buf.buf);
-			object->flags |= SHOWN;
-		}
-	}
-	strbuf_release(&buf);
-	fclose(rls_fout);
-	if (finish_command(&rls))
-		return error(_("rev-list died"));
-	return 0;
-}
-
 /*
  * Write out bundle refs based on the tips already
  * parsed into revs.pending. As a side effect, may
@@ -474,6 +432,38 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs)
 	return ref_count;
 }
 
+struct bundle_prerequisites_info {
+	struct object_array *pending;
+	int fd;
+};
+
+static void write_bundle_prerequisites(struct commit *commit, void *data)
+{
+	struct bundle_prerequisites_info *bpi = data;
+	struct object *object;
+	struct pretty_print_context ctx = { 0 };
+	struct strbuf buf = STRBUF_INIT;
+
+	if (!(commit->object.flags & BOUNDARY))
+		return;
+	strbuf_addf(&buf, "-%s ", oid_to_hex(&commit->object.oid));
+	write_or_die(bpi->fd, buf.buf, buf.len);
+
+	ctx.fmt = CMIT_FMT_ONELINE;
+	ctx.output_encoding = get_log_output_encoding();
+	strbuf_reset(&buf);
+	pretty_print_commit(&ctx, commit, &buf);
+	strbuf_trim(&buf);
+
+	object = (struct object *)commit;
+	object->flags |= UNINTERESTING;
+	add_object_array_with_path(object, buf.buf, bpi->pending, S_IFINVALID,
+				   NULL);
+	strbuf_addch(&buf, '\n');
+	write_or_die(bpi->fd, buf.buf, buf.len);
+	strbuf_release(&buf);
+}
+
 int create_bundle(struct repository *r, const char *path,
 		  int argc, const char **argv, struct strvec *pack_options, int version)
 {
@@ -481,8 +471,10 @@ int create_bundle(struct repository *r, const char *path,
 	int bundle_fd = -1;
 	int bundle_to_stdout;
 	int ref_count = 0;
-	struct rev_info revs;
+	struct rev_info revs, revs_copy;
 	int min_version = the_hash_algo == &hash_algos[GIT_HASH_SHA1] ? 2 : 3;
+	struct bundle_prerequisites_info bpi;
+	int i;
 
 	bundle_to_stdout = !strcmp(path, "-");
 	if (bundle_to_stdout)
@@ -512,10 +504,6 @@ int create_bundle(struct repository *r, const char *path,
 	save_commit_buffer = 0;
 	repo_init_revisions(r, &revs, NULL);
 
-	/* write prerequisites */
-	if (compute_and_write_prerequisites(bundle_fd, &revs, argc, argv))
-		goto err;
-
 	argc = setup_revisions(argc, argv, &revs, NULL);
 
 	if (argc > 1) {
@@ -523,16 +511,37 @@ int create_bundle(struct repository *r, const char *path,
 		goto err;
 	}
 
-	object_array_remove_duplicates(&revs.pending);
+	/* save revs.pending in revs_copy for later use */
+	memcpy(&revs_copy, &revs, sizeof(revs));
+	revs_copy.pending.nr = 0;
+	revs_copy.pending.alloc = 0;
+	revs_copy.pending.objects = NULL;
+	for (i = 0; i < revs.pending.nr; i++) {
+		struct object_array_entry *e = revs.pending.objects + i;
+		if (e)
+			add_object_array_with_path(e->item, e->name,
+						   &revs_copy.pending,
+						   e->mode, e->path);
+	}
 
-	ref_count = write_bundle_refs(bundle_fd, &revs);
+	/* write prerequisites */
+	revs.boundary = 1;
+	if (prepare_revision_walk(&revs))
+		die("revision walk setup failed");
+	bpi.fd = bundle_fd;
+	bpi.pending = &revs_copy.pending;
+	traverse_commit_list(&revs, write_bundle_prerequisites, NULL, &bpi);
+	object_array_remove_duplicates(&revs_copy.pending);
+
+	/* write bundle refs */
+	ref_count = write_bundle_refs(bundle_fd, &revs_copy);
 	if (!ref_count)
 		die(_("Refusing to create empty bundle."));
 	else if (ref_count < 0)
 		goto err;
 
 	/* write pack */
-	if (write_pack_data(bundle_fd, &revs, pack_options))
+	if (write_pack_data(bundle_fd, &revs_copy, pack_options))
 		goto err;
 
 	if (!bundle_to_stdout) {
t/t5607-clone-bundle.sh +2 -2
index 26985f4b44..425258767d 100755
@@ -38,13 +38,13 @@ test_expect_success 'die if bundle file cannot be created' '
 	test_must_fail git bundle create adir --all
 '
 
-test_expect_failure 'bundle --stdin' '
+test_expect_success 'bundle --stdin' '
 	echo master | git bundle create stdin-bundle.bdl --stdin &&
 	git ls-remote stdin-bundle.bdl >output &&
 	grep master output
 '
 
-test_expect_failure 'bundle --stdin <rev-list options>' '
+test_expect_success 'bundle --stdin <rev-list options>' '
 	echo master | git bundle create hybrid-bundle.bdl --stdin tag &&
 	git ls-remote hybrid-bundle.bdl >output &&
 	grep master output
Signature · 1 line
2.30.0.1.gade423aef4
#1 lore raw
Quoted 2 lines · Jiang Xin

Therefore, before calling prepare_revision_walk() function, make a copy on revs.pending for later use.

The in-core objects pointed by the list elements are shared between the original and the copy, and the object flag bits that are used to control the traversal (like SEEN, SHOWN and BOUNDARY bits) would be smudged during the traversal. So depending on how the "later use" uses the copied list, it may or may not be sufficient to just copy the list.

Apparently, you've tested the updated code well enough to send to the list, so it must be sufficient to make a copy of the list to support the way the updated code uses it, but it is not clear how it is so, only from what is in the proposed log message.

author
#2 lore raw

PATCH v2 1/2 bundle: lost objects when removing duplicate pendings

From: Jiang Xin <[email protected]>

git rev-list will list one commit for the following command:

    $ git rev-list 'main^!'
    <tip-commit-of-main-branch>

But providing the same rev-list args to git bundle, fail to create a bundle file.

    $ git bundle create - 'main^!'
    # v2 git bundle
    -<OID> <one-line-message>

fatal: Refusing to create empty bundle.

This is because when removing duplicate objects in function object_array_remove_duplicates(), one unique pending object which has the same name is deleted by mistake. The revision arg 'main^!' in the above example is parsed by handle_revision_arg(), and at lease two different objects will be appended to revs.pending, one points to the parent commit of the "main" branch, and the other points to the tip commit of the "main" branch. These two objects have the same name "main". Only one object is left with the name "main" after calling the function object_array_remove_duplicates().

And what's worse, when adding boundary commits into pending list, we use one-line commit message as names, and the arbitory names may surprise git-bundle.

Only comparing objects themselves (".item") is also not good enough, because user may want to create a bundle with two identical objects but with different reference names, such as: "HEAD" and "refs/heads/main".

Add new function contains_object() which compare both the address and the name of the object.

---
 object.c               |  10 +-
 t/t6020-bundle-misc.sh | 413 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 419 insertions(+), 4 deletions(-)
 create mode 100755 t/t6020-bundle-misc.sh
object.c +6 -4
index 68f80b0b3d..98017bed8e 100644
@@ -412,15 +412,16 @@ void object_array_clear(struct object_array *array)
 }
 
 /*
- * Return true iff array already contains an entry with name.
+ * Return true if array already contains an entry.
  */
-static int contains_name(struct object_array *array, const char *name)
+static int contains_object(struct object_array *array,
+			   const struct object *item, const char *name)
 {
 	unsigned nr = array->nr, i;
 	struct object_array_entry *object = array->objects;
 
 	for (i = 0; i < nr; i++, object++)
-		if (!strcmp(object->name, name))
+		if (item == object->item && !strcmp(object->name, name))
 			return 1;
 	return 0;
 }
@@ -432,7 +433,8 @@ void object_array_remove_duplicates(struct object_array *array)
 
 	array->nr = 0;
 	for (src = 0; src < nr; src++) {
-		if (!contains_name(array, objects[src].name)) {
+		if (!contains_object(array, objects[src].item,
+				     objects[src].name)) {
 			if (src != array->nr)
 				objects[array->nr] = objects[src];
 			array->nr++;
t/t6020-bundle-misc.sh new file +413 -0

413 lines not on this page · raw message

Signature · 1 line
2.30.0.4.g09ee524cd5
author
#3 lore raw

PATCH v2 2/2 bundle: arguments can be read from stdin

In reply to Junio C Hamano

From: Jiang Xin <[email protected]>

In order to create an incremental bundle, we need to pass many arguments to let git-bundle ignore some already packed commits. It will be more convenient to pass args via stdin. But the current implementation does not allow us to do this.

This is because args are parsed twice when creating bundle. The first time for parsing args is in compute_and_write_prerequisites() by running git-rev-list command to write prerequisites in bundle file, and stdin is consumed in this step if "--stdin" option is provided for git-bundle. Later nothing can be read from stdin when running setup_revisions() in create_bundle().

Remove the entire compute_and_write_prerequisites() function, and parse the args once by setup_revisions(). The first step for creating a bundle is to write prerequisites ("-" obj-id SP comment LF), but after calling prepare_revision_walk(), the revs.pending is left empty. Following steps could not work properly without data in revs.pending. Therefore, before calling prepare_revision_walk() function, make a copy revs_copy from revs for later use. Even though revs_copy and revs share the same objects, but changes on these objects will not change the behavior of function write_bundle_refs() and write_pack_data().

Also add testcases for git bundle in t6020, which read args from stdin.

---
 bundle.c                | 109 ++++++++++++++++++++++------------------
 t/t5607-clone-bundle.sh |   4 +-
 t/t6020-bundle-misc.sh  |  85 +++++++++++++++++++++++++++++--
 3 files changed, 142 insertions(+), 56 deletions(-)
bundle.c +59 -50
index cb0e5931ac..693d619551 100644
@@ -338,48 +338,6 @@ static int write_pack_data(int bundle_fd, struct rev_info *revs, struct strvec *
 	return 0;
 }
 
-static int compute_and_write_prerequisites(int bundle_fd,
-					   struct rev_info *revs,
-					   int argc, const char **argv)
-{
-	struct child_process rls = CHILD_PROCESS_INIT;
-	struct strbuf buf = STRBUF_INIT;
-	FILE *rls_fout;
-	int i;
-
-	strvec_pushl(&rls.args,
-		     "rev-list", "--boundary", "--pretty=oneline",
-		     NULL);
-	for (i = 1; i < argc; i++)
-		strvec_push(&rls.args, argv[i]);
-	rls.out = -1;
-	rls.git_cmd = 1;
-	if (start_command(&rls))
-		return -1;
-	rls_fout = xfdopen(rls.out, "r");
-	while (strbuf_getwholeline(&buf, rls_fout, '\n') != EOF) {
-		struct object_id oid;
-		if (buf.len > 0 && buf.buf[0] == '-') {
-			write_or_die(bundle_fd, buf.buf, buf.len);
-			if (!get_oid_hex(buf.buf + 1, &oid)) {
-				struct object *object = parse_object_or_die(&oid,
-									    buf.buf);
-				object->flags |= UNINTERESTING;
-				add_pending_object(revs, object, buf.buf);
-			}
-		} else if (!get_oid_hex(buf.buf, &oid)) {
-			struct object *object = parse_object_or_die(&oid,
-								    buf.buf);
-			object->flags |= SHOWN;
-		}
-	}
-	strbuf_release(&buf);
-	fclose(rls_fout);
-	if (finish_command(&rls))
-		return error(_("rev-list died"));
-	return 0;
-}
-
 /*
  * Write out bundle refs based on the tips already
  * parsed into revs.pending. As a side effect, may
@@ -474,6 +432,38 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs)
 	return ref_count;
 }
 
+struct bundle_prerequisites_info {
+	struct object_array *pending;
+	int fd;
+};
+
+static void write_bundle_prerequisites(struct commit *commit, void *data)
+{
+	struct bundle_prerequisites_info *bpi = data;
+	struct object *object;
+	struct pretty_print_context ctx = { 0 };
+	struct strbuf buf = STRBUF_INIT;
+
+	if (!(commit->object.flags & BOUNDARY))
+		return;
+	strbuf_addf(&buf, "-%s ", oid_to_hex(&commit->object.oid));
+	write_or_die(bpi->fd, buf.buf, buf.len);
+
+	ctx.fmt = CMIT_FMT_ONELINE;
+	ctx.output_encoding = get_log_output_encoding();
+	strbuf_reset(&buf);
+	pretty_print_commit(&ctx, commit, &buf);
+	strbuf_trim(&buf);
+
+	object = (struct object *)commit;
+	object->flags |= UNINTERESTING;
+	add_object_array_with_path(object, buf.buf, bpi->pending, S_IFINVALID,
+				   NULL);
+	strbuf_addch(&buf, '\n');
+	write_or_die(bpi->fd, buf.buf, buf.len);
+	strbuf_release(&buf);
+}
+
 int create_bundle(struct repository *r, const char *path,
 		  int argc, const char **argv, struct strvec *pack_options, int version)
 {
@@ -481,8 +471,10 @@ int create_bundle(struct repository *r, const char *path,
 	int bundle_fd = -1;
 	int bundle_to_stdout;
 	int ref_count = 0;
-	struct rev_info revs;
+	struct rev_info revs, revs_copy;
 	int min_version = the_hash_algo == &hash_algos[GIT_HASH_SHA1] ? 2 : 3;
+	struct bundle_prerequisites_info bpi;
+	int i;
 
 	bundle_to_stdout = !strcmp(path, "-");
 	if (bundle_to_stdout)
@@ -512,10 +504,6 @@ int create_bundle(struct repository *r, const char *path,
 	save_commit_buffer = 0;
 	repo_init_revisions(r, &revs, NULL);
 
-	/* write prerequisites */
-	if (compute_and_write_prerequisites(bundle_fd, &revs, argc, argv))
-		goto err;
-
 	argc = setup_revisions(argc, argv, &revs, NULL);
 
 	if (argc > 1) {
@@ -523,16 +511,37 @@ int create_bundle(struct repository *r, const char *path,
 		goto err;
 	}
 
-	object_array_remove_duplicates(&revs.pending);
+	/* save revs.pending in revs_copy for later use */
+	memcpy(&revs_copy, &revs, sizeof(revs));
+	revs_copy.pending.nr = 0;
+	revs_copy.pending.alloc = 0;
+	revs_copy.pending.objects = NULL;
+	for (i = 0; i < revs.pending.nr; i++) {
+		struct object_array_entry *e = revs.pending.objects + i;
+		if (e)
+			add_object_array_with_path(e->item, e->name,
+						   &revs_copy.pending,
+						   e->mode, e->path);
+	}
 
-	ref_count = write_bundle_refs(bundle_fd, &revs);
+	/* write prerequisites */
+	revs.boundary = 1;
+	if (prepare_revision_walk(&revs))
+		die("revision walk setup failed");
+	bpi.fd = bundle_fd;
+	bpi.pending = &revs_copy.pending;
+	traverse_commit_list(&revs, write_bundle_prerequisites, NULL, &bpi);
+	object_array_remove_duplicates(&revs_copy.pending);
+
+	/* write bundle refs */
+	ref_count = write_bundle_refs(bundle_fd, &revs_copy);
 	if (!ref_count)
 		die(_("Refusing to create empty bundle."));
 	else if (ref_count < 0)
 		goto err;
 
 	/* write pack */
-	if (write_pack_data(bundle_fd, &revs, pack_options))
+	if (write_pack_data(bundle_fd, &revs_copy, pack_options))
 		goto err;
 
 	if (!bundle_to_stdout) {
t/t5607-clone-bundle.sh +2 -2
index 26985f4b44..425258767d 100755
@@ -38,13 +38,13 @@ test_expect_success 'die if bundle file cannot be created' '
 	test_must_fail git bundle create adir --all
 '
 
-test_expect_failure 'bundle --stdin' '
+test_expect_success 'bundle --stdin' '
 	echo master | git bundle create stdin-bundle.bdl --stdin &&
 	git ls-remote stdin-bundle.bdl >output &&
 	grep master output
 '
 
-test_expect_failure 'bundle --stdin <rev-list options>' '
+test_expect_success 'bundle --stdin <rev-list options>' '
 	echo master | git bundle create hybrid-bundle.bdl --stdin tag &&
 	git ls-remote hybrid-bundle.bdl >output &&
 	grep master output
t/t6020-bundle-misc.sh +81 -4

147 lines not on this page · raw message

Signature · 1 line
2.30.0.4.g09ee524cd5
author
#4 lore raw

v3 cover improvements for git-bundle

In reply to Junio C Hamano

From: Jiang Xin <[email protected]>

## Introduce two improvements for git-bundle

+ Commit "bundle: lost objects when removing duplicate pendings", which fixes command like:

        $ git bundle create <file> 'master^!'

+ Commits "bundle: arguments can be read from stdin", which add "--stdin" option support for git-bundle, like:

        $ git bundle create <file> <input

## Changes of v3

1. Forgot to add shebang in file "t/t6020-bundle-misc.sh", which breaks build and test on Windows.

2. Add more testcases in t6020.

## Range diff of v2...v3

1: ba13820340 ! 1: 9df48434f3 bundle: lost objects when removing duplicate pendings

    @@ object.c: void object_array_remove_duplicates(struct object_array *array)
     
      ## t/t6020-bundle-misc.sh (new) ##
     @@
    ++#!/bin/sh
    ++#
    ++# Copyright (c) 2021 Jiang Xin
    ++#
    ++
     +test_description='Test git-bundle'
     +
     +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
    @@ t/t6020-bundle-misc.sh (new)
     +. ./test-lib.sh
     +
     +test_bundle_object_count () {
    -+	git verify-pack -v "$1" >verify.out &&
    ++	bundle=$1 &&
    ++	pack=${bundle%.bdl}.pack &&
    ++	convert_bundle_to_pack <"$bundle" >"$pack" &&
    ++	git index-pack "$pack" &&
    ++	git verify-pack -v "$pack" >verify.out &&
    ++	count=$(grep "^$OID_REGEX " verify.out | wc -l) &&
    ++	test $2 = $count && return 0
    ++	echo object count for $bundle is $count, not $2
    ++	return 1
    ++}
    ++
    ++
    ++test_thin_bundle_object_count () {
    ++	bundle=$1 &&
    ++	pack=${bundle%.bdl}.pack &&
    ++	convert_bundle_to_pack <"$bundle" |
    ++		test_must_fail git index-pack --stdin "$pack" &&
    ++	rm -f "$pack" &&
    ++	convert_bundle_to_pack <"$bundle" |
    ++		git index-pack --stdin --fix-thin "$pack" &&
    ++	git verify-pack -v "$pack" >verify.out &&
     +	count=$(grep "^$OID_REGEX " verify.out | wc -l) &&
     +	test $2 = $count && return 0
    -+	echo object count is $count, not $2
    ++	echo object count for $bundle is $count, not $2
     +	return 1
     +}
     +
    @@ t/t6020-bundle-misc.sh (new)
     +
     +#            (C)   (D, pull/1/head, topic/1)
     +#             o --- o
    -+#            /       \                              (L, tags/v1)
    -+#           /         \        o (H, topic/2)             (M, tags/v2)
    -+#          /    (F)    \      /                                 (N, tags/v3)
    ++#            /       \                              (L)
    ++#           /         \        o (H, topic/2)             (M, tag:v2)
    ++#          /    (F)    \      /                                 (N, tag:v3)
     +#         /      o --------- o (G, pull/2/head)      o --- o --- o (release)
     +#        /      /        \    \                      /       \
     +#  o --- o --- o -------- o -- o ------------------ o ------- o --- o (main)
    -+# (A)   (B)   (E)        (I)  (J)                  (K)       (O)   (P)
    ++# (A)   (B)  (E, tag:v1) (I)  (J)                  (K)       (O)   (P)
     +#
     +test_expect_success 'setup' '
     +	# commit A & B
    @@ t/t6020-bundle-misc.sh (new)
     +	test_tick &&
     +	git commit -m "Commit E" &&
     +	E=$(git rev-parse HEAD) &&
    ++	test_tick &&
    ++	git tag -m "v1" v1 HEAD &&
    ++	TAG1=$(git rev-parse refs/tags/v1) &&
     +
     +	# branch topic/2
     +	git checkout -b topic/2 &&
    @@ t/t6020-bundle-misc.sh (new)
     +	git add release.txt &&
     +	test_tick &&
     +	git commit -m "Commit L" &&
    -+	test_tick &&
    -+	git tag -m "v1" v1 HEAD &&
     +
     +	cat >release.txt <<-EOF &&
     +		Commit M
    @@ t/t6020-bundle-misc.sh (new)
     +	L=$(git rev-parse HEAD~2) &&
     +	M=$(git rev-parse HEAD~) &&
     +	N=$(git rev-parse HEAD) &&
    -+	TAG1=$(git rev-parse refs/tags/v1) &&
     +	TAG2=$(git rev-parse refs/tags/v2) &&
     +	TAG3=$(git rev-parse refs/tags/v3) &&
     +
    @@ t/t6020-bundle-misc.sh (new)
     +		EOF
     +	test_i18ncmp expect actual &&
     +
    -+	convert_bundle_to_pack <special-rev.bdl >special-rev.pack &&
    -+	git index-pack special-rev.pack &&
    -+	test_bundle_object_count special-rev.pack 3
    ++	test_bundle_object_count special-rev.bdl 3
    ++'
    ++
    ++test_expect_success 'create bundle with --max-count option' '
    ++	git bundle create max-count.bdl --max-count 1 \
    ++		main \
    ++		"^release" \
    ++		refs/tags/v1 \
    ++		refs/pull/1/head \
    ++		refs/pull/2/head &&
    ++
    ++	git bundle list-heads max-count.bdl |
    ++		make_user_friendly_and_stable_output >actual &&
    ++	cat >expect <<-EOF &&
    ++		<COMMIT-P> refs/heads/main
    ++		<TAG-1> refs/tags/v1
    ++		EOF
    ++	test_i18ncmp expect actual &&
    ++
    ++	git bundle verify max-count.bdl |
    ++		make_user_friendly_and_stable_output >actual &&
    ++	cat >expect <<-EOF &&
    ++		The bundle contains these 2 refs:
    ++		<COMMIT-P> refs/heads/main
    ++		<TAG-1> refs/tags/v1
    ++		The bundle requires this ref:
    ++		<COMMIT-O>
    ++		EOF
    ++	test_i18ncmp expect actual &&
    ++
    ++	test_bundle_object_count max-count.bdl 4
    ++'
    ++
    ++test_expect_success 'create bundle with --since option' '
    ++	git bundle create since.bdl \
    ++		--since "Thu Apr 7 15:26:13 2005 -0700" \
    ++		--all &&
    ++
    ++	git bundle list-heads since.bdl |
    ++		make_user_friendly_and_stable_output >actual &&
    ++	cat >expect <<-EOF &&
    ++		<COMMIT-P> refs/heads/main
    ++		<COMMIT-N> refs/heads/release
    ++		<TAG-2> refs/tags/v2
    ++		<TAG-3> refs/tags/v3
    ++		<COMMIT-P> HEAD
    ++		EOF
    ++	test_i18ncmp expect actual &&
    ++
    ++	git bundle verify since.bdl |
    ++		make_user_friendly_and_stable_output >actual &&
    ++	cat >expect <<-EOF &&
    ++		The bundle contains these 5 refs:
    ++		<COMMIT-P> refs/heads/main
    ++		<COMMIT-N> refs/heads/release
    ++		<TAG-2> refs/tags/v2
    ++		<TAG-3> refs/tags/v3
    ++		<COMMIT-P> HEAD
    ++		The bundle requires these 2 refs:
    ++		<COMMIT-L>
    ++		<COMMIT-K>
    ++		EOF
    ++	test_i18ncmp expect actual &&
    ++
    ++	test_thin_bundle_object_count since.bdl 16
     +'
     +
     +test_expect_success 'create bundle 1 - no prerequisites' '
    -+	# create bundle from args
     +	git bundle create 1.bdl topic/1 topic/2 &&
     +
     +	cat >expect <<-EOF &&
    @@ t/t6020-bundle-misc.sh (new)
     +		make_user_friendly_and_stable_output >actual &&
     +	test_i18ncmp expect actual &&
     +
    -+	convert_bundle_to_pack <1.bdl >1.pack &&
    -+	git index-pack 1.pack &&
    -+	test_bundle_object_count 1.pack 24
    ++	test_bundle_object_count 1.bdl 24
     +'
     +
     +test_expect_success 'create bundle 2 - has prerequisites' '
    -+	# create bundle from args
     +	git bundle create 2.bdl \
     +		--ignore-missing \
     +		^topic/deleted \
    @@ t/t6020-bundle-misc.sh (new)
     +		make_user_friendly_and_stable_output >actual &&
     +	test_i18ncmp expect actual &&
     +
    -+	convert_bundle_to_pack <2.bdl >2.pack &&
    -+	git index-pack 2.pack &&
    -+	test_bundle_object_count 2.pack 16
    ++	test_bundle_object_count 2.bdl 16
     +'
     +
     +test_expect_success 'fail to verify bundle without prerequisites' '
    @@ t/t6020-bundle-misc.sh (new)
     +'
     +
     +test_expect_success 'create bundle 3 - two refs, same object' '
    -+	# create bundle from args
     +	git bundle create --version=3 3.bdl \
     +		^release \
     +		^topic/1 \
    @@ t/t6020-bundle-misc.sh (new)
     +		make_user_friendly_and_stable_output >actual &&
     +	test_i18ncmp expect actual &&
     +
    -+	convert_bundle_to_pack <3.bdl >3.pack &&
    -+	git index-pack 3.pack &&
    -+	test_bundle_object_count 3.pack 4
    ++	test_bundle_object_count 3.bdl 4
     +'
     +
     +test_expect_success 'create bundle 4 - with tags' '
    -+	# create bundle from args
     +	git bundle create 4.bdl \
     +		^main \
     +		^release \
    @@ t/t6020-bundle-misc.sh (new)
     +		make_user_friendly_and_stable_output >actual &&
     +	test_i18ncmp expect actual &&
     +
    -+	convert_bundle_to_pack <4.bdl >4.pack &&
    -+	git index-pack 4.pack &&
    -+	test_bundle_object_count 4.pack 3
    ++	test_bundle_object_count 4.bdl 3
     +'
     +
     +test_expect_success 'clone from bundle' '

2: a4662f44a8 ! 2: 86ad41e4d4 bundle: arguments can be read from stdin
@@ Commit message
git-bundle. Later nothing can be read from stdin when running setup_revisions() in create_bundle().

- Remove the entire compute_and_write_prerequisites() function, and
- parse the args once by setup_revisions(). The first step for creating
- a bundle is to write prerequisites ("-" obj-id SP comment LF), but after
- calling prepare_revision_walk(), the revs.pending is left empty.
- Following steps could not work properly without data in revs.pending.
- Therefore, before calling prepare_revision_walk() function, make a
- copy revs_copy from revs for later use. Even though revs_copy and
- revs share the same objects, but changes on these objects will not
- change the behavior of function write_bundle_refs() and
- write_pack_data().
+ The solution is to parse args once by removing the entire function
+ compute_and_write_prerequisites() and then calling function
+ setup_revisions(). In order to write prerequisites for bundle, will
+ call prepare_revision_walk() and traverse_commit_list(). But after
+ calling prepare_revision_walk(), the object array revs.pending is
+ left empty, and the following steps could not work properly with the
+ empty object array (revs.pending). Therefore, make a copy of revs
+ to revs_copy for later use right after calling setup_revisions().
+
+ The copy of revs_copy is not a deep copy, it shares the same objects
+ with revs. The object array of revs has been cleared, but objects
+ themselves are still kept. Flags of objects may change after calling
+ prepare_revision_walk(), we can use these changed flags without
+ calling the git rev-list command and parsing its output like the
+ former implementation.

Also add testcases for git bundle in t6020, which read args from stdin.

@@ bundle.c: static int write_pack_data(int bundle_fd, struct rev_info *revs, struc
/*
* Write out bundle refs based on the tips already
* parsed into revs.pending. As a side effect, may
+@@ bundle.c: static int write_bundle_refs(int bundle_fd, struct rev_info *revs)
+ * constraints.
+ */
+ if (!(e->item->flags & SHOWN) && e->item->type == OBJ_COMMIT) {
+- warning(_("ref '%s' is excluded by the rev-list options"),
++ warning(_("ref '%s' is excluded by the limiting options"),
+ e->name);
+ goto skip_write_ref;
+ }
@@ bundle.c: static int write_bundle_refs(int bundle_fd, struct rev_info *revs)
return ref_count;
}
@@ t/t5607-clone-bundle.sh: test_expect_success 'die if bundle file cannot be creat
grep master output

## t/t6020-bundle-misc.sh ##
-@@ t/t6020-bundle-misc.sh: test_expect_success 'create bundle 1 - no prerequisites' '
- # create bundle from args
+@@ t/t6020-bundle-misc.sh: test_expect_success 'create bundle with --since option' '
+ '
+
+ test_expect_success 'create bundle 1 - no prerequisites' '
++ # create bundle from args
git bundle create 1.bdl topic/1 topic/2 &&

+ # create bundle from stdin
@@ t/t6020-bundle-misc.sh: test_expect_success 'create bundle 1 - no prerequisites'
make_user_friendly_and_stable_output >actual &&
test_i18ncmp expect actual &&

+- test_bundle_object_count 1.bdl 24
+ git bundle verify stdin-1.bdl |
+ make_user_friendly_and_stable_output >actual &&
+ test_i18ncmp expect actual &&
+
- convert_bundle_to_pack <1.bdl >1.pack &&
- git index-pack 1.pack &&
-- test_bundle_object_count 1.pack 24
-+ test_bundle_object_count 1.pack 24 &&
-+
-+ convert_bundle_to_pack <stdin-1.bdl >stdin-1.pack &&
-+ git index-pack stdin-1.pack &&
-+ test_bundle_object_count stdin-1.pack 24
++ test_bundle_object_count 1.bdl 24 &&
++ test_bundle_object_count stdin-1.bdl 24
'

test_expect_success 'create bundle 2 - has prerequisites' '
++ # create bundle from args
+ git bundle create 2.bdl \
+ --ignore-missing \
+ ^topic/deleted \
@@ t/t6020-bundle-misc.sh: test_expect_success 'create bundle 2 - has prerequisites' '
^topic/2 \
release &&
@@ t/t6020-bundle-misc.sh: test_expect_success 'create bundle 2 - has prerequisites
make_user_friendly_and_stable_output >actual &&
test_i18ncmp expect actual &&

+- test_bundle_object_count 2.bdl 16
+ git bundle verify stdin-2.bdl |
+ make_user_friendly_and_stable_output >actual &&
+ test_i18ncmp expect actual &&
+
- convert_bundle_to_pack <2.bdl >2.pack &&
- git index-pack 2.pack &&
-- test_bundle_object_count 2.pack 16
-+ test_bundle_object_count 2.pack 16 &&
-+
-+ convert_bundle_to_pack <stdin-2.bdl >stdin-2.pack &&
-+ git index-pack stdin-2.pack &&
-+ test_bundle_object_count stdin-2.pack 16
++ test_bundle_object_count 2.bdl 16 &&
++ test_bundle_object_count stdin-2.bdl 16
'

test_expect_success 'fail to verify bundle without prerequisites' '
@@ t/t6020-bundle-misc.sh: test_expect_success 'fail to verify bundle without prere
test_i18ncmp expect actual
'

+ test_expect_success 'create bundle 3 - two refs, same object' '
++ # create bundle from args
+ git bundle create --version=3 3.bdl \
+ ^release \
+ ^topic/1 \
@@ t/t6020-bundle-misc.sh: test_expect_success 'create bundle 3 - two refs, same object' '
main \
HEAD &&
@@ t/t6020-bundle-misc.sh: test_expect_success 'create bundle 3 - two refs, same ob
make_user_friendly_and_stable_output >actual &&
test_i18ncmp expect actual &&

+- test_bundle_object_count 3.bdl 4
+ git bundle verify stdin-3.bdl |
+ make_user_friendly_and_stable_output >actual &&
+ test_i18ncmp expect actual &&
+
- convert_bundle_to_pack <3.bdl >3.pack &&
- git index-pack 3.pack &&
-- test_bundle_object_count 3.pack 4
-+ test_bundle_object_count 3.pack 4 &&
-+
-+ convert_bundle_to_pack <stdin-3.bdl >stdin-3.pack &&
-+ git index-pack stdin-3.pack &&
-+ test_bundle_object_count stdin-3.pack 4
++ test_bundle_object_count 3.bdl 4 &&
++ test_bundle_object_count stdin-3.bdl 4
'

test_expect_success 'create bundle 4 - with tags' '
++ # create bundle from args
+ git bundle create 4.bdl \
+ ^main \
+ ^release \
@@ t/t6020-bundle-misc.sh: test_expect_success 'create bundle 4 - with tags' '
^topic/2 \
--all &&
@@ t/t6020-bundle-misc.sh: test_expect_success 'create bundle 4 - with tags' '
make_user_friendly_and_stable_output >actual &&
test_i18ncmp expect actual &&

+- test_bundle_object_count 4.bdl 3
+ git bundle verify stdin-4.bdl |
+ make_user_friendly_and_stable_output >actual &&
+ test_i18ncmp expect actual &&
+
- convert_bundle_to_pack <4.bdl >4.pack &&
- git index-pack 4.pack &&
-- test_bundle_object_count 4.pack 3
-+ test_bundle_object_count 4.pack 3 &&
-+
-+ convert_bundle_to_pack <stdin-4.bdl >stdin-4.pack &&
-+ git index-pack stdin-4.pack &&
-+ test_bundle_object_count stdin-4.pack 3
++ test_bundle_object_count 4.bdl 3 &&
++ test_bundle_object_count stdin-4.bdl 3
'

test_expect_success 'clone from bundle' '

--

Jiang Xin (2):
bundle: lost objects when removing duplicate pendings
bundle: arguments can be read from stdin

 bundle.c                | 111 ++++----
 object.c                |  10 +-
 t/t5607-clone-bundle.sh |   4 +-
 t/t6020-bundle-misc.sh  | 557 ++++++++++++++++++++++++++++++++++++++++
 4 files changed, 625 insertions(+), 57 deletions(-)
 create mode 100755 t/t6020-bundle-misc.sh
Signature · 1 line
2.30.0.2.g06d2f50715
author
#5 lore raw

PATCH v3 1/2 bundle: lost objects when removing duplicate pendings

In reply to Junio C Hamano

From: Jiang Xin <[email protected]>

git rev-list will list one commit for the following command:

    $ git rev-list 'main^!'
    <tip-commit-of-main-branch>

But providing the same rev-list args to git bundle, fail to create a bundle file.

    $ git bundle create - 'main^!'
    # v2 git bundle
    -<OID> <one-line-message>

fatal: Refusing to create empty bundle.

This is because when removing duplicate objects in function object_array_remove_duplicates(), one unique pending object which has the same name is deleted by mistake. The revision arg 'main^!' in the above example is parsed by handle_revision_arg(), and at lease two different objects will be appended to revs.pending, one points to the parent commit of the "main" branch, and the other points to the tip commit of the "main" branch. These two objects have the same name "main". Only one object is left with the name "main" after calling the function object_array_remove_duplicates().

And what's worse, when adding boundary commits into pending list, we use one-line commit message as names, and the arbitory names may surprise git-bundle.

Only comparing objects themselves (".item") is also not good enough, because user may want to create a bundle with two identical objects but with different reference names, such as: "HEAD" and "refs/heads/main".

Add new function contains_object() which compare both the address and the name of the object.

---
 object.c               |  10 +-
 t/t6020-bundle-misc.sh | 488 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 494 insertions(+), 4 deletions(-)
 create mode 100755 t/t6020-bundle-misc.sh
object.c +6 -4
index 68f80b0b3d..98017bed8e 100644
@@ -412,15 +412,16 @@ void object_array_clear(struct object_array *array)
 }
 
 /*
- * Return true iff array already contains an entry with name.
+ * Return true if array already contains an entry.
  */
-static int contains_name(struct object_array *array, const char *name)
+static int contains_object(struct object_array *array,
+			   const struct object *item, const char *name)
 {
 	unsigned nr = array->nr, i;
 	struct object_array_entry *object = array->objects;
 
 	for (i = 0; i < nr; i++, object++)
-		if (!strcmp(object->name, name))
+		if (item == object->item && !strcmp(object->name, name))
 			return 1;
 	return 0;
 }
@@ -432,7 +433,8 @@ void object_array_remove_duplicates(struct object_array *array)
 
 	array->nr = 0;
 	for (src = 0; src < nr; src++) {
-		if (!contains_name(array, objects[src].name)) {
+		if (!contains_object(array, objects[src].item,
+				     objects[src].name)) {
 			if (src != array->nr)
 				objects[array->nr] = objects[src];
 			array->nr++;
t/t6020-bundle-misc.sh new file +488 -0

488 lines not on this page · raw message

Signature · 1 line
2.30.0.2.g06d2f50715
#6 lore raw

Re: [PATCH v3 1/2] bundle: lost objects when removing duplicate pendings

Quoted 55 lines · Jiang Xin

From: Jiang Xin <[email protected]>

git rev-list will list one commit for the following command:

    $ git rev-list 'main^!'
    <tip-commit-of-main-branch>

But providing the same rev-list args to git bundle, fail to create a bundle file.

    $ git bundle create - 'main^!'
    # v2 git bundle
    -<OID> <one-line-message>

fatal: Refusing to create empty bundle.

This is because when removing duplicate objects in function object_array_remove_duplicates(), one unique pending object which has the same name is deleted by mistake. The revision arg 'main^!' in the above example is parsed by handle_revision_arg(), and at lease two different objects will be appended to revs.pending, one points to the parent commit of the "main" branch, and the other points to the tip commit of the "main" branch. These two objects have the same name "main". Only one object is left with the name "main" after calling the function object_array_remove_duplicates().

And what's worse, when adding boundary commits into pending list, we use one-line commit message as names, and the arbitory names may surprise git-bundle.

Only comparing objects themselves (".item") is also not good enough, because user may want to create a bundle with two identical objects but with different reference names, such as: "HEAD" and "refs/heads/main".

Add new function contains_object() which compare both the address and the name of the object.

---
 object.c               |  10 +-
 t/t6020-bundle-misc.sh | 488 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 494 insertions(+), 4 deletions(-)
 create mode 100755 t/t6020-bundle-misc.sh
object.c +1 -1
index 68f80b0b3d..98017bed8e 100644
@@ -412,15 +412,16 @@ void object_array_clear(struct object_array *array)
 }
 
 /*
- * Return true iff array already contains an entry with name.
+ * Return true if array already contains an entry.
  */

object.c void object_array_clear(struct object_array *array) · line 424

Quoted 10 lines
object.c +3 -2
-static int contains_name(struct object_array *array, const char *name)
+static int contains_object(struct object_array *array,
+			   const struct object *item, const char *name)
 {
 	unsigned nr = array->nr, i;
 	struct object_array_entry *object = array->objects;
 
 	for (i = 0; i < nr; i++, object++)
-		if (!strcmp(object->name, name))
+		if (item == object->item && !strcmp(object->name, name))

I think the comparison of item == object->item is a bit too fragile. Yes, we all know item must be an entry of array.
However, it's separated from its usage may lead to misuse in the future. Perhaps using oidcmp or adding a comment to indicate that item must be an entry of array.

Quoted 28 lines
object.c +2 -1
 			return 1;
 	return 0;
 }
@@ -432,7 +433,8 @@ void object_array_remove_duplicates(struct object_array *array)
 
 	array->nr = 0;
 	for (src = 0; src < nr; src++) {
-		if (!contains_name(array, objects[src].name)) {
+		if (!contains_object(array, objects[src].item,
+				     objects[src].name)) {
 			if (src != array->nr)
 				objects[array->nr] = objects[src];
 			array->nr++;
t/t6020-bundle-misc.sh new file +9 -0
new file mode 100755
index 0000000000..d15e67c8f7
@@ -0,0 +1,488 @@
+#!/bin/sh
+#
+# Copyright (c) 2021 Jiang Xin
+#
+
+test_description='Test git-bundle'
+
+GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
+export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME

t/t6020-bundle-misc.sh line 19

Quoted 10 lines
t/t6020-bundle-misc.sh new file +10 -0
+
+. ./test-lib.sh
+
+test_bundle_object_count () {
+	bundle=$1 &&
+	pack=${bundle%.bdl}.pack &&
+	convert_bundle_to_pack <"$bundle" >"$pack" &&
+	git index-pack "$pack" &&
+	git verify-pack -v "$pack" >verify.out &&
+	count=$(grep "^$OID_REGEX " verify.out | wc -l) &&

I think we can use 'grep -c' instead of grep .. | wc -l.

Or

	grep "^$OID_REGEX " verify.out >verify.filtered &&
	test_line_count = $2 verify.filtered

The same comment applied to test_thin_bundle_object_count

Quoted 17 lines
t/t6020-bundle-misc.sh new file +17 -0
+	test $2 = $count && return 0
+	echo object count for $bundle is $count, not $2
+	return 1
+}
+
+
+test_thin_bundle_object_count () {
+	bundle=$1 &&
+	pack=${bundle%.bdl}.pack &&
+	convert_bundle_to_pack <"$bundle" |
+		test_must_fail git index-pack --stdin "$pack" &&
+	rm -f "$pack" &&
+	convert_bundle_to_pack <"$bundle" |
+		git index-pack --stdin --fix-thin "$pack" &&
+	git verify-pack -v "$pack" >verify.out &&
+	count=$(grep "^$OID_REGEX " verify.out | wc -l) &&
+	test $2 = $count && return 0

t/t6020-bundle-misc.sh line 46

Quoted 10 lines
t/t6020-bundle-misc.sh new file +10 -0
+	echo object count for $bundle is $count, not $2
+	return 1
+}
+
+convert_bundle_to_pack () {
+	while read x && test -n "$x"
+	do
+		:;
+	done
+	cat

I'm not sure what you would expect in this case, but in my experience, I replace this whole block with

	sed '1,/^$/d'

also works.

IOW, I would apply this on top of your change:

----8<-----

t/t6020-bundle-misc.sh +5 -13
index 1f60fe180e..3a428454d7 100755
@@ -16,10 +16,8 @@ test_bundle_object_count () {
 	convert_bundle_to_pack <"$bundle" >"$pack" &&
 	git index-pack "$pack" &&
 	git verify-pack -v "$pack" >verify.out &&
-	count=$(grep "^$OID_REGEX " verify.out | wc -l) &&
-	test $2 = $count && return 0
-	echo object count for $bundle is $count, not $2
-	return 1
+	grep "^$OID_REGEX " verify.out >verify.filtered &&
+	test_line_count = $2 verify.filtered
 }
 
 
@@ -32,18 +30,12 @@ test_thin_bundle_object_count () {
 	convert_bundle_to_pack <"$bundle" |
 		git index-pack --stdin --fix-thin "$pack" &&
 	git verify-pack -v "$pack" >verify.out &&
-	count=$(grep "^$OID_REGEX " verify.out | wc -l) &&
-	test $2 = $count && return 0
-	echo object count for $bundle is $count, not $2
-	return 1
+	grep "^$OID_REGEX " verify.out >verify.filtered &&
+	test_line_count = $2 verify.filtered
 }
 
 convert_bundle_to_pack () {
-	while read x && test -n "$x"
-	do
-		:;
-	done
-	cat
+	sed '1,/^$/d'
 }
 
 # Format the output of git commands to make a user-friendly and stable

----->8-----

For the below change, I haven't checked but I think test_commit should work, no?

-- Danh

Quoted 445 lines
t/t6020-bundle-misc.sh new file +442 -0
+}
+
+# Format the output of git commands to make a user-friendly and stable
+# text.  We can easily prepare the expect text without having to worry
+# about future changes of the commit ID and spaces of the output.
+make_user_friendly_and_stable_output () {
+	sed \
+		-e "s/ *\$//" \
+		-e "s/$A/<COMMIT-A>/g" \
+		-e "s/$B/<COMMIT-B>/g" \
+		-e "s/$C/<COMMIT-C>/g" \
+		-e "s/$D/<COMMIT-D>/g" \
+		-e "s/$E/<COMMIT-E>/g" \
+		-e "s/$F/<COMMIT-F>/g" \
+		-e "s/$G/<COMMIT-G>/g" \
+		-e "s/$H/<COMMIT-H>/g" \
+		-e "s/$I/<COMMIT-I>/g" \
+		-e "s/$J/<COMMIT-J>/g" \
+		-e "s/$K/<COMMIT-K>/g" \
+		-e "s/$L/<COMMIT-L>/g" \
+		-e "s/$M/<COMMIT-M>/g" \
+		-e "s/$N/<COMMIT-N>/g" \
+		-e "s/$O/<COMMIT-O>/g" \
+		-e "s/$P/<COMMIT-P>/g" \
+		-e "s/$(echo $A | cut -c1-7)[0-9a-f]*/<OID-A>/g" \
+		-e "s/$(echo $B | cut -c1-7)[0-9a-f]*/<OID-B>/g" \
+		-e "s/$(echo $C | cut -c1-7)[0-9a-f]*/<OID-C>/g" \
+		-e "s/$(echo $D | cut -c1-7)[0-9a-f]*/<OID-D>/g" \
+		-e "s/$(echo $E | cut -c1-7)[0-9a-f]*/<OID-E>/g" \
+		-e "s/$(echo $F | cut -c1-7)[0-9a-f]*/<OID-F>/g" \
+		-e "s/$(echo $G | cut -c1-7)[0-9a-f]*/<OID-G>/g" \
+		-e "s/$(echo $H | cut -c1-7)[0-9a-f]*/<OID-H>/g" \
+		-e "s/$(echo $I | cut -c1-7)[0-9a-f]*/<OID-I>/g" \
+		-e "s/$(echo $J | cut -c1-7)[0-9a-f]*/<OID-J>/g" \
+		-e "s/$(echo $K | cut -c1-7)[0-9a-f]*/<OID-K>/g" \
+		-e "s/$(echo $L | cut -c1-7)[0-9a-f]*/<OID-L>/g" \
+		-e "s/$(echo $M | cut -c1-7)[0-9a-f]*/<OID-M>/g" \
+		-e "s/$(echo $N | cut -c1-7)[0-9a-f]*/<OID-N>/g" \
+		-e "s/$(echo $O | cut -c1-7)[0-9a-f]*/<OID-O>/g" \
+		-e "s/$(echo $P | cut -c1-7)[0-9a-f]*/<OID-P>/g" \
+		-e "s/$TAG1/<TAG-1>/g" \
+		-e "s/$TAG2/<TAG-2>/g" \
+		-e "s/$TAG3/<TAG-3>/g" \
+		-e "s/$(echo $TAG1 | cut -c1-7)[0-9a-f]*/<OID-TAG-1>/g" \
+		-e "s/$(echo $TAG2 | cut -c1-7)[0-9a-f]*/<OID-TAG-2>/g" \
+		-e "s/$(echo $TAG3 | cut -c1-7)[0-9a-f]*/<OID-TAG-3>/g" \
+		-e "s/$ZERO_OID/<ZERO-OID>/g"
+}
+
+#            (C)   (D, pull/1/head, topic/1)
+#             o --- o
+#            /       \                              (L)
+#           /         \        o (H, topic/2)             (M, tag:v2)
+#          /    (F)    \      /                                 (N, tag:v3)
+#         /      o --------- o (G, pull/2/head)      o --- o --- o (release)
+#        /      /        \    \                      /       \
+#  o --- o --- o -------- o -- o ------------------ o ------- o --- o (main)
+# (A)   (B)  (E, tag:v1) (I)  (J)                  (K)       (O)   (P)
+#
+test_expect_success 'setup' '
+	# commit A & B
+	cat >main.txt <<-EOF &&
+		Commit A
+		EOF
+	git add main.txt &&
+	test_tick &&
+	git commit -m "Commit A" &&
+
+	cat >main.txt <<-EOF &&
+		Commit B
+		EOF
+	git add main.txt &&
+	test_tick &&
+	git commit -m "Commit B" &&
+	A=$(git rev-parse HEAD~) &&
+	B=$(git rev-parse HEAD) &&
+
+	# branch topic/1
+	git checkout -b topic/1 &&
+	cat >topic-1.txt <<-EOF &&
+		Commit C
+		EOF
+	git add topic-1.txt &&
+	test_tick &&
+	git commit -m "Commit C" &&
+
+	cat >topic-1.txt <<-EOF &&
+		Commit D
+		EOF
+	git add -u &&
+	test_tick &&
+	git commit -m "Commit D" &&
+	git update-ref refs/pull/1/head HEAD &&
+	C=$(git rev-parse topic/1~) &&
+	D=$(git rev-parse topic/1) &&
+
+	# commit E
+	git checkout main &&
+	cat >main.txt <<-EOF &&
+		Commit E
+		EOF
+	git add main.txt &&
+	test_tick &&
+	git commit -m "Commit E" &&
+	E=$(git rev-parse HEAD) &&
+	test_tick &&
+	git tag -m "v1" v1 HEAD &&
+	TAG1=$(git rev-parse refs/tags/v1) &&
+
+	# branch topic/2
+	git checkout -b topic/2 &&
+	cat >topic-2.txt <<-EOF &&
+		Commit F
+		EOF
+	git add topic-2.txt &&
+	test_tick &&
+	git commit -m "Commit F" &&
+
+	cat >topic-2.txt <<-EOF &&
+		Commit G
+		EOF
+	git add -u &&
+	test_tick &&
+	git commit -m "Commit G" &&
+	git update-ref refs/pull/2/head HEAD &&
+
+	cat >topic-2.txt <<-EOF &&
+		Commit H
+		EOF
+	git add -u &&
+	test_tick &&
+	git commit -m "Commit H" &&
+	F=$(git rev-parse topic/2~2) &&
+	G=$(git rev-parse topic/2~) &&
+	H=$(git rev-parse topic/2) &&
+
+	# merge commit I & J
+	git checkout main &&
+	test_tick &&
+	git merge --no-ff --no-edit topic/1 &&
+	test_tick &&
+	git merge --no-ff --no-edit refs/pull/2/head &&
+	I=$(git rev-parse HEAD~) &&
+	J=$(git rev-parse HEAD) &&
+
+	# commit K
+	git checkout main &&
+	cat >main.txt <<-EOF &&
+		Commit K
+		EOF
+	git add main.txt &&
+	test_tick &&
+	git commit -m "Commit K" &&
+	K=$(git rev-parse HEAD) &&
+
+	# branch release
+	git checkout -b release &&
+	cat >release.txt <<-EOF &&
+		Commit L
+		EOF
+	git add release.txt &&
+	test_tick &&
+	git commit -m "Commit L" &&
+
+	cat >release.txt <<-EOF &&
+		Commit M
+		EOF
+	git add -u &&
+	test_tick &&
+	git commit -m "Commit M" &&
+	test_tick &&
+	git tag -m "v2" v2 HEAD &&
+
+	cat >release.txt <<-EOF &&
+		Commit N
+		EOF
+	git add -u &&
+	test_tick &&
+	git commit -m "Commit N" &&
+	test_tick &&
+	git tag -m "v3" v3 HEAD &&
+	L=$(git rev-parse HEAD~2) &&
+	M=$(git rev-parse HEAD~) &&
+	N=$(git rev-parse HEAD) &&
+	TAG2=$(git rev-parse refs/tags/v2) &&
+	TAG3=$(git rev-parse refs/tags/v3) &&
+
+	# merge commit O
+	git checkout main &&
+	test_tick &&
+	git merge --no-ff --no-edit tags/v2 &&
+	O=$(git rev-parse HEAD) &&
+
+	# commit P
+	git checkout main &&
+	cat >main.txt <<-EOF &&
+		Commit P
+		EOF
+	git add main.txt &&
+	test_tick &&
+	git commit -m "Commit P" &&
+	P=$(git rev-parse HEAD)
+'
+
+test_expect_success 'create bundle from special rev: main^!' '
+	git bundle create special-rev.bdl "main^!" &&
+
+	git bundle list-heads special-rev.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	cat >expect <<-EOF &&
+		<COMMIT-P> refs/heads/main
+		EOF
+	test_i18ncmp expect actual &&
+
+	git bundle verify special-rev.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	cat >expect <<-EOF &&
+		The bundle contains this ref:
+		<COMMIT-P> refs/heads/main
+		The bundle requires this ref:
+		<COMMIT-O>
+		EOF
+	test_i18ncmp expect actual &&
+
+	test_bundle_object_count special-rev.bdl 3
+'
+
+test_expect_success 'create bundle with --max-count option' '
+	git bundle create max-count.bdl --max-count 1 \
+		main \
+		"^release" \
+		refs/tags/v1 \
+		refs/pull/1/head \
+		refs/pull/2/head &&
+
+	git bundle list-heads max-count.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	cat >expect <<-EOF &&
+		<COMMIT-P> refs/heads/main
+		<TAG-1> refs/tags/v1
+		EOF
+	test_i18ncmp expect actual &&
+
+	git bundle verify max-count.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	cat >expect <<-EOF &&
+		The bundle contains these 2 refs:
+		<COMMIT-P> refs/heads/main
+		<TAG-1> refs/tags/v1
+		The bundle requires this ref:
+		<COMMIT-O>
+		EOF
+	test_i18ncmp expect actual &&
+
+	test_bundle_object_count max-count.bdl 4
+'
+
+test_expect_success 'create bundle with --since option' '
+	git bundle create since.bdl \
+		--since "Thu Apr 7 15:26:13 2005 -0700" \
+		--all &&
+
+	git bundle list-heads since.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	cat >expect <<-EOF &&
+		<COMMIT-P> refs/heads/main
+		<COMMIT-N> refs/heads/release
+		<TAG-2> refs/tags/v2
+		<TAG-3> refs/tags/v3
+		<COMMIT-P> HEAD
+		EOF
+	test_i18ncmp expect actual &&
+
+	git bundle verify since.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	cat >expect <<-EOF &&
+		The bundle contains these 5 refs:
+		<COMMIT-P> refs/heads/main
+		<COMMIT-N> refs/heads/release
+		<TAG-2> refs/tags/v2
+		<TAG-3> refs/tags/v3
+		<COMMIT-P> HEAD
+		The bundle requires these 2 refs:
+		<COMMIT-L>
+		<COMMIT-K>
+		EOF
+	test_i18ncmp expect actual &&
+
+	test_thin_bundle_object_count since.bdl 16
+'
+
+test_expect_success 'create bundle 1 - no prerequisites' '
+	git bundle create 1.bdl topic/1 topic/2 &&
+
+	cat >expect <<-EOF &&
+		The bundle contains these 2 refs:
+		<COMMIT-D> refs/heads/topic/1
+		<COMMIT-H> refs/heads/topic/2
+		The bundle records a complete history.
+		EOF
+
+	# verify bundle, which has no prerequisites
+	git bundle verify 1.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	test_i18ncmp expect actual &&
+
+	test_bundle_object_count 1.bdl 24
+'
+
+test_expect_success 'create bundle 2 - has prerequisites' '
+	git bundle create 2.bdl \
+		--ignore-missing \
+		^topic/deleted \
+		^$D \
+		^topic/2 \
+		release &&
+
+	cat >expect <<-EOF &&
+		The bundle contains this ref:
+		<COMMIT-N> refs/heads/release
+		The bundle requires these 3 refs:
+		<COMMIT-D>
+		<COMMIT-E>
+		<COMMIT-G>
+		EOF
+
+	git bundle verify 2.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	test_i18ncmp expect actual &&
+
+	test_bundle_object_count 2.bdl 16
+'
+
+test_expect_success 'fail to verify bundle without prerequisites' '
+	git init --bare test1.git &&
+
+	cat >expect <<-EOF &&
+		error: Repository lacks these prerequisite commits:
+		error: <COMMIT-D>
+		error: <COMMIT-E>
+		error: <COMMIT-G>
+		EOF
+
+	test_must_fail git -C test1.git bundle verify ../2.bdl 2>&1 |
+		make_user_friendly_and_stable_output >actual &&
+	test_i18ncmp expect actual
+'
+
+test_expect_success 'create bundle 3 - two refs, same object' '
+	git bundle create --version=3 3.bdl \
+		^release \
+		^topic/1 \
+		^topic/2 \
+		main \
+		HEAD &&
+
+	cat >expect <<-EOF &&
+		The bundle contains these 2 refs:
+		<COMMIT-P> refs/heads/main
+		<COMMIT-P> HEAD
+		The bundle requires these 2 refs:
+		<COMMIT-M>
+		<COMMIT-K>
+		EOF
+
+	git bundle verify 3.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	test_i18ncmp expect actual &&
+
+	test_bundle_object_count 3.bdl 4
+'
+
+test_expect_success 'create bundle 4 - with tags' '
+	git bundle create 4.bdl \
+		^main \
+		^release \
+		^topic/1 \
+		^topic/2 \
+		--all &&
+
+	cat >expect <<-EOF &&
+		The bundle contains these 3 refs:
+		<TAG-1> refs/tags/v1
+		<TAG-2> refs/tags/v2
+		<TAG-3> refs/tags/v3
+		The bundle records a complete history.
+		EOF
+
+	git bundle verify 4.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	test_i18ncmp expect actual &&
+
+	test_bundle_object_count 4.bdl 3
+'
+
+test_expect_success 'clone from bundle' '
+	git clone --mirror 1.bdl mirror.git &&
+	git -C mirror.git show-ref |
+		make_user_friendly_and_stable_output >actual &&
+	cat >expect <<-EOF &&
+		<COMMIT-D> refs/heads/topic/1
+		<COMMIT-H> refs/heads/topic/2
+		EOF
+	test_cmp expect actual &&
+
+	git -C mirror.git fetch ../2.bdl "+refs/*:refs/*" &&
+	git -C mirror.git show-ref |
+		make_user_friendly_and_stable_output >actual &&
+	cat >expect <<-EOF &&
+		<COMMIT-N> refs/heads/release
+		<COMMIT-D> refs/heads/topic/1
+		<COMMIT-H> refs/heads/topic/2
+		EOF
+	test_cmp expect actual &&
+
+	git -C mirror.git fetch ../3.bdl "+refs/*:refs/*" &&
+	git -C mirror.git show-ref |
+		make_user_friendly_and_stable_output >actual &&
+	cat >expect <<-EOF &&
+		<COMMIT-P> refs/heads/main
+		<COMMIT-N> refs/heads/release
+		<COMMIT-D> refs/heads/topic/1
+		<COMMIT-H> refs/heads/topic/2
+		EOF
+	test_cmp expect actual &&
+
+	git -C mirror.git fetch ../4.bdl "+refs/*:refs/*" &&
+	git -C mirror.git show-ref |
+		make_user_friendly_and_stable_output >actual &&
+	cat >expect <<-EOF &&
+		<COMMIT-P> refs/heads/main
+		<COMMIT-N> refs/heads/release
+		<COMMIT-D> refs/heads/topic/1
+		<COMMIT-H> refs/heads/topic/2
+		<TAG-1> refs/tags/v1
+		<TAG-2> refs/tags/v2
+		<TAG-3> refs/tags/v3
+		EOF
+	test_cmp expect actual
+'
+
+test_done
Signature · 1 line
2.30.0.2.g06d2f50715
Signature · 1 line
Danh
author
#7 lore raw

Re: [PATCH v3 1/2] bundle: lost objects when removing duplicate pendings

Quoted 8 lines · Đoàn Trần Công Danh <[email protected]> 于2021年1月7日周四 下午11:37写道:
Quoted 8 lines
(quoted diff) +2 -1
-static int contains_name(struct object_array *array, const char *name)
+static int contains_object(struct object_array *array,
+                        const struct object *item, const char *name)
 {
      unsigned nr = array->nr, i;
      struct object_array_entry *object = array->objects;
Quoted 9 lines

Review

Quoted 3 lines
      for (i = 0; i < nr; i++, object++)
(quoted diff) +1 -1
-             if (!strcmp(object->name, name))
+             if (item == object->item && !strcmp(object->name, name))

I think the comparison of item == object->item is a bit too fragile. Yes, we all know item must be an entry of array.
However, it's separated from its usage may lead to misuse in the future. Perhaps using oidcmp or adding a comment to indicate that item must be an entry of array.

You can find the same usage on comparing address of objects in other places:

+ https://github.com/git/git/blob/v2.30.0/bundle.c#L447
+ https://github.com/git/git/blob/v2.30.0/commit.c#L954

Both item and object->item point to address of a shared object in parsed_object_pool of the repository, we can compare two objects by checking address of them safely.

Quoted 9 lines

Review

Quoted 7 lines
(quoted diff) +7 -0
+test_bundle_object_count () {
+     bundle=$1 &&
+     pack=${bundle%.bdl}.pack &&
+     convert_bundle_to_pack <"$bundle" >"$pack" &&
+     git index-pack "$pack" &&
+     git verify-pack -v "$pack" >verify.out &&
+     count=$(grep "^$OID_REGEX " verify.out | wc -l) &&

I think we can use 'grep -c' instead of grep .. | wc -l.

This function is borrowed from t5510-fetch.sh, and will change like this. Thanks.

Quoted 11 lines

Review

Quoted 7 lines
(quoted diff) +7 -0
+
+convert_bundle_to_pack () {
+     while read x && test -n "$x"
+     do
+             :;
+     done
+     cat

I'm not sure what you would expect in this case, but in my experience, I replace this whole block with

Quoted 1 line
        sed '1,/^$/d'

This function is used to convert bundle file to pack file by strip the header, which has a signature, prerequisites, references. This function is also borrowed from "t5510-fetch.sh".

Quoted 10 lines

For the below change, I haven't checked but I think test_commit should work, no?

Quoted 9 lines
(quoted diff) +9 -0
+test_expect_success 'setup' '
+     # commit A & B
+     cat >main.txt <<-EOF &&
+             Commit A
+             EOF
+     git add main.txt &&
+     test_tick &&
+     git commit -m "Commit A" &&
+
Quoted 8 lines

Review

Quoted 8 lines
(quoted diff) +8 -0
+     cat >main.txt <<-EOF &&
+             Commit B
+             EOF
+     git add main.txt &&
+     test_tick &&
+     git commit -m "Commit B" &&
+     A=$(git rev-parse HEAD~) &&
+     B=$(git rev-parse HEAD) &&

I should refactor these code, but I forgot. After examine the test_commit function, I have to write a new helper, because it does not meet my needs.
1. It should not create a tag every time.
2. The tag it created is not an annotated tag.
3. I need to store the object id to a variable.

So I write a new helper test_commit_setvar() in next reroll. And make this testcase much simpler.

test_expect_success 'setup' '

        # branch main: commit A & B
        test_commit_setvar A "Commit A" main.txt &&
        test_commit_setvar B "Commit B" main.txt &&
        # branch topic/1: commit C & D, refs/pull/1/head
        git checkout -b topic/1 &&
        test_commit_setvar C "Commit C" topic-1.txt &&
        test_commit_setvar D "Commit D" topic-1.txt &&
        git update-ref refs/pull/1/head HEAD &&
        # branch topic/1: commit E, tag v1
        git checkout main &&
        test_commit_setvar E "Commit E" main.txt &&
        test_commit_setvar TAG1 --tag v1 &&
        # branch topic/2: commit F & G, refs/pull/2/head
        git checkout -b topic/2 &&
        test_commit_setvar F "Commit F" topic-2.txt &&
        test_commit_setvar G "Commit G" topic-2.txt &&
        git update-ref refs/pull/2/head HEAD &&
        test_commit_setvar H "Commit H" topic-2.txt &&
        # branch main: merge commit I & J
        git checkout main &&
        test_tick &&
        test_commit_setvar I --merge topic/1 "Merge commit I" &&
        test_commit_setvar J --merge refs/pull/2/head "Merge commit J" &&
        # branch main: commit K
        git checkout main &&
        test_commit_setvar K "Commit K" main.txt &&
        # branch release:
        git checkout -b release &&
        test_commit_setvar L "Commit L" release.txt &&
        test_commit_setvar M "Commit M" release.txt &&
        test_commit_setvar TAG2 --tag v2 &&
        test_commit_setvar N "Commit N" release.txt &&
        test_commit_setvar TAG3 --tag v3 &&
        # branch main: merge commit O, commit P
        git checkout main &&
        test_commit_setvar O --merge tags/v2 "Merge commit O" &&
        test_commit_setvar P "Commit P" main.txt
'

--
Jiang Xin

author
#8 lore raw

v4 cover Improvements for git-bundle

In reply to Đoàn Trần Công Danh

From: Jiang Xin <[email protected]>

## Two improvements for git-bundle

1. Commit "bundle: lost objects when removing duplicate pendings", which fixes command like:

        $ git bundle create <file> 'master^!'

2. Commits "bundle: arguments can be read from stdin", which add "--stdin" option support for git-bundle, like:

        $ git bundle create <file> <input

## Changes of v4

+ Refactor t6020.

## Range diff of v3...v4

1: 9df48434f3 ! 1: 0abad6486a bundle: lost objects when removing duplicate pendings

    @@ t/t6020-bundle-misc.sh (new)
     +
     +. ./test-lib.sh
     +
    ++# Check count of objects in a bundle file.
    ++# We can use "--thin" opiton to check thin pack, which must be fixed by
    ++# command `git-index-pack --fix-thin --stdin`.
     +test_bundle_object_count () {
    ++	thin= &&
    ++	if test "$1" = "--thin"
    ++	then
    ++		thin=yes
    ++		shift
    ++	fi &&
    ++	if test $# -ne 2
    ++	then
    ++		echo >&2 "args should be: <bundle> <count>"
    ++		return 1
    ++	fi
     +	bundle=$1 &&
     +	pack=${bundle%.bdl}.pack &&
     +	convert_bundle_to_pack <"$bundle" >"$pack" &&
    -+	git index-pack "$pack" &&
    -+	git verify-pack -v "$pack" >verify.out &&
    -+	count=$(grep "^$OID_REGEX " verify.out | wc -l) &&
    ++	if test -n "$thin"
    ++	then
    ++		test_must_fail git index-pack "$pack" &&
    ++		mv "$pack" "$pack"-thin &&
    ++		cat "$pack"-thin |
    ++			git index-pack --stdin --fix-thin "$pack"
    ++	else
    ++		git index-pack "$pack"
    ++	fi &&
    ++	git verify-pack -v "$pack" >verify.out
    ++	if test $? -ne 0
    ++	then
    ++		echo >&2 "error: fail to convert $bundle to $pack"
    ++		return 1
    ++	fi
    ++	count=$(grep -c "^$OID_REGEX " verify.out) &&
     +	test $2 = $count && return 0
    -+	echo object count for $bundle is $count, not $2
    -+	return 1
    -+}
    -+
    -+
    -+test_thin_bundle_object_count () {
    -+	bundle=$1 &&
    -+	pack=${bundle%.bdl}.pack &&
    -+	convert_bundle_to_pack <"$bundle" |
    -+		test_must_fail git index-pack --stdin "$pack" &&
    -+	rm -f "$pack" &&
    -+	convert_bundle_to_pack <"$bundle" |
    -+		git index-pack --stdin --fix-thin "$pack" &&
    -+	git verify-pack -v "$pack" >verify.out &&
    -+	count=$(grep "^$OID_REGEX " verify.out | wc -l) &&
    -+	test $2 = $count && return 0
    -+	echo object count for $bundle is $count, not $2
    ++	echo >&2 "error: object count for $bundle is $count, not $2"
     +	return 1
     +}
     +
    ++# Display the pack data contained in the bundle file, bypassing the
    ++# header that contains the signature, prerequisites and references.
     +convert_bundle_to_pack () {
     +	while read x && test -n "$x"
     +	do
    @@ t/t6020-bundle-misc.sh (new)
     +	cat
     +}
     +
    ++# Create a commit or tag and set the variable with the object ID.
    ++test_commit_setvar () {
    ++	notick= &&
    ++	signoff= &&
    ++	indir= &&
    ++	merge= &&
    ++	tag= &&
    ++	var= &&
    ++	while test $# != 0
    ++	do
    ++		case "$1" in
    ++		--merge)
    ++			merge=yes
    ++			;;
    ++		--tag)
    ++			tag=yes
    ++			;;
    ++		--notick)
    ++			notick=yes
    ++			;;
    ++		--signoff)
    ++			signoff="$1"
    ++			;;
    ++		-C)
    ++			indir="$2"
    ++			shift
    ++			;;
    ++		-*)
    ++			echo >&2 "error: unknown option $1"
    ++			return 1
    ++			;;
    ++		*)
    ++			test -n "$var" && break
    ++			var=$1
    ++			;;
    ++		esac
    ++		shift
    ++	done &&
    ++	indir=${indir:+"$indir"/} &&
    ++	if test $# -eq 0
    ++	then
    ++		echo >&2 "no args provided"
    ++		return 1
    ++	fi &&
    ++	if test -z "$notick"
    ++	then
    ++		test_tick
    ++	fi &&
    ++	if test -n "$merge"
    ++	then
    ++		git ${indir:+ -C "$indir"} merge --no-edit --no-ff \
    ++			${2:+-m "$2"} "$1" &&
    ++		oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD)
    ++	elif test -n "$tag"
    ++	then
    ++		git ${indir:+ -C "$indir"} tag -m "$1" "$1" &&
    ++		oid=$(git ${indir:+ -C "$indir"} rev-parse "$1")
    ++	else
    ++		file=${2:-"$1.t"} &&
    ++		echo "${3-$1}" > "$indir$file" &&
    ++		git ${indir:+ -C "$indir"} add "$file" &&
    ++		git ${indir:+ -C "$indir"} commit $signoff -m "$1" &&
    ++		oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD)
    ++	fi &&
    ++	eval $var=$oid
    ++}
    ++
    ++
     +# Format the output of git commands to make a user-friendly and stable
     +# text.  We can easily prepare the expect text without having to worry
     +# about future changes of the commit ID and spaces of the output.
     +make_user_friendly_and_stable_output () {
     +	sed \
    -+		-e "s/ *\$//" \
    -+		-e "s/$A/<COMMIT-A>/g" \
    -+		-e "s/$B/<COMMIT-B>/g" \
    -+		-e "s/$C/<COMMIT-C>/g" \
    -+		-e "s/$D/<COMMIT-D>/g" \
    -+		-e "s/$E/<COMMIT-E>/g" \
    -+		-e "s/$F/<COMMIT-F>/g" \
    -+		-e "s/$G/<COMMIT-G>/g" \
    -+		-e "s/$H/<COMMIT-H>/g" \
    -+		-e "s/$I/<COMMIT-I>/g" \
    -+		-e "s/$J/<COMMIT-J>/g" \
    -+		-e "s/$K/<COMMIT-K>/g" \
    -+		-e "s/$L/<COMMIT-L>/g" \
    -+		-e "s/$M/<COMMIT-M>/g" \
    -+		-e "s/$N/<COMMIT-N>/g" \
    -+		-e "s/$O/<COMMIT-O>/g" \
    -+		-e "s/$P/<COMMIT-P>/g" \
    ++		-e "s/$A/<COMMIT-A>/" \
    ++		-e "s/$B/<COMMIT-B>/" \
    ++		-e "s/$C/<COMMIT-C>/" \
    ++		-e "s/$D/<COMMIT-D>/" \
    ++		-e "s/$E/<COMMIT-E>/" \
    ++		-e "s/$F/<COMMIT-F>/" \
    ++		-e "s/$G/<COMMIT-G>/" \
    ++		-e "s/$H/<COMMIT-H>/" \
    ++		-e "s/$I/<COMMIT-I>/" \
    ++		-e "s/$J/<COMMIT-J>/" \
    ++		-e "s/$K/<COMMIT-K>/" \
    ++		-e "s/$L/<COMMIT-L>/" \
    ++		-e "s/$M/<COMMIT-M>/" \
    ++		-e "s/$N/<COMMIT-N>/" \
    ++		-e "s/$O/<COMMIT-O>/" \
    ++		-e "s/$P/<COMMIT-P>/" \
    ++		-e "s/$TAG1/<TAG-1>/" \
    ++		-e "s/$TAG2/<TAG-2>/" \
    ++		-e "s/$TAG3/<TAG-3>/" \
     +		-e "s/$(echo $A | cut -c1-7)[0-9a-f]*/<OID-A>/g" \
     +		-e "s/$(echo $B | cut -c1-7)[0-9a-f]*/<OID-B>/g" \
     +		-e "s/$(echo $C | cut -c1-7)[0-9a-f]*/<OID-C>/g" \
    @@ t/t6020-bundle-misc.sh (new)
     +		-e "s/$(echo $N | cut -c1-7)[0-9a-f]*/<OID-N>/g" \
     +		-e "s/$(echo $O | cut -c1-7)[0-9a-f]*/<OID-O>/g" \
     +		-e "s/$(echo $P | cut -c1-7)[0-9a-f]*/<OID-P>/g" \
    -+		-e "s/$TAG1/<TAG-1>/g" \
    -+		-e "s/$TAG2/<TAG-2>/g" \
    -+		-e "s/$TAG3/<TAG-3>/g" \
     +		-e "s/$(echo $TAG1 | cut -c1-7)[0-9a-f]*/<OID-TAG-1>/g" \
     +		-e "s/$(echo $TAG2 | cut -c1-7)[0-9a-f]*/<OID-TAG-2>/g" \
     +		-e "s/$(echo $TAG3 | cut -c1-7)[0-9a-f]*/<OID-TAG-3>/g" \
    -+		-e "s/$ZERO_OID/<ZERO-OID>/g"
    ++		-e "s/ *\$//"
     +}
     +
     +#            (C)   (D, pull/1/head, topic/1)
    @@ t/t6020-bundle-misc.sh (new)
     +# (A)   (B)  (E, tag:v1) (I)  (J)                  (K)       (O)   (P)
     +#
     +test_expect_success 'setup' '
    -+	# commit A & B
    -+	cat >main.txt <<-EOF &&
    -+		Commit A
    -+		EOF
    -+	git add main.txt &&
    -+	test_tick &&
    -+	git commit -m "Commit A" &&
    ++	# Try to make a stable fixed width for abbreviated commit ID,
    ++	# this fixed-width oid will be replaced with "<OID>".
    ++	git config core.abbrev 7 &&
     +
    -+	cat >main.txt <<-EOF &&
    -+		Commit B
    -+		EOF
    -+	git add main.txt &&
    -+	test_tick &&
    -+	git commit -m "Commit B" &&
    -+	A=$(git rev-parse HEAD~) &&
    -+	B=$(git rev-parse HEAD) &&
    ++	# branch main: commit A & B
    ++	test_commit_setvar A "Commit A" main.txt &&
    ++	test_commit_setvar B "Commit B" main.txt &&
     +
    -+	# branch topic/1
    ++	# branch topic/1: commit C & D, refs/pull/1/head
     +	git checkout -b topic/1 &&
    -+	cat >topic-1.txt <<-EOF &&
    -+		Commit C
    -+		EOF
    -+	git add topic-1.txt &&
    -+	test_tick &&
    -+	git commit -m "Commit C" &&
    -+
    -+	cat >topic-1.txt <<-EOF &&
    -+		Commit D
    -+		EOF
    -+	git add -u &&
    -+	test_tick &&
    -+	git commit -m "Commit D" &&
    ++	test_commit_setvar C "Commit C" topic-1.txt &&
    ++	test_commit_setvar D "Commit D" topic-1.txt &&
     +	git update-ref refs/pull/1/head HEAD &&
    -+	C=$(git rev-parse topic/1~) &&
    -+	D=$(git rev-parse topic/1) &&
     +
    -+	# commit E
    ++	# branch topic/1: commit E, tag v1
     +	git checkout main &&
    -+	cat >main.txt <<-EOF &&
    -+		Commit E
    -+		EOF
    -+	git add main.txt &&
    -+	test_tick &&
    -+	git commit -m "Commit E" &&
    -+	E=$(git rev-parse HEAD) &&
    -+	test_tick &&
    -+	git tag -m "v1" v1 HEAD &&
    -+	TAG1=$(git rev-parse refs/tags/v1) &&
    -+
    -+	# branch topic/2
    -+	git checkout -b topic/2 &&
    -+	cat >topic-2.txt <<-EOF &&
    -+		Commit F
    -+		EOF
    -+	git add topic-2.txt &&
    -+	test_tick &&
    -+	git commit -m "Commit F" &&
    ++	test_commit_setvar E "Commit E" main.txt &&
    ++	test_commit_setvar TAG1 --tag v1 &&
     +
    -+	cat >topic-2.txt <<-EOF &&
    -+		Commit G
    -+		EOF
    -+	git add -u &&
    -+	test_tick &&
    -+	git commit -m "Commit G" &&
    ++	# branch topic/2: commit F & G, refs/pull/2/head
    ++	git checkout -b topic/2 &&
    ++	test_commit_setvar F "Commit F" topic-2.txt &&
    ++	test_commit_setvar G "Commit G" topic-2.txt &&
     +	git update-ref refs/pull/2/head HEAD &&
    ++	test_commit_setvar H "Commit H" topic-2.txt &&
     +
    -+	cat >topic-2.txt <<-EOF &&
    -+		Commit H
    -+		EOF
    -+	git add -u &&
    -+	test_tick &&
    -+	git commit -m "Commit H" &&
    -+	F=$(git rev-parse topic/2~2) &&
    -+	G=$(git rev-parse topic/2~) &&
    -+	H=$(git rev-parse topic/2) &&
    -+
    -+	# merge commit I & J
    ++	# branch main: merge commit I & J
     +	git checkout main &&
    -+	test_tick &&
    -+	git merge --no-ff --no-edit topic/1 &&
    -+	test_tick &&
    -+	git merge --no-ff --no-edit refs/pull/2/head &&
    -+	I=$(git rev-parse HEAD~) &&
    -+	J=$(git rev-parse HEAD) &&
    -+
    -+	# commit K
    ++	test_commit_setvar I --merge topic/1 "Merge commit I" &&
    ++	test_commit_setvar J --merge refs/pull/2/head "Merge commit J" &&
    ++
    ++	# branch main: commit K
     +	git checkout main &&
    -+	cat >main.txt <<-EOF &&
    -+		Commit K
    -+		EOF
    -+	git add main.txt &&
    -+	test_tick &&
    -+	git commit -m "Commit K" &&
    -+	K=$(git rev-parse HEAD) &&
    ++	test_commit_setvar K "Commit K" main.txt &&
     +
    -+	# branch release
    ++	# branch release:
     +	git checkout -b release &&
    -+	cat >release.txt <<-EOF &&
    -+		Commit L
    -+		EOF
    -+	git add release.txt &&
    -+	test_tick &&
    -+	git commit -m "Commit L" &&
    ++	test_commit_setvar L "Commit L" release.txt &&
    ++	test_commit_setvar M "Commit M" release.txt &&
    ++	test_commit_setvar TAG2 --tag v2 &&
    ++	test_commit_setvar N "Commit N" release.txt &&
    ++	test_commit_setvar TAG3 --tag v3 &&
     +
    -+	cat >release.txt <<-EOF &&
    -+		Commit M
    -+		EOF
    -+	git add -u &&
    -+	test_tick &&
    -+	git commit -m "Commit M" &&
    -+	test_tick &&
    -+	git tag -m "v2" v2 HEAD &&
    -+
    -+	cat >release.txt <<-EOF &&
    -+		Commit N
    -+		EOF
    -+	git add -u &&
    -+	test_tick &&
    -+	git commit -m "Commit N" &&
    -+	test_tick &&
    -+	git tag -m "v3" v3 HEAD &&
    -+	L=$(git rev-parse HEAD~2) &&
    -+	M=$(git rev-parse HEAD~) &&
    -+	N=$(git rev-parse HEAD) &&
    -+	TAG2=$(git rev-parse refs/tags/v2) &&
    -+	TAG3=$(git rev-parse refs/tags/v3) &&
    -+
    -+	# merge commit O
    -+	git checkout main &&
    -+	test_tick &&
    -+	git merge --no-ff --no-edit tags/v2 &&
    -+	O=$(git rev-parse HEAD) &&
    -+
    -+	# commit P
    ++	# branch main: merge commit O, commit P
     +	git checkout main &&
    -+	cat >main.txt <<-EOF &&
    -+		Commit P
    -+		EOF
    -+	git add main.txt &&
    -+	test_tick &&
    -+	git commit -m "Commit P" &&
    -+	P=$(git rev-parse HEAD)
    ++	test_commit_setvar O --merge tags/v2 "Merge commit O" &&
    ++	test_commit_setvar P "Commit P" main.txt
     +'
     +
     +test_expect_success 'create bundle from special rev: main^!' '
    @@ t/t6020-bundle-misc.sh (new)
     +'
     +
     +test_expect_success 'create bundle with --since option' '
    ++	since="Thu Apr 7 15:26:13 2005 -0700" &&
    ++	git log -1 --pretty="%ad" $M >actual &&
    ++	echo "$since" >expect &&
    ++	test_cmp expect actual &&
    ++
     +	git bundle create since.bdl \
    -+		--since "Thu Apr 7 15:26:13 2005 -0700" \
    -+		--all &&
    ++		--since "$since" --all &&
     +
     +	git bundle list-heads since.bdl |
     +		make_user_friendly_and_stable_output >actual &&
    @@ t/t6020-bundle-misc.sh (new)
     +		EOF
     +	test_i18ncmp expect actual &&
     +
    -+	test_thin_bundle_object_count since.bdl 16
    ++	test_bundle_object_count --thin since.bdl 16
     +'
     +
     +test_expect_success 'create bundle 1 - no prerequisites' '

2: 86ad41e4d4 = 2: 48ef4aa44e bundle: arguments can be read from stdin

--
Jiang Xin (2):
bundle: lost objects when removing duplicate pendings
bundle: arguments can be read from stdin

 bundle.c                | 111 ++++----
 object.c                |  10 +-
 t/t5607-clone-bundle.sh |   4 +-
 t/t6020-bundle-misc.sh  | 546 ++++++++++++++++++++++++++++++++++++++++
 4 files changed, 614 insertions(+), 57 deletions(-)
 create mode 100755 t/t6020-bundle-misc.sh
Signature · 1 line
2.30.0.2.g06d2f50715
author
#9 lore raw

PATCH v4 1/2 bundle: lost objects when removing duplicate pendings

In reply to Đoàn Trần Công Danh

From: Jiang Xin <[email protected]>

git rev-list will list one commit for the following command:

    $ git rev-list 'main^!'
    <tip-commit-of-main-branch>

But providing the same rev-list args to git bundle, fail to create a bundle file.

    $ git bundle create - 'main^!'
    # v2 git bundle
    -<OID> <one-line-message>

fatal: Refusing to create empty bundle.

This is because when removing duplicate objects in function object_array_remove_duplicates(), one unique pending object which has the same name is deleted by mistake. The revision arg 'main^!' in the above example is parsed by handle_revision_arg(), and at lease two different objects will be appended to revs.pending, one points to the parent commit of the "main" branch, and the other points to the tip commit of the "main" branch. These two objects have the same name "main". Only one object is left with the name "main" after calling the function object_array_remove_duplicates().

And what's worse, when adding boundary commits into pending list, we use one-line commit message as names, and the arbitory names may surprise git-bundle.

Only comparing objects themselves (".item") is also not good enough, because user may want to create a bundle with two identical objects but with different reference names, such as: "HEAD" and "refs/heads/main".

Add new function contains_object() which compare both the address and the name of the object.

---
 object.c               |  10 +-
 t/t6020-bundle-misc.sh | 477 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 483 insertions(+), 4 deletions(-)
 create mode 100755 t/t6020-bundle-misc.sh
object.c +6 -4
index 68f80b0b3d..98017bed8e 100644
@@ -412,15 +412,16 @@ void object_array_clear(struct object_array *array)
 }
 
 /*
- * Return true iff array already contains an entry with name.
+ * Return true if array already contains an entry.
  */
-static int contains_name(struct object_array *array, const char *name)
+static int contains_object(struct object_array *array,
+			   const struct object *item, const char *name)
 {
 	unsigned nr = array->nr, i;
 	struct object_array_entry *object = array->objects;
 
 	for (i = 0; i < nr; i++, object++)
-		if (!strcmp(object->name, name))
+		if (item == object->item && !strcmp(object->name, name))
 			return 1;
 	return 0;
 }
@@ -432,7 +433,8 @@ void object_array_remove_duplicates(struct object_array *array)
 
 	array->nr = 0;
 	for (src = 0; src < nr; src++) {
-		if (!contains_name(array, objects[src].name)) {
+		if (!contains_object(array, objects[src].item,
+				     objects[src].name)) {
 			if (src != array->nr)
 				objects[array->nr] = objects[src];
 			array->nr++;
t/t6020-bundle-misc.sh new file +477 -0

477 lines not on this page · raw message

Signature · 1 line
2.30.0.2.g06d2f50715
#10 lore raw

Re: [PATCH v4 1/2] bundle: lost objects when removing duplicate pendings

Quoted 29 lines · Jiang Xin
t/t6020-bundle-misc.sh new file +23 -0
new file mode 100755
index 0000000000..c4447ca88f
@@ -0,0 +1,477 @@
+#!/bin/sh
+#
+# Copyright (c) 2021 Jiang Xin
+#
+
+test_description='Test git-bundle'
+
+GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
+export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
+
+. ./test-lib.sh
+
+# Check count of objects in a bundle file.
+# We can use "--thin" opiton to check thin pack, which must be fixed by
+# command `git-index-pack --fix-thin --stdin`.
+test_bundle_object_count () {
+	thin= &&
+	if test "$1" = "--thin"
+	then
+		thin=yes
+		shift
+	fi &&
+	if test $# -ne 2

t/t6020-bundle-misc.sh line 33

Quoted 10 lines
t/t6020-bundle-misc.sh new file +10 -0
+	then
+		echo >&2 "args should be: <bundle> <count>"
+		return 1
+	fi
+	bundle=$1 &&
+	pack=${bundle%.bdl}.pack &&
+	convert_bundle_to_pack <"$bundle" >"$pack" &&
+	if test -n "$thin"
+	then
+		test_must_fail git index-pack "$pack" &&

This is overly strict, isn't it?

Imagine a case where the objects newer revisions introduce have *no* resemblance to the objects in the prerequisites' trees---the resulting pack will have no object that is expressed as a delta against anything outside the pack, and the above "index-pack" would succeed.

Besides, "git pack-objects --thin" is *not* obligated to create a pack that lacks one or more objects. The "--thin" option merely *allows* pack-objects to omit base objects if it is convenient to do so.

t/t6020-bundle-misc.sh line 36

Quoted 3 lines
t/t6020-bundle-misc.sh new file +3 -0
+		mv "$pack" "$pack"-thin &&
+		cat "$pack"-thin |
+			git index-pack --stdin --fix-thin "$pack"

This side is good, but do not cat a single file into a pipe. The whole "then" clause would become

	then
		mv "$pack" "$pack-thin" &&
		git index-pack --stdin --fix-thin "$pack" <"$pack-thin"
	else

I would think.

t/t6020-bundle-misc.sh line 45

Quoted 9 lines
t/t6020-bundle-misc.sh new file +9 -0
+	else
+		git index-pack "$pack"
+	fi &&
+	git verify-pack -v "$pack" >verify.out
+	if test $? -ne 0
+	then
+		echo >&2 "error: fail to convert $bundle to $pack"
+		return 1
+	fi

At this point, we are not testing the bundle subcommand, but is testing "git index-pack --fix-thin" that we run ourselves. Is it essential to ensure $pack is sane here? I doubt it.

t/t6020-bundle-misc.sh line 46

Quoted 1 line
t/t6020-bundle-misc.sh new file +1 -0
+	count=$(grep -c "^$OID_REGEX " verify.out) &&

And if there is no need to run verify-pack, then we can do count=$(git show-index "${pack%pack}idx" | wc -l) instead, perhaps?

Quoted 4 lines
t/t6020-bundle-misc.sh new file +4 -0
+	test $2 = $count && return 0
+	echo >&2 "error: object count for $bundle is $count, not $2"
+	return 1
+}

t/t6020-bundle-misc.sh line 60

Quoted 10 lines
t/t6020-bundle-misc.sh new file +10 -0
+
+# Display the pack data contained in the bundle file, bypassing the
+# header that contains the signature, prerequisites and references.
+convert_bundle_to_pack () {
+	while read x && test -n "$x"
+	do
+		:;
+	done
+	cat
+}

This looks somewhat familiar. Perhaps extract out necessary helpers including this one into t/test-bundle-lib or something similar in a preparatory step before this patch?

Quoted 28 lines
t/t6020-bundle-misc.sh new file +28 -0
+# Create a commit or tag and set the variable with the object ID.
+test_commit_setvar () {
+	notick= &&
+	signoff= &&
+	indir= &&
+	merge= &&
+	tag= &&
+	var= &&
+	while test $# != 0
+	do
+		case "$1" in
+		--merge)
+			merge=yes
+			;;
+		--tag)
+			tag=yes
+			;;
+		--notick)
+			notick=yes
+			;;
+		--signoff)
+			signoff="$1"
+			;;
+		-C)
+			indir="$2"
+			shift
+			;;
+		-*)

t/t6020-bundle-misc.sh line 99

Quoted 10 lines
t/t6020-bundle-misc.sh new file +10 -0
+			echo >&2 "error: unknown option $1"
+			return 1
+			;;
+		*)
+			test -n "$var" && break
+			var=$1
+			;;
+		esac
+		shift
+	done &&

At this point, if $var is still empty, the caller is buggy, and ...

Quoted 18 lines
t/t6020-bundle-misc.sh new file +18 -0
+	indir=${indir:+"$indir"/} &&
+	if test $# -eq 0
+	then
+		echo >&2 "no args provided"
+		return 1
+	fi &&
+	if test -z "$notick"
+	then
+		test_tick
+	fi &&
+	if test -n "$merge"
+	then
+		git ${indir:+ -C "$indir"} merge --no-edit --no-ff \
+			${2:+-m "$2"} "$1" &&
+		oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD)
+	elif test -n "$tag"
+	then
+		git ${indir:+ -C "$indir"} tag -m "$1" "$1" &&

t/t6020-bundle-misc.sh line 127

Quoted 10 lines
t/t6020-bundle-misc.sh new file +10 -0
+		oid=$(git ${indir:+ -C "$indir"} rev-parse "$1")
+	else
+		file=${2:-"$1.t"} &&
+		echo "${3-$1}" > "$indir$file" &&
+		git ${indir:+ -C "$indir"} add "$file" &&
+		git ${indir:+ -C "$indir"} commit $signoff -m "$1" &&
+		oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD)
+	fi &&
+	eval $var=$oid
+}

... it will cause a failure in 'eval' we have here. Not good.

t/t6020-bundle-misc.sh line 132

Quoted 3 lines
t/t6020-bundle-misc.sh new file +3 -0
+# Format the output of git commands to make a user-friendly and stable
+# text.  We can easily prepare the expect text without having to worry
+# about future changes of the commit ID and spaces of the output.

Hmph. This relies on 7 hexdigits being sufficient to uniquely identify all objects involved in the test? It should be OK in practice.

Is there a point in having both <COMMIT-A> and <OID-A>? I would have expected that all these "full object name" conversions are unneeded.

Quoted 41 lines
t/t6020-bundle-misc.sh new file +41 -0
+make_user_friendly_and_stable_output () {
+	sed \
+		-e "s/$A/<COMMIT-A>/" \
+		-e "s/$B/<COMMIT-B>/" \
+		-e "s/$C/<COMMIT-C>/" \
+		-e "s/$D/<COMMIT-D>/" \
+		-e "s/$E/<COMMIT-E>/" \
+		-e "s/$F/<COMMIT-F>/" \
+		-e "s/$G/<COMMIT-G>/" \
+		-e "s/$H/<COMMIT-H>/" \
+		-e "s/$I/<COMMIT-I>/" \
+		-e "s/$J/<COMMIT-J>/" \
+		-e "s/$K/<COMMIT-K>/" \
+		-e "s/$L/<COMMIT-L>/" \
+		-e "s/$M/<COMMIT-M>/" \
+		-e "s/$N/<COMMIT-N>/" \
+		-e "s/$O/<COMMIT-O>/" \
+		-e "s/$P/<COMMIT-P>/" \
+		-e "s/$TAG1/<TAG-1>/" \
+		-e "s/$TAG2/<TAG-2>/" \
+		-e "s/$TAG3/<TAG-3>/" \
+		-e "s/$(echo $A | cut -c1-7)[0-9a-f]*/<OID-A>/g" \
+		-e "s/$(echo $B | cut -c1-7)[0-9a-f]*/<OID-B>/g" \
+		-e "s/$(echo $C | cut -c1-7)[0-9a-f]*/<OID-C>/g" \
+		-e "s/$(echo $D | cut -c1-7)[0-9a-f]*/<OID-D>/g" \
+		-e "s/$(echo $E | cut -c1-7)[0-9a-f]*/<OID-E>/g" \
+		-e "s/$(echo $F | cut -c1-7)[0-9a-f]*/<OID-F>/g" \
+		-e "s/$(echo $G | cut -c1-7)[0-9a-f]*/<OID-G>/g" \
+		-e "s/$(echo $H | cut -c1-7)[0-9a-f]*/<OID-H>/g" \
+		-e "s/$(echo $I | cut -c1-7)[0-9a-f]*/<OID-I>/g" \
+		-e "s/$(echo $J | cut -c1-7)[0-9a-f]*/<OID-J>/g" \
+		-e "s/$(echo $K | cut -c1-7)[0-9a-f]*/<OID-K>/g" \
+		-e "s/$(echo $L | cut -c1-7)[0-9a-f]*/<OID-L>/g" \
+		-e "s/$(echo $M | cut -c1-7)[0-9a-f]*/<OID-M>/g" \
+		-e "s/$(echo $N | cut -c1-7)[0-9a-f]*/<OID-N>/g" \
+		-e "s/$(echo $O | cut -c1-7)[0-9a-f]*/<OID-O>/g" \
+		-e "s/$(echo $P | cut -c1-7)[0-9a-f]*/<OID-P>/g" \
+		-e "s/$(echo $TAG1 | cut -c1-7)[0-9a-f]*/<OID-TAG-1>/g" \
+		-e "s/$(echo $TAG2 | cut -c1-7)[0-9a-f]*/<OID-TAG-2>/g" \
+		-e "s/$(echo $TAG3 | cut -c1-7)[0-9a-f]*/<OID-TAG-3>/g" \
+		-e "s/ *\$//"

t/t6020-bundle-misc.sh line 243

Quoted 10 lines
t/t6020-bundle-misc.sh new file +9 -0
+}
...
+test_expect_success 'create bundle from special rev: main^!' '
+	git bundle create special-rev.bdl "main^!" &&
+
+	git bundle list-heads special-rev.bdl |
+		make_user_friendly_and_stable_output >actual &&
+	cat >expect <<-EOF &&
+		<COMMIT-P> refs/heads/main
+		EOF

We prefer to indent these more like so:

	cat >expect <<-\EOF &&
	<COMMIT-P> refs/heads/main
	EOF

i.e. the indent of the line with <<EOF on it and the indent of the line with the matching EOF are the same. Also, quote EOF to signal that the body of the here text should be taken as-is without $var substitution.

author
#11 lore raw

Re: [PATCH v4 1/2] bundle: lost objects when removing duplicate pendings

Quoted 36 lines · Junio C Hamano <[email protected]> 于2021年1月9日周六 上午10:11写道:
Quoted 33 lines · Jiang Xin
t/t6020-bundle-misc.sh new file +27 -0
new file mode 100755
index 0000000000..c4447ca88f
@@ -0,0 +1,477 @@
+#!/bin/sh
+#
+# Copyright (c) 2021 Jiang Xin
+#
+
+test_description='Test git-bundle'
+
+GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
+export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
+
+. ./test-lib.sh
+
+# Check count of objects in a bundle file.
+# We can use "--thin" opiton to check thin pack, which must be fixed by
+# command `git-index-pack --fix-thin --stdin`.
+test_bundle_object_count () {
+     thin= &&
+     if test "$1" = "--thin"
+     then
+             thin=yes
+             shift
+     fi &&
+     if test $# -ne 2
+     then
+             echo >&2 "args should be: <bundle> <count>"
+             return 1
+     fi
Quoted 8 lines

t/t6020-bundle-misc.sh line 33

Quoted 6 lines
t/t6020-bundle-misc.sh new file +6 -0
+     bundle=$1 &&
+     pack=${bundle%.bdl}.pack &&
+     convert_bundle_to_pack <"$bundle" >"$pack" &&
+     if test -n "$thin"
+     then
+             test_must_fail git index-pack "$pack" &&

This is overly strict, isn't it?

I want to make sure that the created bundle file which contains a thin pack is not changed, but now I think this check is not necessary, testing the count of objects is enough.

Quoted 18 lines

Imagine a case where the objects newer revisions introduce have *no* resemblance to the objects in the prerequisites' trees---the resulting pack will have no object that is expressed as a delta against anything outside the pack, and the above "index-pack" would succeed.

Besides, "git pack-objects --thin" is *not* obligated to create a pack that lacks one or more objects. The "--thin" option merely *allows* pack-objects to omit base objects if it is convenient to do so.

t/t6020-bundle-misc.sh line 36

Quoted 3 lines
t/t6020-bundle-misc.sh new file +3 -0
+             mv "$pack" "$pack"-thin &&
+             cat "$pack"-thin |
+                     git index-pack --stdin --fix-thin "$pack"

This side is good, but do not cat a single file into a pipe. The whole "then" clause would become

Quoted 6 lines
        then
                mv "$pack" "$pack-thin" &&
                git index-pack --stdin --fix-thin "$pack" <"$pack-thin"
        else

I would think.

That's better.

Quoted 10 lines

t/t6020-bundle-misc.sh line 45

Quoted 9 lines
t/t6020-bundle-misc.sh new file +9 -0
+     else
+             git index-pack "$pack"
+     fi &&
+     git verify-pack -v "$pack" >verify.out
+     if test $? -ne 0
+     then
+             echo >&2 "error: fail to convert $bundle to $pack"
+             return 1
+     fi
Quoted 3 lines

At this point, we are not testing the bundle subcommand, but is testing "git index-pack --fix-thin" that we run ourselves. Is it essential to ensure $pack is sane here? I doubt it.

If not check the return error code of git index-pack, the report message will be "error: object count for $bundle is , not $2". I want to give a specific error message for developer.

Quoted 4 lines

t/t6020-bundle-misc.sh line 46

Quoted 1 line
t/t6020-bundle-misc.sh new file +1 -0
+     count=$(grep -c "^$OID_REGEX " verify.out) &&

And if there is no need to run verify-pack, then we can do count=$(git show-index "${pack%pack}idx" | wc -l) instead, perhaps?

Will do.

Quoted 45 lines
Quoted 4 lines
t/t6020-bundle-misc.sh new file +4 -0
+     test $2 = $count && return 0
+     echo >&2 "error: object count for $bundle is $count, not $2"
+     return 1
+}

t/t6020-bundle-misc.sh line 60

Quoted 10 lines
t/t6020-bundle-misc.sh new file +10 -0
+
+# Display the pack data contained in the bundle file, bypassing the
+# header that contains the signature, prerequisites and references.
+convert_bundle_to_pack () {
+     while read x && test -n "$x"
+     do
+             :;
+     done
+     cat
+}

This looks somewhat familiar. Perhaps extract out necessary helpers including this one into t/test-bundle-lib or something similar in a preparatory step before this patch?

Quoted 26 lines
t/t6020-bundle-misc.sh new file +26 -0
+# Create a commit or tag and set the variable with the object ID.
+test_commit_setvar () {
+     notick= &&
+     signoff= &&
+     indir= &&
+     merge= &&
+     tag= &&
+     var= &&
+     while test $# != 0
+     do
+             case "$1" in
+             --merge)
+                     merge=yes
+                     ;;
+             --tag)
+                     tag=yes
+                     ;;
+             --notick)
+                     notick=yes
+                     ;;
+             --signoff)
+                     signoff="$1"
+                     ;;
+             -C)
+                     indir="$2"
+                     shift
Quoted 8 lines

t/t6020-bundle-misc.sh line 95

Quoted 8 lines
t/t6020-bundle-misc.sh new file +8 -0
+                     ;;
+             -*)
+                     echo >&2 "error: unknown option $1"
+                     return 1
+                     ;;
+             *)
+                     test -n "$var" && break
+                     var=$1

The loop ends only if $var has been assigned a value, or no other args. Will report error if no other args later.

Quoted 6 lines

t/t6020-bundle-misc.sh line 99

Quoted 4 lines
t/t6020-bundle-misc.sh new file +4 -0
+                     ;;
+             esac
+             shift
+     done &&

At this point, if $var is still empty, the caller is buggy, and ...

See the above note.

Quoted 39 lines
Quoted 18 lines
t/t6020-bundle-misc.sh new file +18 -0
+     indir=${indir:+"$indir"/} &&
+     if test $# -eq 0
+     then
+             echo >&2 "no args provided"
+             return 1
+     fi &&
+     if test -z "$notick"
+     then
+             test_tick
+     fi &&
+     if test -n "$merge"
+     then
+             git ${indir:+ -C "$indir"} merge --no-edit --no-ff \
+                     ${2:+-m "$2"} "$1" &&
+             oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD)
+     elif test -n "$tag"
+     then
+             git ${indir:+ -C "$indir"} tag -m "$1" "$1" &&

t/t6020-bundle-misc.sh line 127

Quoted 10 lines
t/t6020-bundle-misc.sh new file +10 -0
+             oid=$(git ${indir:+ -C "$indir"} rev-parse "$1")
+     else
+             file=${2:-"$1.t"} &&
+             echo "${3-$1}" > "$indir$file" &&
+             git ${indir:+ -C "$indir"} add "$file" &&
+             git ${indir:+ -C "$indir"} commit $signoff -m "$1" &&
+             oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD)
+     fi &&
+     eval $var=$oid
+}

... it will cause a failure in 'eval' we have here. Not good.

t/t6020-bundle-misc.sh line 132

Quoted 3 lines
t/t6020-bundle-misc.sh new file +3 -0
+# Format the output of git commands to make a user-friendly and stable
+# text.  We can easily prepare the expect text without having to worry
+# about future changes of the commit ID and spaces of the output.

Hmph. This relies on 7 hexdigits being sufficient to uniquely identify all objects involved in the test? It should be OK in practice.

Quoted 3 lines

Is there a point in having both <COMMIT-A> and <OID-A>? I would have expected that all these "full object name" conversions are unneeded.

Will do.

Quoted 63 lines
Quoted 41 lines
t/t6020-bundle-misc.sh new file +41 -0
+make_user_friendly_and_stable_output () {
+     sed \
+             -e "s/$A/<COMMIT-A>/" \
+             -e "s/$B/<COMMIT-B>/" \
+             -e "s/$C/<COMMIT-C>/" \
+             -e "s/$D/<COMMIT-D>/" \
+             -e "s/$E/<COMMIT-E>/" \
+             -e "s/$F/<COMMIT-F>/" \
+             -e "s/$G/<COMMIT-G>/" \
+             -e "s/$H/<COMMIT-H>/" \
+             -e "s/$I/<COMMIT-I>/" \
+             -e "s/$J/<COMMIT-J>/" \
+             -e "s/$K/<COMMIT-K>/" \
+             -e "s/$L/<COMMIT-L>/" \
+             -e "s/$M/<COMMIT-M>/" \
+             -e "s/$N/<COMMIT-N>/" \
+             -e "s/$O/<COMMIT-O>/" \
+             -e "s/$P/<COMMIT-P>/" \
+             -e "s/$TAG1/<TAG-1>/" \
+             -e "s/$TAG2/<TAG-2>/" \
+             -e "s/$TAG3/<TAG-3>/" \
+             -e "s/$(echo $A | cut -c1-7)[0-9a-f]*/<OID-A>/g" \
+             -e "s/$(echo $B | cut -c1-7)[0-9a-f]*/<OID-B>/g" \
+             -e "s/$(echo $C | cut -c1-7)[0-9a-f]*/<OID-C>/g" \
+             -e "s/$(echo $D | cut -c1-7)[0-9a-f]*/<OID-D>/g" \
+             -e "s/$(echo $E | cut -c1-7)[0-9a-f]*/<OID-E>/g" \
+             -e "s/$(echo $F | cut -c1-7)[0-9a-f]*/<OID-F>/g" \
+             -e "s/$(echo $G | cut -c1-7)[0-9a-f]*/<OID-G>/g" \
+             -e "s/$(echo $H | cut -c1-7)[0-9a-f]*/<OID-H>/g" \
+             -e "s/$(echo $I | cut -c1-7)[0-9a-f]*/<OID-I>/g" \
+             -e "s/$(echo $J | cut -c1-7)[0-9a-f]*/<OID-J>/g" \
+             -e "s/$(echo $K | cut -c1-7)[0-9a-f]*/<OID-K>/g" \
+             -e "s/$(echo $L | cut -c1-7)[0-9a-f]*/<OID-L>/g" \
+             -e "s/$(echo $M | cut -c1-7)[0-9a-f]*/<OID-M>/g" \
+             -e "s/$(echo $N | cut -c1-7)[0-9a-f]*/<OID-N>/g" \
+             -e "s/$(echo $O | cut -c1-7)[0-9a-f]*/<OID-O>/g" \
+             -e "s/$(echo $P | cut -c1-7)[0-9a-f]*/<OID-P>/g" \
+             -e "s/$(echo $TAG1 | cut -c1-7)[0-9a-f]*/<OID-TAG-1>/g" \
+             -e "s/$(echo $TAG2 | cut -c1-7)[0-9a-f]*/<OID-TAG-2>/g" \
+             -e "s/$(echo $TAG3 | cut -c1-7)[0-9a-f]*/<OID-TAG-3>/g" \
+             -e "s/ *\$//"

t/t6020-bundle-misc.sh line 243

Quoted 10 lines
t/t6020-bundle-misc.sh new file +9 -0
+}
...
+test_expect_success 'create bundle from special rev: main^!' '
+     git bundle create special-rev.bdl "main^!" &&
+
+     git bundle list-heads special-rev.bdl |
+             make_user_friendly_and_stable_output >actual &&
+     cat >expect <<-EOF &&
+             <COMMIT-P> refs/heads/main
+             EOF

We prefer to indent these more like so:

        cat >expect <<-\EOF &&
        <COMMIT-P> refs/heads/main
        EOF

i.e. the indent of the line with <<EOF on it and the indent of the line with the matching EOF are the same. Also, quote EOF to signal that the body of the here text should be taken as-is without $var substitution.

#12 lore raw

Re: [PATCH v4 1/2] bundle: lost objects when removing duplicate pendings

Quoted 14 lines · Jiang Xin
Quoted 14 lines
Quoted 14 lines
t/t6020-bundle-misc.sh new file +13 -0
+# Create a commit or tag and set the variable with the object ID.
+test_commit_setvar () {
+     notick= &&
+     signoff= &&
+     indir= &&
+     merge= &&
+     tag= &&
+     var= &&
+     while test $# != 0
+     do
+             case "$1" in
...
+             -*)
+                     echo >&2 "error: unknown option $1"
Quoted 8 lines
Quoted 5 lines

t/t6020-bundle-misc.sh line 95

Quoted 5 lines
t/t6020-bundle-misc.sh new file +5 -0
+                     return 1
+                     ;;
+             *)
+                     test -n "$var" && break
+                     var=$1

The loop ends only if $var has been assigned a value, or no other args. Will report error if no other args later.

"We see an arg that is not an dashed option. We already saw an arg and taken it as the name of the variable, so break" was a misleading way to structure this loop.

It looked as if it was just refusing to parse the remainder of the command line, so that a check after the loop would complain if there is still remaining arguments (as if to warn "var given twice"). But that is not what the post-loop check does; it expects there still are some argument left to be processed in mode-specific code that follows, so it happens to work as intended.

That is brittle, though. The current code may always consume one or more extra arguments in $merge/$tag/other specific mode in the code after the loop, but a new mode that will get added in the future to sit next to --merge and --tag may learn all necessary info in the command line parsing loop above, without any need for extra args to be processed after the loop. And $#=0 may not always be an error at that point.

I forgot to notice / mention it, but now you made me to look at the loop again, I see this part

	-C)
		indir="$2"
		shift
		;;

does not ensure we are getting something sensible in -C; that potential bug by the caller also happens to be covered by the post-loop "we require at least one argument that we can use as an arg" check, but as I said already, that feels rather brittle.

Anyway, let's queue the patches as-is and see what others think.

Thanks.

author
#13 lore raw

v5 cover improvements for git-bundle

From: Jiang Xin <[email protected]>

Introduce two improvements for git-bundle

+ Commit "bundle: lost objects when removing duplicate pendings", which fixes command like:

        $ git bundle create <file> 'master^!'

+ Commits "bundle: arguments can be read from stdin", which add "--stdin" option support for git-bundle, like:

        $ git bundle create <file> <input

## Changes since v4

+ New patch 1: Add helper functions in 't/test-bundle-functions.sh' for git-bundle.
+ Move t/t6020 to patch 1.

--

Jiang Xin (3):
test: add helper functions for git-bundle
bundle: lost objects when removing duplicate pendings
bundle: arguments can be read from stdin

 bundle.c                   | 109 +++++----
 object.c                   |  10 +-
 t/t5510-fetch.sh           |  26 +--
 t/t5607-clone-bundle.sh    |   4 +-
 t/t6020-bundle-misc.sh     | 465 +++++++++++++++++++++++++++++++++++++
 t/test-bundle-functions.sh |  47 ++++
 6 files changed, 583 insertions(+), 78 deletions(-)
 create mode 100755 t/t6020-bundle-misc.sh
 create mode 100644 t/test-bundle-functions.sh
Signature · 1 line
2.30.0.2.g06d2f50715
author
#14 lore raw

PATCH v5 1/3 test: add helper functions for git-bundle

In reply to Junio C Hamano

From: Jiang Xin <[email protected]>

Move git-bundle related functions from t5510 to a library, and this lib will be shared with a new testcase t6020 which finds a known breakage of "git-bundle".

---
 t/t5510-fetch.sh           |  26 +--
 t/t6020-bundle-misc.sh     | 396 +++++++++++++++++++++++++++++++++++++
 t/test-bundle-functions.sh |  47 +++++
 3 files changed, 447 insertions(+), 22 deletions(-)
 create mode 100755 t/t6020-bundle-misc.sh
 create mode 100644 t/test-bundle-functions.sh
t/t5510-fetch.sh +4 -22
index 2013051a64..1e398380eb 100755
@@ -6,22 +6,10 @@ test_description='Per branch config variables affects "git fetch".
 '
 
 . ./test-lib.sh
+. "$TEST_DIRECTORY"/test-bundle-functions.sh
 
 D=$(pwd)
 
-test_bundle_object_count () {
-	git verify-pack -v "$1" >verify.out &&
-	test "$2" = $(grep "^$OID_REGEX " verify.out | wc -l)
-}
-
-convert_bundle_to_pack () {
-	while read x && test -n "$x"
-	do
-		:;
-	done
-	cat
-}
-
 test_expect_success setup '
 	echo >file original &&
 	git add file &&
@@ -312,9 +300,7 @@ test_expect_success 'unbundle 1' '
 
 test_expect_success 'bundle 1 has only 3 files ' '
 	cd "$D" &&
-	convert_bundle_to_pack <bundle1 >bundle.pack &&
-	git index-pack bundle.pack &&
-	test_bundle_object_count bundle.pack 3
+	test_bundle_object_count bundle1 3
 '
 
 test_expect_success 'unbundle 2' '
@@ -329,9 +315,7 @@ test_expect_success 'bundle does not prerequisite objects' '
 	git add file2 &&
 	git commit -m add.file2 file2 &&
 	git bundle create bundle3 -1 HEAD &&
-	convert_bundle_to_pack <bundle3 >bundle.pack &&
-	git index-pack bundle.pack &&
-	test_bundle_object_count bundle.pack 3
+	test_bundle_object_count bundle3 3
 '
 
 test_expect_success 'bundle should be able to create a full history' '
@@ -884,9 +868,7 @@ test_expect_success 'all boundary commits are excluded' '
 	git merge otherside &&
 	ad=$(git log --no-walk --format=%ad HEAD) &&
 	git bundle create twoside-boundary.bdl main --since="$ad" &&
-	convert_bundle_to_pack <twoside-boundary.bdl >twoside-boundary.pack &&
-	pack=$(git index-pack --fix-thin --stdin <twoside-boundary.pack) &&
-	test_bundle_object_count .git/objects/pack/pack-${pack##pack	}.pack 3
+	test_bundle_object_count --thin twoside-boundary.bdl 3
 '
 
 test_expect_success 'fetch --prune prints the remotes url' '
t/t6020-bundle-misc.sh new file +396 -0

396 lines not on this page · raw message

t/test-bundle-functions.sh new file +47 -0

47 lines not on this page · raw message

Signature · 1 line
2.30.0.2.g06d2f50715
#15 lore raw

Re: [PATCH v5 1/3] test: add helper functions for git-bundle

Quoted 36 lines · Jiang Xin
t/t6020-bundle-misc.sh new file +36 -0
+# Create a commit or tag and set the variable with the object ID.
+test_commit_setvar () {
+	notick=
+	signoff=
+	indir=
+	merge=
+	tag=
+	var=
+
+	while test $# != 0
+	do
+		case "$1" in
+		--merge)
+			merge=t
+			;;
+		--tag)
+			tag=t
+			;;
+		--notick)
+			notick=t
+			;;
+		--signoff)
+			signoff="$1"
+			;;
+		-C)
+			shift
+			indir="$1"
+			;;
+		-*)
+			echo >&2 "error: unknown option $1"
+			return 1
+			;;
+		*)
+			break
+			;;
+		esac

t/t6020-bundle-misc.sh line 59

Quoted 10 lines
t/t6020-bundle-misc.sh new file +10 -0
+		shift
+	done
+
+	var=$1
+	shift
+	if test -z "$var"
+	then
+		echo >&2 "error: var is not defined"
+		return 1
+	fi

We need to check $# immediately after the loop to ensure that we can carve out $var and at least another arg. [*Nit 1*]

The previous round required the command line to have at least one after the loop (including parsing of $var) parsed it, but now we fall through from here when a command line were:

    test_commit_setvar --merge -C there VAR

and because "$1" does not exist, such an error is propagated down to "git merge" not getting the side branch, "git tag" not getting the object to tag, etc.

Quoted 7 lines
t/t6020-bundle-misc.sh new file +7 -0
+	indir=${indir:+"$indir"/}
+	if test -z "$notick"
+	then
+		test_tick
+	fi &&
+	if test -n "$merge"
+	then

t/t6020-bundle-misc.sh line 76

Quoted 10 lines
t/t6020-bundle-misc.sh new file +10 -0
+		git ${indir:+ -C "$indir"} merge --no-edit --no-ff \
+			${2:+-m "$2"} "$1" &&
+		oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD)
+	elif test -n "$tag"
+	then
+		git ${indir:+ -C "$indir"} tag -m "$1" "$1" &&
+		oid=$(git ${indir:+ -C "$indir"} rev-parse "$1")
+	else
+		file=${2:-"$1.t"} &&
+		echo "${3-$1}" > "$indir$file" &&

Style? [*Nit 2*]

Quoted 6 lines
t/t6020-bundle-misc.sh new file +6 -0
+		git ${indir:+ -C "$indir"} add "$file" &&
+		git ${indir:+ -C "$indir"} commit $signoff -m "$1" &&
+		oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD)
+	fi &&
+	eval $var=$oid
+}

t/t6020-bundle-misc.sh line 90

Quoted 10 lines
t/t6020-bundle-misc.sh new file +8 -0
 
 
+
+
+# Format the output of git commands to make a user-friendly and stable
+# text.  We can easily prepare the expect text without having to worry
+# about future changes of the commit ID and spaces of the output.
+make_user_friendly_and_stable_output () {
+	sed \
+		-e "s/$(echo $A | cut -c1-7)[0-9a-f]*/<COMMIT-A>/g" \

Is "$(echo $A | cut -c1-7)" the same as "${A%${A#???????}}"? If so, the latter may be a bit shorter.

Quoted 33 lines
t/test-bundle-functions.sh new file +27 -0
new file mode 100644
index 0000000000..0853eb1eca
@@ -0,0 +1,47 @@
+# Library of git-bundle related functions.
+
+# Display the pack data contained in the bundle file, bypassing the
+# header that contains the signature, prerequisites and references.
+convert_bundle_to_pack () {
+	while read x && test -n "$x"
+	do
+		:;
+	done
+	cat
+}
+
+# Check count of objects in a bundle file.
+# We can use "--thin" opiton to check thin pack, which must be fixed by
+# command `git-index-pack --fix-thin --stdin`.
+test_bundle_object_count () {
+	thin=
+	if test "$1" = "--thin"
+	then
+		thin=t
+		shift
+	fi
+	if test $# -ne 2
+	then
+		echo >&2 "args should be: <bundle> <count>"
+		return 1
+	fi

t/test-bundle-functions.sh line 37

Quoted 10 lines
t/test-bundle-functions.sh new file +10 -0
+	bundle=$1
+	pack=$bundle.pack
+	convert_bundle_to_pack <"$bundle" >"$pack" &&
+	if test -n "$thin"
+	then
+		mv "$pack" "$bundle.thin.pack" &&
+		git index-pack --stdin --fix-thin "$pack" <"$bundle.thin.pack"
+	else
+		git index-pack "$pack"
+	fi

I wonder why we shouldn't always do "--fix-thin", so that the caller does not even have to bother passing the "--thin" option.

Is this to protect us from "git bundle" creating a bundle that contains a thin pack when it should not? A caller that knows it is storing a fully connected history can deliberately omit "--thin" from the command line and make sure "index-pack" that is not asked to do "--fix-thin" indeed finds the pack data fully self-contained, so it may be a good idea to have these two separate codepaths after all. OK.

t/test-bundle-functions.sh line 42

Quoted 5 lines
t/test-bundle-functions.sh new file +5 -0
+	if test $? -ne 0
+	then
+		echo >&2 "error: fail to convert $bundle or index-pack"
+		return 1
+	fi

Do we even need the "error" message? "git index-pack" would have already given some error message to its standard error stream, no? If so

	if test -n "$thin"
	then
		...
	fi || return 1

would be sufficient, I guess.

t/test-bundle-functions.sh line 47

Quoted 5 lines
t/test-bundle-functions.sh new file +5 -0
+	count=$(git show-index <"${pack%pack}idx" | wc -l) &&
+	test $2 = $count && return 0
+	echo >&2 "error: object count for $bundle is $count, not $2"
+	return 1
+}

Looking good except for a few minor nits I mentioned above.

Thanks.

author
#16 lore raw

v6 cover improvements for git-bundle

From: Jiang Xin <[email protected]>

Introduce two improvements for git-bundle

+ Commit "bundle: lost objects when removing duplicate pendings", which fixes command like:

        $ git bundle create <file> 'master^!'

+ Commits "bundle: arguments can be read from stdin", which add "--stdin" option support for git-bundle, like:

        $ git bundle create <file> <input

## Changes since v5:

Quoted 18 lines · Junio C Hamano
Quoted 7 lines · Jiang Xin

+     var=$1
+     shift
+     if test -z "$var"
+     then
+             echo >&2 "error: var is not defined"
+             return 1
+     fi

We need to check $# immediately after the loop to ensure that we can carve out $var and at least another arg.  [*Nit 1*]

The previous round required the command line to have at least one after the loop (including parsing of $var) parsed it, but now we fall through from here when a command line were:

Quoted 5 lines

    test_commit_setvar --merge -C there VAR

and because "$1" does not exist, such an error is propagated down to "git merge" not getting the side branch, "git tag" not getting the object to tag, etc.

range-diff v5...v6 (part-1):

    @@ t/t6020-bundle-misc.sh (new)
     +		esac
     +		shift
     +	done
    -+
    -+	var=$1
    -+	shift
    -+	if test -z "$var"
    ++	if test $# -lt 2
     +	then
    -+		echo >&2 "error: var is not defined"
    ++		echo >&2 "error: test_commit_setvar must have at least 2 arguments"
     +		return 1
     +	fi
    ++	var=$1
    ++	shift
     +	indir=${indir:+"$indir"/}
     +	if test -z "$notick"
     +	then
Quoted 5 lines
Quoted 3 lines

+     else
+             file=${2:-"$1.t"} &&
+             echo "${3-$1}" > "$indir$file" &&

Style?  [*Nit 2*]

range-diff v5...v6 (part-2):

* Add new arg "${2:-HEAD}" for git tag.
* Fix [*Nit 2*].

    @@ t/t6020-bundle-misc.sh (new)
     +		oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD)
     +	elif test -n "$tag"
     +	then
    -+		git ${indir:+ -C "$indir"} tag -m "$1" "$1" &&
    ++		git ${indir:+ -C "$indir"} tag -m "$1" "$1" "${2:-HEAD}" &&
     +		oid=$(git ${indir:+ -C "$indir"} rev-parse "$1")
     +	else
     +		file=${2:-"$1.t"} &&
    -+		echo "${3-$1}" > "$indir$file" &&
    ++		echo "${3-$1}" >"$indir$file" &&
     +		git ${indir:+ -C "$indir"} add "$file" &&
     +		git ${indir:+ -C "$indir"} commit $signoff -m "$1" &&
     +		oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD)
Quoted 9 lines

t/t6020-bundle-misc.sh line 90

Quoted 6 lines
t/t6020-bundle-misc.sh new file +6 -0
+# Format the output of git commands to make a user-friendly and stable
+# text.  We can easily prepare the expect text without having to worry
+# about future changes of the commit ID and spaces of the output.
+make_user_friendly_and_stable_output () {
+     sed \
+             -e "s/$(echo $A | cut -c1-7)[0-9a-f]*/<COMMIT-A>/g" \

Is "$(echo $A | cut -c1-7)" the same as "${A%${A#???????}}"?  If so, the latter may be a bit shorter.

range-diff v5...v6 (part-3):

    @@ t/t6020-bundle-misc.sh (new)
     +	eval $var=$oid
     +}
     +
    -+
     +# Format the output of git commands to make a user-friendly and stable
     +# text.  We can easily prepare the expect text without having to worry
     +# about future changes of the commit ID and spaces of the output.
     +make_user_friendly_and_stable_output () {
     +	sed \
    -+		-e "s/$(echo $A | cut -c1-7)[0-9a-f]*/<COMMIT-A>/g" \
    -+		-e "s/$(echo $B | cut -c1-7)[0-9a-f]*/<COMMIT-B>/g" \
    -+		-e "s/$(echo $C | cut -c1-7)[0-9a-f]*/<COMMIT-C>/g" \
    -+		-e "s/$(echo $D | cut -c1-7)[0-9a-f]*/<COMMIT-D>/g" \
    -+		-e "s/$(echo $E | cut -c1-7)[0-9a-f]*/<COMMIT-E>/g" \
    -+		-e "s/$(echo $F | cut -c1-7)[0-9a-f]*/<COMMIT-F>/g" \
    -+		-e "s/$(echo $G | cut -c1-7)[0-9a-f]*/<COMMIT-G>/g" \
    -+		-e "s/$(echo $H | cut -c1-7)[0-9a-f]*/<COMMIT-H>/g" \
    -+		-e "s/$(echo $I | cut -c1-7)[0-9a-f]*/<COMMIT-I>/g" \
    -+		-e "s/$(echo $J | cut -c1-7)[0-9a-f]*/<COMMIT-J>/g" \
    -+		-e "s/$(echo $K | cut -c1-7)[0-9a-f]*/<COMMIT-K>/g" \
    -+		-e "s/$(echo $L | cut -c1-7)[0-9a-f]*/<COMMIT-L>/g" \
    -+		-e "s/$(echo $M | cut -c1-7)[0-9a-f]*/<COMMIT-M>/g" \
    -+		-e "s/$(echo $N | cut -c1-7)[0-9a-f]*/<COMMIT-N>/g" \
    -+		-e "s/$(echo $O | cut -c1-7)[0-9a-f]*/<COMMIT-O>/g" \
    -+		-e "s/$(echo $P | cut -c1-7)[0-9a-f]*/<COMMIT-P>/g" \
    -+		-e "s/$(echo $TAG1 | cut -c1-7)[0-9a-f]*/<TAG-1>/g" \
    -+		-e "s/$(echo $TAG2 | cut -c1-7)[0-9a-f]*/<TAG-2>/g" \
    -+		-e "s/$(echo $TAG3 | cut -c1-7)[0-9a-f]*/<TAG-3>/g" \
    ++		-e "s/${A%${A#???????}}[0-9a-f]*/<COMMIT-A>/g" \
    ++		-e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" \
    ++		-e "s/${C%${C#???????}}[0-9a-f]*/<COMMIT-C>/g" \
    ++		-e "s/${D%${D#???????}}[0-9a-f]*/<COMMIT-D>/g" \
    ++		-e "s/${E%${E#???????}}[0-9a-f]*/<COMMIT-E>/g" \
    ++		-e "s/${F%${F#???????}}[0-9a-f]*/<COMMIT-F>/g" \
    ++		-e "s/${G%${G#???????}}[0-9a-f]*/<COMMIT-G>/g" \
    ++		-e "s/${H%${H#???????}}[0-9a-f]*/<COMMIT-H>/g" \
    ++		-e "s/${I%${I#???????}}[0-9a-f]*/<COMMIT-I>/g" \
    ++		-e "s/${J%${J#???????}}[0-9a-f]*/<COMMIT-J>/g" \
    ++		-e "s/${K%${K#???????}}[0-9a-f]*/<COMMIT-K>/g" \
    ++		-e "s/${L%${L#???????}}[0-9a-f]*/<COMMIT-L>/g" \
    ++		-e "s/${M%${M#???????}}[0-9a-f]*/<COMMIT-M>/g" \
    ++		-e "s/${N%${N#???????}}[0-9a-f]*/<COMMIT-N>/g" \
    ++		-e "s/${O%${O#???????}}[0-9a-f]*/<COMMIT-O>/g" \
    ++		-e "s/${P%${P#???????}}[0-9a-f]*/<COMMIT-P>/g" \
    ++		-e "s/${TAG1%${TAG1#???????}}[0-9a-f]*/<TAG-1>/g" \
    ++		-e "s/${TAG2%${TAG2#???????}}[0-9a-f]*/<TAG-2>/g" \
    ++		-e "s/${TAG3%${TAG3#???????}}[0-9a-f]*/<TAG-3>/g" \
     +		-e "s/ *\$//"
     +}
     +
Quoted 10 lines

Do we even need the "error" message?  "git index-pack" would have already given some error message to its standard error stream, no? If so

        if test -n "$thin"
        then
                ...
        fi || return 1

would be sufficient, I guess.

range-diff v5...v6 (part-4):

1: fa7516b2ec ! 1: 900bb16178 test: add helper functions for git-bundle

    @@ t/test-bundle-functions.sh (new)
     +		git index-pack --stdin --fix-thin "$pack" <"$bundle.thin.pack"
     +	else
     +		git index-pack "$pack"
    -+	fi
    -+	if test $? -ne 0
    -+	then
    -+		echo >&2 "error: fail to convert $bundle or index-pack"
    -+		return 1
    -+	fi
    ++	fi || return 1
     +	count=$(git show-index <"${pack%pack}idx" | wc -l) &&
     +	test $2 = $count && return 0
     +	echo >&2 "error: object count for $bundle is $count, not $2"
2:  ea543de111 = 2:  1bbf0ab213 bundle: lost objects when removing duplicate pendings
3:  18f0d48814 = 3:  7ac0751821 bundle: arguments can be read from stdin

--

Jiang Xin (3):
test: add helper functions for git-bundle
bundle: lost objects when removing duplicate pendings
bundle: arguments can be read from stdin

 bundle.c                   | 109 +++++----
 object.c                   |  10 +-
 t/t5510-fetch.sh           |  26 +--
 t/t5607-clone-bundle.sh    |   4 +-
 t/t6020-bundle-misc.sh     | 463 +++++++++++++++++++++++++++++++++++++
 t/test-bundle-functions.sh |  42 ++++
 6 files changed, 576 insertions(+), 78 deletions(-)
 create mode 100755 t/t6020-bundle-misc.sh
 create mode 100644 t/test-bundle-functions.sh
Signature · 1 line
2.28.0.15.gba9e81f0bd
author
#17 lore raw

PATCH v6 1/3 test: add helper functions for git-bundle

In reply to Junio C Hamano

From: Jiang Xin <[email protected]>

Move git-bundle related functions from t5510 to a library, and this lib will be shared with a new testcase t6020 which finds a known breakage of "git-bundle".

---
 t/t5510-fetch.sh           |  26 +--
 t/t6020-bundle-misc.sh     | 394 +++++++++++++++++++++++++++++++++++++
 t/test-bundle-functions.sh |  42 ++++
 3 files changed, 440 insertions(+), 22 deletions(-)
 create mode 100755 t/t6020-bundle-misc.sh
 create mode 100644 t/test-bundle-functions.sh
t/t5510-fetch.sh +4 -22
index 2013051a64..1e398380eb 100755
@@ -6,22 +6,10 @@ test_description='Per branch config variables affects "git fetch".
 '
 
 . ./test-lib.sh
+. "$TEST_DIRECTORY"/test-bundle-functions.sh
 
 D=$(pwd)
 
-test_bundle_object_count () {
-	git verify-pack -v "$1" >verify.out &&
-	test "$2" = $(grep "^$OID_REGEX " verify.out | wc -l)
-}
-
-convert_bundle_to_pack () {
-	while read x && test -n "$x"
-	do
-		:;
-	done
-	cat
-}
-
 test_expect_success setup '
 	echo >file original &&
 	git add file &&
@@ -312,9 +300,7 @@ test_expect_success 'unbundle 1' '
 
 test_expect_success 'bundle 1 has only 3 files ' '
 	cd "$D" &&
-	convert_bundle_to_pack <bundle1 >bundle.pack &&
-	git index-pack bundle.pack &&
-	test_bundle_object_count bundle.pack 3
+	test_bundle_object_count bundle1 3
 '
 
 test_expect_success 'unbundle 2' '
@@ -329,9 +315,7 @@ test_expect_success 'bundle does not prerequisite objects' '
 	git add file2 &&
 	git commit -m add.file2 file2 &&
 	git bundle create bundle3 -1 HEAD &&
-	convert_bundle_to_pack <bundle3 >bundle.pack &&
-	git index-pack bundle.pack &&
-	test_bundle_object_count bundle.pack 3
+	test_bundle_object_count bundle3 3
 '
 
 test_expect_success 'bundle should be able to create a full history' '
@@ -884,9 +868,7 @@ test_expect_success 'all boundary commits are excluded' '
 	git merge otherside &&
 	ad=$(git log --no-walk --format=%ad HEAD) &&
 	git bundle create twoside-boundary.bdl main --since="$ad" &&
-	convert_bundle_to_pack <twoside-boundary.bdl >twoside-boundary.pack &&
-	pack=$(git index-pack --fix-thin --stdin <twoside-boundary.pack) &&
-	test_bundle_object_count .git/objects/pack/pack-${pack##pack	}.pack 3
+	test_bundle_object_count --thin twoside-boundary.bdl 3
 '
 
 test_expect_success 'fetch --prune prints the remotes url' '
t/t6020-bundle-misc.sh new file +394 -0

394 lines not on this page · raw message

t/test-bundle-functions.sh new file +42 -0

42 lines not on this page · raw message

Signature · 1 line
2.28.0.15.gba9e81f0bd
#18 lore raw

Runaway sed memory use in test on older sed+glibc (was "Re: [PATCH v6 1/3] test: add helper functions for git-bundle")

Quoted 23 lines · Jiang Xin

From: Jiang Xin <[email protected]>

Move git-bundle related functions from t5510 to a library, and this lib will be shared with a new testcase t6020 which finds a known breakage of "git-bundle".
[...]

t/t6020-bundle-misc.sh new file +17 -0
+
+# Format the output of git commands to make a user-friendly and stable
+# text.  We can easily prepare the expect text without having to worry
+# about future changes of the commit ID and spaces of the output.
+make_user_friendly_and_stable_output () {
+	sed \
+		-e "s/${A%${A#???????}}[0-9a-f]*/<COMMIT-A>/g" \
+		-e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" \
+		-e "s/${C%${C#???????}}[0-9a-f]*/<COMMIT-C>/g" \
+		-e "s/${D%${D#???????}}[0-9a-f]*/<COMMIT-D>/g" \
+		-e "s/${E%${E#???????}}[0-9a-f]*/<COMMIT-E>/g" \
+		-e "s/${F%${F#???????}}[0-9a-f]*/<COMMIT-F>/g" \
+		-e "s/${G%${G#???????}}[0-9a-f]*/<COMMIT-G>/g" \
+		-e "s/${H%${H#???????}}[0-9a-f]*/<COMMIT-H>/g" \
+		-e "s/${I%${I#???????}}[0-9a-f]*/<COMMIT-I>/g" \
+		-e "s/${J%${J#???????}}[0-9a-f]*/<COMMIT-J>/g" \
+		-e "s/${K%${K#???????}}[0-9a-f]*/<COMMIT-K>/g" \

t/t6020-bundle-misc.sh line 108

Quoted 10 lines
t/t6020-bundle-misc.sh new file +10 -0
+		-e "s/${L%${L#???????}}[0-9a-f]*/<COMMIT-L>/g" \
+		-e "s/${M%${M#???????}}[0-9a-f]*/<COMMIT-M>/g" \
+		-e "s/${N%${N#???????}}[0-9a-f]*/<COMMIT-N>/g" \
+		-e "s/${O%${O#???????}}[0-9a-f]*/<COMMIT-O>/g" \
+		-e "s/${P%${P#???????}}[0-9a-f]*/<COMMIT-P>/g" \
+		-e "s/${TAG1%${TAG1#???????}}[0-9a-f]*/<TAG-1>/g" \
+		-e "s/${TAG2%${TAG2#???????}}[0-9a-f]*/<TAG-2>/g" \
+		-e "s/${TAG3%${TAG3#???????}}[0-9a-f]*/<TAG-3>/g" \
+		-e "s/ *\$//"
+}

On one of the gcc farm boxes, a i386 box (gcc45) this fails because sed gets killed after >500MB of memory use (I was just eyeballing it in htop) on the "reate bundle from special rev: main^!" test. This with GNU sed 4.2.2.

I suspect this regex pattern creates some runaway behavior in sed that's since been fixed (or maybe it's the glibc regex engine?). The glibc is 2.19-18+deb8u10:

+ git bundle list-heads special-rev.bdl
+ make_user_friendly_and_stable_output
+ sed -e s/[0-9a-f]*/<COMMIT-A>/g -e s/[0-9a-f]*/<COMMIT-B>/g -e s/[0-9a-f]*/<COMMIT-C>/g -e s/[0-9a-f]*/<COMMIT-D>/g -e s/[0-9a-f]*/<COMMIT-E>/g -e s/[0-9a-f]*/<COMMIT-F>/g -e s/[0-9a-f]*/<COMMIT-G>/g -e s/[0-9a-f]*/<COMMIT-H>/g -e s/[0-9a-f]*/<COMMIT-I>/g -e s/[0-9a-f]*/<COMMIT-J>/g -e s/[0-9a-f]*/<COMMIT-K>/g -e s/[0-9a-f]*/<COMMIT-L>/g -e s/[0-9a-f]*/<COMMIT-M>/g -e s/[0-9a-f]*/<COMMIT-N>/g -e s/[0-9a-f]*/<COMMIT-O>/g -e s/[0-9a-f]*/<COMMIT-P>/g -e s/[0-9a-f]*/<TAG-1>/g -e s/[0-9a-f]*/<TAG-2>/g -e s/[0-9a-f]*/<TAG-3>/g -e s/ *$//
sed: couldn't re-allocate memory

author
#19 lore raw

Re: Runaway sed memory use in test on older sed+glibc (was "Re: [PATCH v6 1/3] test: add helper functions for git-bundle")

Ævar Arnfjörð Bjarmason <[email protected]> 于2021年5月27日周四

Quoted 54 lines · 上午2:51写道:
Quoted 27 lines · Jiang Xin

From: Jiang Xin <[email protected]>

Move git-bundle related functions from t5510 to a library, and this lib
will be shared with a new testcase t6020 which finds a known breakage of
"git-bundle".
[...]

(quoted diff) +2 -0
+
+# Format the output of git commands to make a user-friendly and

stable
+# text.  We can easily prepare the expect text without having to worry

(quoted diff) +14 -0
+# about future changes of the commit ID and spaces of the output.
+make_user_friendly_and_stable_output () {
+     sed \
+             -e "s/${A%${A#???????}}[0-9a-f]*/<COMMIT-A>/g" \
+             -e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" \
+             -e "s/${C%${C#???????}}[0-9a-f]*/<COMMIT-C>/g" \
+             -e "s/${D%${D#???????}}[0-9a-f]*/<COMMIT-D>/g" \
+             -e "s/${E%${E#???????}}[0-9a-f]*/<COMMIT-E>/g" \
+             -e "s/${F%${F#???????}}[0-9a-f]*/<COMMIT-F>/g" \
+             -e "s/${G%${G#???????}}[0-9a-f]*/<COMMIT-G>/g" \
+             -e "s/${H%${H#???????}}[0-9a-f]*/<COMMIT-H>/g" \
+             -e "s/${I%${I#???????}}[0-9a-f]*/<COMMIT-I>/g" \
+             -e "s/${J%${J#???????}}[0-9a-f]*/<COMMIT-J>/g" \
+             -e "s/${K%${K#???????}}[0-9a-f]*/<COMMIT-K>/g" \

Review

Quoted 10 lines
(quoted diff) +10 -0
+             -e "s/${L%${L#???????}}[0-9a-f]*/<COMMIT-L>/g" \
+             -e "s/${M%${M#???????}}[0-9a-f]*/<COMMIT-M>/g" \
+             -e "s/${N%${N#???????}}[0-9a-f]*/<COMMIT-N>/g" \
+             -e "s/${O%${O#???????}}[0-9a-f]*/<COMMIT-O>/g" \
+             -e "s/${P%${P#???????}}[0-9a-f]*/<COMMIT-P>/g" \
+             -e "s/${TAG1%${TAG1#???????}}[0-9a-f]*/<TAG-1>/g" \
+             -e "s/${TAG2%${TAG2#???????}}[0-9a-f]*/<TAG-2>/g" \
+             -e "s/${TAG3%${TAG3#???????}}[0-9a-f]*/<TAG-3>/g" \
+             -e "s/ *\$//"
+}

On one of the gcc farm boxes, a i386 box (gcc45) this fails because sed
gets killed after >500MB of memory use (I was just eyeballing it in htop) on the "reate bundle from special rev: main^!" test. This with GNU
sed 4.2.2.

I suspect this regex pattern creates some runaway behavior in sed that's
since been fixed (or maybe it's the glibc regex engine?). The glibc is 2.19-18+deb8u10:

Quoted 13 lines

    + git bundle list-heads special-rev.bdl
    + make_user_friendly_and_stable_output
    + sed -e s/[0-9a-f]*/<COMMIT-A>/g -e s/[0-9a-f]*/<COMMIT-B>/g -e s/[0-9a-f]*/<COMMIT-C>/g -e s/[0-9a-f]*/<COMMIT-D>/g -e s/[0-9a-f]*/<COMMIT-E>/g -e s/[0-9a-f]*/<COMMIT-F>/g -e s/[0-9a-f]*/<COMMIT-G>/g -e s/[0-9a-f]*/<COMMIT-H>/g -e s/[0-9a-f]*/<COMMIT-I>/g -e s/[0-9a-f]*/<COMMIT-J>/g -e s/[0-9a-f]*/<COMMIT-K>/g -e s/[0-9a-f]*/<COMMIT-L>/g -e s/[0-9a-f]*/<COMMIT-M>/g -e s/[0-9a-f]*/<COMMIT-N>/g -e s/[0-9a-f]*/<COMMIT-O>/g -e s/[0-9a-f]*/<COMMIT-P>/g -e s/[0-9a-f]*/<TAG-1>/g -e s/[0-9a-f]*/<TAG-2>/g -e s/[0-9a-f]*/<TAG-3>/g -e s/ *$//
    sed: couldn't re-allocate memory

I wrote a program on macOS to check memory footprint for sed and perl. See:

    https://github.com/jiangxin/compare-sed-perl

Test result:

    $ go build && ./compare-sed-perl
    Command: sed  ..., MaxRSS: 901120
    Command: gsed ..., MaxRSS: 2056192
    Command: perl ..., MaxRSS: 2269184

It seems that sed (both the builtin version on macOS and GNU sed v4.8) has less memory consumed than perl.

Can you run this program on the i386 box (gcc45) to check memory consumed by sed and perl?

If this issue can be resolved by replacing sed with perl, the following patch may help:

--- >8 ---
From: Jiang Xin <[email protected]>
Date: Thu, 27 May 2021 14:31:49 +0800
Subject: [PATCH] test: use perl for complex text replacement

Ævar reported that the function make_user_friendly_and_stable_output() failed on a i386 box (gcc45) in the gcc farm boxes with error:

sed: couldn't re-allocate memory

It seems that sed (GNU sed 4.2.2) gets killed after >500MB of memory use on the "create bundle from special rev: main^!" test.

Call perl instead of sed for complex text replacement.

---
 t/t5411/common-functions.sh | 27 ++++++++++++------------
 t/t5548-push-porcelain.sh   | 20 +++++++++---------
 t/t6020-bundle-misc.sh      | 42 ++++++++++++++++++-------------------
 3 files changed, 45 insertions(+), 44 deletions(-)
t/t5411/common-functions.sh +14 -13
index 6694858e18..b6d33bdfdc 100644
@@ -39,19 +39,20 @@ create_commits_in () {
 # remove some locale error messages. The emitted human-readable errors are
 # redundant to the more machine-readable output the tests already assert.
 make_user_friendly_and_stable_output () {
-	sed \
-		-e "s/  *\$//" \
-		-e "s/  */ /g" \
-		-e "s/'/\"/g" \
-		-e "s/	/    /g" \
-		-e "s/$A/<COMMIT-A>/g" \
-		-e "s/$B/<COMMIT-B>/g" \
-		-e "s/$TAG/<TAG-v123>/g" \
-		-e "s/$ZERO_OID/<ZERO-OID>/g" \
-		-e "s/$(echo $A | cut -c1-7)[0-9a-f]*/<OID-A>/g" \
-		-e "s/$(echo $B | cut -c1-7)[0-9a-f]*/<OID-B>/g" \
-		-e "s#To $URL_PREFIX/upstream.git#To <URL/of/upstream.git>#" \
-		-e "/^error: / d"
+	perl -ne "
+		s/  *\$//;
+		s/  */ /g;
+		s/'/\"/g;
+		s/	/    /g;
+		s/$A/<COMMIT-A>/g;
+		s/$B/<COMMIT-B>/g;
+		s/$TAG/<TAG-v123>/g;
+		s/$ZERO_OID/<ZERO-OID>/g;
+		s/$(echo $A | cut -c1-7)[0-9a-f]*/<OID-A>/g;
+		s/$(echo $B | cut -c1-7)[0-9a-f]*/<OID-B>/g;
+		s#To $URL_PREFIX/upstream.git#To <URL/of/upstream.git>#;
+		next if /^error: .*$/;
+		print"
 }
 
 filter_out_user_friendly_and_stable_output () {
t/t5548-push-porcelain.sh +10 -10
index 5a761f3642..95e216973d 100755
@@ -44,16 +44,16 @@ create_commits_in () {
 # without having to worry about future changes of the commit ID and spaces
 # of the output.
 make_user_friendly_and_stable_output () {
-	sed \
-		-e "s/  *\$//" \
-		-e "s/   */ /g" \
-		-e "s/	/    /g" \
-		-e "s/$A/<COMMIT-A>/g" \
-		-e "s/$B/<COMMIT-B>/g" \
-		-e "s/$ZERO_OID/<ZERO-OID>/g" \
-		-e "s/$(echo $A | cut -c1-7)[0-9a-f]*/<OID-A>/g" \
-		-e "s/$(echo $B | cut -c1-7)[0-9a-f]*/<OID-B>/g" \
-		-e "s#To $URL_PREFIX/upstream.git#To <URL/of/upstream.git>#"
+	perl -pe "
+		s/  *\$//;
+		s/   */ /g;
+		s/	/    /g;
+		s/$A/<COMMIT-A>/g;
+		s/$B/<COMMIT-B>/g;
+		s/$ZERO_OID/<ZERO-OID>/g;
+		s/$(echo $A | cut -c1-7)[0-9a-f]*/<OID-A>/g;
+		s/$(echo $B | cut -c1-7)[0-9a-f]*/<OID-B>/g;
+		s#To $URL_PREFIX/upstream.git#To <URL/of/upstream.git>#"
 }
 
 setup_upstream_and_workbench () {
t/t6020-bundle-misc.sh +21 -21

48 lines not on this page · raw message

Signature · 1 line
2.32.0.rc0
#20 lore raw

Re: Runaway sed memory use in test on older sed+glibc (was "Re: [PATCH v6 1/3] test: add helper functions for git-bundle")

Quoted 70 lines · Jiang Xin

Ævar Arnfjörð Bjarmason <[email protected]> 于2021年5月27日周四

Quoted 54 lines · 上午2:51写道:
Quoted 27 lines · Jiang Xin

From: Jiang Xin <[email protected]>

Move git-bundle related functions from t5510 to a library, and this lib
will be shared with a new testcase t6020 which finds a known breakage of
"git-bundle".
[...]

(quoted diff) +2 -0
+
+# Format the output of git commands to make a user-friendly and

stable
+# text.  We can easily prepare the expect text without having to worry

(quoted diff) +14 -0
+# about future changes of the commit ID and spaces of the output.
+make_user_friendly_and_stable_output () {
+     sed \
+             -e "s/${A%${A#???????}}[0-9a-f]*/<COMMIT-A>/g" \
+             -e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" \
+             -e "s/${C%${C#???????}}[0-9a-f]*/<COMMIT-C>/g" \
+             -e "s/${D%${D#???????}}[0-9a-f]*/<COMMIT-D>/g" \
+             -e "s/${E%${E#???????}}[0-9a-f]*/<COMMIT-E>/g" \
+             -e "s/${F%${F#???????}}[0-9a-f]*/<COMMIT-F>/g" \
+             -e "s/${G%${G#???????}}[0-9a-f]*/<COMMIT-G>/g" \
+             -e "s/${H%${H#???????}}[0-9a-f]*/<COMMIT-H>/g" \
+             -e "s/${I%${I#???????}}[0-9a-f]*/<COMMIT-I>/g" \
+             -e "s/${J%${J#???????}}[0-9a-f]*/<COMMIT-J>/g" \
+             -e "s/${K%${K#???????}}[0-9a-f]*/<COMMIT-K>/g" \

Review

Quoted 10 lines
(quoted diff) +10 -0
+             -e "s/${L%${L#???????}}[0-9a-f]*/<COMMIT-L>/g" \
+             -e "s/${M%${M#???????}}[0-9a-f]*/<COMMIT-M>/g" \
+             -e "s/${N%${N#???????}}[0-9a-f]*/<COMMIT-N>/g" \
+             -e "s/${O%${O#???????}}[0-9a-f]*/<COMMIT-O>/g" \
+             -e "s/${P%${P#???????}}[0-9a-f]*/<COMMIT-P>/g" \
+             -e "s/${TAG1%${TAG1#???????}}[0-9a-f]*/<TAG-1>/g" \
+             -e "s/${TAG2%${TAG2#???????}}[0-9a-f]*/<TAG-2>/g" \
+             -e "s/${TAG3%${TAG3#???????}}[0-9a-f]*/<TAG-3>/g" \
+             -e "s/ *\$//"
+}

On one of the gcc farm boxes, a i386 box (gcc45) this fails because sed
gets killed after >500MB of memory use (I was just eyeballing it in htop) on the "reate bundle from special rev: main^!" test. This with GNU
sed 4.2.2.

I suspect this regex pattern creates some runaway behavior in sed that's
since been fixed (or maybe it's the glibc regex engine?). The glibc is 2.19-18+deb8u10:

Quoted 13 lines

    + git bundle list-heads special-rev.bdl
    + make_user_friendly_and_stable_output
    + sed -e s/[0-9a-f]*/<COMMIT-A>/g -e s/[0-9a-f]*/<COMMIT-B>/g -e s/[0-9a-f]*/<COMMIT-C>/g -e s/[0-9a-f]*/<COMMIT-D>/g -e s/[0-9a-f]*/<COMMIT-E>/g -e s/[0-9a-f]*/<COMMIT-F>/g -e s/[0-9a-f]*/<COMMIT-G>/g -e s/[0-9a-f]*/<COMMIT-H>/g -e s/[0-9a-f]*/<COMMIT-I>/g -e s/[0-9a-f]*/<COMMIT-J>/g -e s/[0-9a-f]*/<COMMIT-K>/g -e s/[0-9a-f]*/<COMMIT-L>/g -e s/[0-9a-f]*/<COMMIT-M>/g -e s/[0-9a-f]*/<COMMIT-N>/g -e s/[0-9a-f]*/<COMMIT-O>/g -e s/[0-9a-f]*/<COMMIT-P>/g -e s/[0-9a-f]*/<TAG-1>/g -e s/[0-9a-f]*/<TAG-2>/g -e s/[0-9a-f]*/<TAG-3>/g -e s/ *$//
    sed: couldn't re-allocate memory

Quoted 4 lines

I wrote a program on macOS to check memory footprint for sed and perl. See:

    https://github.com/jiangxin/compare-sed-perl

Interesting use of Go for as a /usr/bin/time -v replacement :)

After changing your int64 to int32 and digging up how to cross-compile Go I get similar results, it's because your test has actual short SHA-1s in the "-e 's///g'"'s, but notice how in the trace I have it's e.g. "s/[0-9a-f]*/<COMMIT-A>/g".

That's the problem, so that Go command won't reproduce it. Anyway, changing the test to emit to "input" first and running this shows it:

avar@gcc45:/run/user/1632/git/t/trash directory.t6020-bundle-misc$ /usr/bin/time -v sed -e 's/[0-9a-f]*/<COMMIT-A>/g' -e 's/[0-9a-f]*/<COMMIT-B>/g' -e 's/[0-9a-f]*/<COMMIT-C>/g' -e 's/[0-9a-f]*/<COMMIT-D>/g' -e 's/[0-9a-f]*/<COMMIT-E>/g' -e 's/[0-9a-f]*/<COMMIT-F>/g' -e 's/[0-9a-f]*/<COMMIT-G>/g' -e 's/[0-9a-f]*/<COMMIT-H>/g' -e 's/[0-9a-f]*/<COMMIT-I>/g' -e 's/[0-9a-f]*/<COMMIT-J>/g' -e 's/[0-9a-f]*/<COMMIT-K>/g' -e 's/[0-9a-f]*/<COMMIT-L>/g' -e 's/[0-9a-f]*/<COMMIT-M>/g' -e 's/[0-9a-f]*/<COMMIT-N>/g' -e 's/[0-9a-f]*/<COMMIT-O>/g' -e 's/[0-9a-f]*/<COMMIT-P>/g' -e 's/[0-9a-f]*/<TAG-1>/g' -e 's/[0-9a-f]*/<TAG-2>/g' -e 's/[0-9a-f]*/<TAG-3>/g' -e 's/ *$//' <input
sed: couldn't re-allocate memory
Command exited with non-zero status 4
Command being timed: "sed -e s/[0-9a-f]*/<COMMIT-A>/g -e s/[0-9a-f]*/<COMMIT-B>/g -e s/[0-9a-f]*/<COMMIT-C>/g -e s/[0-9a-f]*/<COMMIT-D>/g -e s/[0-9a-f]*/<COMMIT-E>/g -e s/[0-9a-f]*/<COMMIT-F>/g -e s/[0-9a-f]*/<COMMIT-G>/g -e s/[0-9a-f]*/<COMMIT-H>/g -e s/[0-9a-f]*/<COMMIT-I>/g -e s/[0-9a-f]*/<COMMIT-J>/g -e s/[0-9a-f]*/<COMMIT-K>/g -e s/[0-9a-f]*/<COMMIT-L>/g -e s/[0-9a-f]*/<COMMIT-M>/g -e s/[0-9a-f]*/<COMMIT-N>/g -e s/[0-9a-f]*/<COMMIT-O>/g -e s/[0-9a-f]*/<COMMIT-P>/g -e s/[0-9a-f]*/<TAG-1>/g -e s/[0-9a-f]*/<TAG-2>/g -e s/[0-9a-f]*/<TAG-3>/g -e s/ *$//" User time (seconds): 130.00
System time (seconds): 2.42
Percent of CPU this job got: 100%
Elapsed (wall clock) time (h:mm:ss or m:ss): 2:12.41
Average shared text size (kbytes): 0
Average unshared data size (kbytes): 0
Average stack size (kbytes): 0
Average total size (kbytes): 0
Maximum resident set size (kbytes): 1030968
Average resident set size (kbytes): 0
Major (requiring I/O) page faults: 0
Minor (reclaiming a frame) page faults: 257333
Voluntary context switches: 1
Involuntary context switches: 12578
Swaps: 0
File system inputs: 0
File system outputs: 0
Socket messages sent: 0
Socket messages received: 0
Signals delivered: 0
Page size (bytes): 4096
Exit status: 4

But no, the issue as it turns out is not Perl v.s. Sed, it's that there's some bug in the shellscript / tooling version (happens with both dash 0.5.7-4 and bash 4.3-11+deb8u2 on that box) where those expansions like ${A%${A#??????0?}} resolve to nothing.

So if we make that:

        cat >input &&
        cat input >&2 &&
        sed -e "s/${A%${A#??????0?}}[0-9a-f]*/<COMMIT-A>/g" <input >input.tmp && mv input.tmp input &&
        cat input >&2 &&
        sed -e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" <input >input.tmp && mv input.tmp input &&
        cat input >&2 &&

We get things like:

+ sed -e s/[0-9a-f]*/<COMMIT-A>/g
+ mv input.tmp input
+ cat input
<COMMIT-A> <COMMIT-A>r<COMMIT-A>s<COMMIT-A>/<COMMIT-A>h<COMMIT-A>s<COMMIT-A>/<COMMIT-A>m<COMMIT-A>i<COMMIT-A>n<COMMIT-A>
+ sed -e s/[0-9a-f]*/<COMMIT-B>/g
+ mv input.tmp input
+ cat input
<COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B> <COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B>r<COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B>s<COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B>/<COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B>h<COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B>s<COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B>/<COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B>m<COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B>i<COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B>n<COMMIT-B><<COMMIT-B>C<COMMIT-B>O<COMMIT-B>M<COMMIT-B>M<COMMIT-B>I<COMMIT-B>T<COMMIT-B>-<COMMIT-B>A<COMMIT-B>><COMMIT-B> [...]

etc. I.e. it's the sed expression itself that's the issue. I.e. you should be able to reproduce this locally with something like:

    echo 0 | sed -e 's/[0-9]*/<BEGIN>0<END>/g' -e 's/[0-9]*/<BEGIN>0<END>/g' -e 's/[0-9]*/<BEGIN>0<END>/g' -e 's/[0-9]*/<BEGIN>0<END>/g' -e 's/[0-9]*/<BEGIN>0<END>/g' -e 's/[0-9]*/<BEGIN>0<END>/g' -e 's/[0-9]*/<BEGIN>0<END>/g' -e 's/[0-9]*/<BEGIN>0<END>/g'

If not just copy the -e a few more times.

Anyway, looking at this whole test file with fresh eyes this pattern seems very strange. You duplicated most of test_commit with this test_commit_setvar. It's a bit more verbosity but why not just use:

    test_commit ...
    A=$(git rev-parse HEAD)

Or teach test_commit a --rev-parse option or something and:

    A=$(test_commit ...)

This make_user_friendly_and_stable_output then actually loses information, e.g. sometimes the bundle output you're testing emits trailing spaces, but the normalization function overzelously trims that.

I think this whole thing would be much simpler with the above and then something like:

@@ -146,7 +126,8 @@ test_expect_success 'setup' '

# branch main: merge commit I & J
git checkout main &&
- test_commit_setvar --merge I topic/1 "Merge commit I" &&
+ git merge --no-edit --no-ff -m"Merge commit I" topic/1 &&
+ I=$(git rev-parse HEAD) &&
test_commit_setvar --merge J refs/pull/2/head "Merge commit J" &&

# branch main: commit K
@@ -172,18 +153,18 @@ test_expect_success 'create bundle from special rev: main^!' '

git bundle list-heads special-rev.bdl |
make_user_friendly_and_stable_output >actual &&
- cat >expect <<-\EOF &&
- <COMMIT-P> refs/heads/main
+ cat >expect <<-EOF &&
+ $P refs/heads/main
EOF
test_cmp expect actual &&

Or just add a --merge option to test_commit itself.

#21 lore raw

Re: Runaway sed memory use in test on older sed+glibc (was "Re: [PATCH v6 1/3] test: add helper functions for git-bundle")

Quoted 10 lines · Ævar Arnfjörð Bjarmason

Anyway, looking at this whole test file with fresh eyes this pattern seems very strange. You duplicated most of test_commit with this test_commit_setvar. It's a bit more verbosity but why not just use:

    test_commit ...
    A=$(git rev-parse HEAD)

Or teach test_commit a --rev-parse option or something and:

    A=$(test_commit ...)

The latter would be nice (and maybe even could be the default, as the stdout is usually just going to --verbose output anyway). But I don't think it works, because the $() is a subshell, so we lose the side effect of test_tick incrementing the timestamp counter.

I wasn't following this thread carefully, but your suggestion to just do "A=$(git rev-parse HEAD)" seems quite readable to me.

-Peff

#22 lore raw

Re: Runaway sed memory use in test on older sed+glibc (was "Re: [PATCH v6 1/3] test: add helper functions for git-bundle")

In reply to Ævar Arnfjörð Bjarmason
Quoted 73 lines · Ævar Arnfjörð Bjarmason
Quoted 70 lines · Jiang Xin

Ævar Arnfjörð Bjarmason <[email protected]> 于2021年5月27日周四

Quoted 67 lines · 上午2:51写道:
Quoted 27 lines · Jiang Xin

From: Jiang Xin <[email protected]>

Move git-bundle related functions from t5510 to a library, and this lib
will be shared with a new testcase t6020 which finds a known breakage of
"git-bundle".
[...]

(quoted diff) +2 -0
+
+# Format the output of git commands to make a user-friendly and

stable
+# text.  We can easily prepare the expect text without having to worry

(quoted diff) +14 -0
+# about future changes of the commit ID and spaces of the output.
+make_user_friendly_and_stable_output () {
+     sed \
+             -e "s/${A%${A#???????}}[0-9a-f]*/<COMMIT-A>/g" \
+             -e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" \
+             -e "s/${C%${C#???????}}[0-9a-f]*/<COMMIT-C>/g" \
+             -e "s/${D%${D#???????}}[0-9a-f]*/<COMMIT-D>/g" \
+             -e "s/${E%${E#???????}}[0-9a-f]*/<COMMIT-E>/g" \
+             -e "s/${F%${F#???????}}[0-9a-f]*/<COMMIT-F>/g" \
+             -e "s/${G%${G#???????}}[0-9a-f]*/<COMMIT-G>/g" \
+             -e "s/${H%${H#???????}}[0-9a-f]*/<COMMIT-H>/g" \
+             -e "s/${I%${I#???????}}[0-9a-f]*/<COMMIT-I>/g" \
+             -e "s/${J%${J#???????}}[0-9a-f]*/<COMMIT-J>/g" \
+             -e "s/${K%${K#???????}}[0-9a-f]*/<COMMIT-K>/g" \

Review

Quoted 10 lines
(quoted diff) +10 -0
+             -e "s/${L%${L#???????}}[0-9a-f]*/<COMMIT-L>/g" \
+             -e "s/${M%${M#???????}}[0-9a-f]*/<COMMIT-M>/g" \
+             -e "s/${N%${N#???????}}[0-9a-f]*/<COMMIT-N>/g" \
+             -e "s/${O%${O#???????}}[0-9a-f]*/<COMMIT-O>/g" \
+             -e "s/${P%${P#???????}}[0-9a-f]*/<COMMIT-P>/g" \
+             -e "s/${TAG1%${TAG1#???????}}[0-9a-f]*/<TAG-1>/g" \
+             -e "s/${TAG2%${TAG2#???????}}[0-9a-f]*/<TAG-2>/g" \
+             -e "s/${TAG3%${TAG3#???????}}[0-9a-f]*/<TAG-3>/g" \
+             -e "s/ *\$//"
+}

On one of the gcc farm boxes, a i386 box (gcc45) this fails because sed
gets killed after >500MB of memory use (I was just eyeballing it in htop) on the "reate bundle from special rev: main^!" test. This with GNU
sed 4.2.2.

I suspect this regex pattern creates some runaway behavior in sed that's
since been fixed (or maybe it's the glibc regex engine?). The glibc is 2.19-18+deb8u10:

    + git bundle list-heads special-rev.bdl
    + make_user_friendly_and_stable_output
    + sed -e s/[0-9a-f]*/<COMMIT-A>/g -e s/[0-9a-f]*/<COMMIT-B>/g -e s/[0-9a-f]*/<COMMIT-C>/g -e s/[0-9a-f]*/<COMMIT-D>/g -e s/[0-9a-f]*/<COMMIT-E>/g -e s/[0-9a-f]*/<COMMIT-F>/g -e s/[0-9a-f]*/<COMMIT-G>/g -e s/[0-9a-f]*/<COMMIT-H>/g -e s/[0-9a-f]*/<COMMIT-I>/g -e s/[0-9a-f]*/<COMMIT-J>/g -e s/[0-9a-f]*/<COMMIT-K>/g -e s/[0-9a-f]*/<COMMIT-L>/g -e s/[0-9a-f]*/<COMMIT-M>/g -e s/[0-9a-f]*/<COMMIT-N>/g -e s/[0-9a-f]*/<COMMIT-O>/g -e s/[0-9a-f]*/<COMMIT-P>/g -e s/[0-9a-f]*/<TAG-1>/g -e s/[0-9a-f]*/<TAG-2>/g -e s/[0-9a-f]*/<TAG-3>/g -e s/ *$//
    sed: couldn't re-allocate memory

Quoted 6 lines
Quoted 4 lines

I wrote a program on macOS to check memory footprint for sed and perl. See:

    https://github.com/jiangxin/compare-sed-perl

Interesting use of Go for as a /usr/bin/time -v replacement :)

Here's a Ruby version:
https://dpaste.com/FYT2QKHJE

I'm not sure if will be useful in this particular case, but Ruby code always ends up simpler ;)

Signature · 1 line
Felipe Contreras
author
#23 lore raw

Re: Runaway sed memory use in test on older sed+glibc (was "Re: [PATCH v6 1/3] test: add helper functions for git-bundle")

Quoted 7 lines · Felipe Contreras <[email protected]> 于2021年5月28日周五 上午3:19写道:
Quoted 5 lines · Ævar Arnfjörð Bjarmason
Quoted 3 lines · Jiang Xin

I wrote a program on macOS to check memory footprint for sed and perl. See:

Quoted 6 lines
Quoted 3 lines
Quoted 1 line

https://github.com/jiangxin/compare-sed-perl

Interesting use of Go for as a /usr/bin/time -v replacement :)

Here's a Ruby version:
https://dpaste.com/FYT2QKHJE

Nice, it's much simpler.

Quoted 5 lines

I'm not sure if will be useful in this particular case, but Ruby code always ends up simpler ;)

--
Felipe Contreras

author
#24 lore raw

Re: Runaway sed memory use in test on older sed+glibc (was "Re: [PATCH v6 1/3] test: add helper functions for git-bundle")

In reply to Ævar Arnfjörð Bjarmason
Quoted 5 lines · Ævar Arnfjörð Bjarmason <[email protected]> 于2021年5月27日周四 下午8:49写道:

But no, the issue as it turns out is not Perl v.s. Sed, it's that there's some bug in the shellscript / tooling version (happens with both dash 0.5.7-4 and bash 4.3-11+deb8u2 on that box) where those expansions like ${A%${A#??????0?}} resolve to nothing.

That's the root cause. It can be reproduced by running the following test script:

```
#!/bin/sh
# test script: test.sh

test_commit_setvar () {
        var=$1 &&
        oid=1234567890123456789012345678901234567890 &&
        eval $var=$oid
}

test_commit_setvar A
echo "A: $A"
echo "Abbrev of A: ${A%${A#???????}}"
```

By running different version of dash, we can see that dash 0.5.7 fail the test:

```
$ /opt/dash/0.5.11/bin/dash test.sh
A: 1234567890123456789012345678901234567890
Abbrev of A: 1234567

$ /opt/dash/0.5.7/bin/dash test.sh
A: 1234567890123456789012345678901234567890
Abbrev of A:
```

This issue can be fixed using the following example:

```
#!/bin/sh

test_commit_setvar () {
        var=$1 &&
        oid=1234567890123456789012345678901234567890 &&
        suffix=${oid#???????} &&
        oid=${oid%$suffix} &&
        eval $var=$oid
}

test_commit_setvar A
echo "Abbrev of A: $A"
```
Quoted 6 lines

Anyway, looking at this whole test file with fresh eyes this pattern seems very strange. You duplicated most of test_commit with this test_commit_setvar. It's a bit more verbosity but why not just use:

    test_commit ...
    A=$(git rev-parse HEAD)

The function "test_commit()" in "test-lib-function.sh" always creates tags and it cannot make merge commit. So I rewrite a new function which reuse the scaffold of "test_commit".

BTW, sorry for the late reply, will send patch later.

--
Jiang Xin

#25 lore raw

Re: Runaway sed memory use in test on older sed+glibc (was "Re: [PATCH v6 1/3] test: add helper functions for git-bundle")

Quoted 39 lines · Jiang Xin
Quoted 5 lines · Ævar Arnfjörð Bjarmason <[email protected]> 于2021年5月27日周四 下午8:49写道:

But no, the issue as it turns out is not Perl v.s. Sed, it's that there's some bug in the shellscript / tooling version (happens with both dash 0.5.7-4 and bash 4.3-11+deb8u2 on that box) where those expansions like ${A%${A#??????0?}} resolve to nothing.

That's the root cause. It can be reproduced by running the following test script:

```
#!/bin/sh
# test script: test.sh

test_commit_setvar () {
        var=$1 &&
        oid=1234567890123456789012345678901234567890 &&
        eval $var=$oid
}

test_commit_setvar A
echo "A: $A"
echo "Abbrev of A: ${A%${A#???????}}"
```

By running different version of dash, we can see that dash 0.5.7 fail the test:

```
$ /opt/dash/0.5.11/bin/dash test.sh
A: 1234567890123456789012345678901234567890
Abbrev of A: 1234567

$ /opt/dash/0.5.7/bin/dash test.sh
A: 1234567890123456789012345678901234567890
Abbrev of A:
```

This issue can be fixed using the following example:

Quoted 14 lines
```
#!/bin/sh

test_commit_setvar () {
        var=$1 &&
        oid=1234567890123456789012345678901234567890 &&
        suffix=${oid#???????} &&
        oid=${oid%$suffix} &&
        eval $var=$oid
}

test_commit_setvar A
echo "Abbrev of A: $A"
```

*nod*

Quoted 10 lines
Quoted 6 lines

Anyway, looking at this whole test file with fresh eyes this pattern seems very strange. You duplicated most of test_commit with this test_commit_setvar. It's a bit more verbosity but why not just use:

    test_commit ...
    A=$(git rev-parse HEAD)

The function "test_commit()" in "test-lib-function.sh" always creates tags and it cannot make merge commit. So I rewrite a new function which reuse the scaffold of "test_commit".

It's had a --no-tag since 3803a3a099 (t: add --no-tag option to test_commit, 2021-02-09). I also have patches in "next" to add more options, you can just add more, having a --merge and maybe a way to tell it to eval the rev-parse into a given variable seem like sensible additions.

Quoted 1 line

BTW, sorry for the late reply, will send patch later.

My main point was that looking at this I think it's very much over the complexity v.s. benefit line on the "complexity" side.

Even if there wasn't a --no-tag just using "test_commit" with a "git tag -d" and "commit_X=$(git rev-parse HEAD)" is less magical and more readable.

I.e. the mostly copy/pasted from test-lib-functions.sh function is ~70 lines, the whole setup function is 50 lines.

And as I noted with the whitespace getting lost in the munging the end result is actually less reliable than just doing a test_cmp with $(git rev-parse ...) instead of <COMMIT-XYZ>.

If you were trying to avoid the whitespace warnings then see the "'s/Z$//'" pattern in t0000-basic.sh for how we've usually tested that, i.e. had a "Z" at the end mark intentional whitespace for test_cmp-alike.

There's a big value in the test suite being mostly consistent (which it somewhat isn't, but we're hopefully getting there). I.e. the goal isn't to optimize each test file to be as small as possible, but to e.g. have the next person maintaining it not wondering where <COMMIT-P> comes from, understanding some test_commit-alike that eval's variables into existence, how it's subtly different (if at all) from test_commit etc.

author
#26 lore raw

Re: Runaway sed memory use in test on older sed+glibc (was "Re: [PATCH v6 1/3] test: add helper functions for git-bundle")

Quoted 98 lines · Ævar Arnfjörð Bjarmason <[email protected]> 于2021年6月1日周二 下午8:04写道:
Quoted 39 lines · Jiang Xin
Quoted 5 lines · Ævar Arnfjörð Bjarmason <[email protected]> 于2021年5月27日周四 下午8:49写道:

But no, the issue as it turns out is not Perl v.s. Sed, it's that there's some bug in the shellscript / tooling version (happens with both dash 0.5.7-4 and bash 4.3-11+deb8u2 on that box) where those expansions like ${A%${A#??????0?}} resolve to nothing.

That's the root cause. It can be reproduced by running the following test script:

```
#!/bin/sh
# test script: test.sh

test_commit_setvar () {
        var=$1 &&
        oid=1234567890123456789012345678901234567890 &&
        eval $var=$oid
}

test_commit_setvar A
echo "A: $A"
echo "Abbrev of A: ${A%${A#???????}}"
```

By running different version of dash, we can see that dash 0.5.7 fail the test:

```
$ /opt/dash/0.5.11/bin/dash test.sh
A: 1234567890123456789012345678901234567890
Abbrev of A: 1234567

$ /opt/dash/0.5.7/bin/dash test.sh
A: 1234567890123456789012345678901234567890
Abbrev of A:
```

This issue can be fixed using the following example:

Quoted 14 lines
```
#!/bin/sh

test_commit_setvar () {
        var=$1 &&
        oid=1234567890123456789012345678901234567890 &&
        suffix=${oid#???????} &&
        oid=${oid%$suffix} &&
        eval $var=$oid
}

test_commit_setvar A
echo "Abbrev of A: $A"
```

*nod*

Quoted 10 lines
Quoted 6 lines

Anyway, looking at this whole test file with fresh eyes this pattern seems very strange. You duplicated most of test_commit with this test_commit_setvar. It's a bit more verbosity but why not just use:

    test_commit ...
    A=$(git rev-parse HEAD)

The function "test_commit()" in "test-lib-function.sh" always creates tags and it cannot make merge commit. So I rewrite a new function which reuse the scaffold of "test_commit".

It's had a --no-tag since 3803a3a099 (t: add --no-tag option to test_commit, 2021-02-09). I also have patches in "next" to add more options, you can just add more, having a --merge and maybe a way to tell it to eval the rev-parse into a given variable seem like sensible additions.

Quoted 1 line

BTW, sorry for the late reply, will send patch later.

My main point was that looking at this I think it's very much over the complexity v.s. benefit line on the "complexity" side.

Even if there wasn't a --no-tag just using "test_commit" with a "git tag -d" and "commit_X=$(git rev-parse HEAD)" is less magical and more readable.

I.e. the mostly copy/pasted from test-lib-functions.sh function is ~70 lines, the whole setup function is 50 lines.

And as I noted with the whitespace getting lost in the munging the end result is actually less reliable than just doing a test_cmp with $(git rev-parse ...) instead of <COMMIT-XYZ>.

If you were trying to avoid the whitespace warnings then see the "'s/Z$//'" pattern in t0000-basic.sh for how we've usually tested that, i.e. had a "Z" at the end mark intentional whitespace for test_cmp-alike.

Quoted 6 lines

There's a big value in the test suite being mostly consistent (which it somewhat isn't, but we're hopefully getting there). I.e. the goal isn't to optimize each test file to be as small as possible, but to e.g. have the next person maintaining it not wondering where <COMMIT-P> comes from, understanding some test_commit-alike that eval's variables into existence, how it's subtly different (if at all) from test_commit etc.

Will send a patch for quick fix for t6020 which is broken on older version of bash.

After changes on "test_commit()" of "test-lib-function.sh" has been merge to master branch, I will try to refactor t6020 again to remove test_commit_setvar() and reuse test_commit().

author
#27 lore raw

PATCH v4 1/2 t6020: fix bash incompatible issue

From: Jiang Xin <[email protected]>

Ævar reported that the function make_user_friendly_and_stable_output() failed on a i386 box (gcc45) in the gcc farm boxes with error:

sed: couldn't re-allocate memory

It turns out that older versions of bash (4.3) or dash (0.5.7) cannot evaluate expression like ${A%${A#???????}} used to get the leading 7 characters of variable A.

Replace the complex expressions so that t6020 works on older version of bash or dash.

---
 t/t6020-bundle-misc.sh | 41 +++++++++++++++++++++--------------------
 1 file changed, 21 insertions(+), 20 deletions(-)
t/t6020-bundle-misc.sh +21 -20
index 881f72fd44..c6a8ea7f76 100755
@@ -77,7 +77,8 @@ test_commit_setvar () {
 		git ${indir:+ -C "$indir"} commit $signoff -m "$1" &&
 		oid=$(git ${indir:+ -C "$indir"} rev-parse HEAD)
 	fi &&
-	eval $var=$oid
+	suffix=${oid#???????} &&
+	eval $var=${oid%$suffix}
 }
 
 # Format the output of git commands to make a user-friendly and stable
@@ -85,25 +86,25 @@ test_commit_setvar () {
 # about future changes of the commit ID and spaces of the output.
 make_user_friendly_and_stable_output () {
 	sed \
-		-e "s/${A%${A#???????}}[0-9a-f]*/<COMMIT-A>/g" \
-		-e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" \
-		-e "s/${C%${C#???????}}[0-9a-f]*/<COMMIT-C>/g" \
-		-e "s/${D%${D#???????}}[0-9a-f]*/<COMMIT-D>/g" \
-		-e "s/${E%${E#???????}}[0-9a-f]*/<COMMIT-E>/g" \
-		-e "s/${F%${F#???????}}[0-9a-f]*/<COMMIT-F>/g" \
-		-e "s/${G%${G#???????}}[0-9a-f]*/<COMMIT-G>/g" \
-		-e "s/${H%${H#???????}}[0-9a-f]*/<COMMIT-H>/g" \
-		-e "s/${I%${I#???????}}[0-9a-f]*/<COMMIT-I>/g" \
-		-e "s/${J%${J#???????}}[0-9a-f]*/<COMMIT-J>/g" \
-		-e "s/${K%${K#???????}}[0-9a-f]*/<COMMIT-K>/g" \
-		-e "s/${L%${L#???????}}[0-9a-f]*/<COMMIT-L>/g" \
-		-e "s/${M%${M#???????}}[0-9a-f]*/<COMMIT-M>/g" \
-		-e "s/${N%${N#???????}}[0-9a-f]*/<COMMIT-N>/g" \
-		-e "s/${O%${O#???????}}[0-9a-f]*/<COMMIT-O>/g" \
-		-e "s/${P%${P#???????}}[0-9a-f]*/<COMMIT-P>/g" \
-		-e "s/${TAG1%${TAG1#???????}}[0-9a-f]*/<TAG-1>/g" \
-		-e "s/${TAG2%${TAG2#???????}}[0-9a-f]*/<TAG-2>/g" \
-		-e "s/${TAG3%${TAG3#???????}}[0-9a-f]*/<TAG-3>/g" \
+		-e "s/$A[0-9a-f]*/<COMMIT-A>/g" \
+		-e "s/$B[0-9a-f]*/<COMMIT-B>/g" \
+		-e "s/$C[0-9a-f]*/<COMMIT-C>/g" \
+		-e "s/$D[0-9a-f]*/<COMMIT-D>/g" \
+		-e "s/$E[0-9a-f]*/<COMMIT-E>/g" \
+		-e "s/$F[0-9a-f]*/<COMMIT-F>/g" \
+		-e "s/$G[0-9a-f]*/<COMMIT-G>/g" \
+		-e "s/$H[0-9a-f]*/<COMMIT-H>/g" \
+		-e "s/$I[0-9a-f]*/<COMMIT-I>/g" \
+		-e "s/$J[0-9a-f]*/<COMMIT-J>/g" \
+		-e "s/$K[0-9a-f]*/<COMMIT-K>/g" \
+		-e "s/$L[0-9a-f]*/<COMMIT-L>/g" \
+		-e "s/$M[0-9a-f]*/<COMMIT-M>/g" \
+		-e "s/$N[0-9a-f]*/<COMMIT-N>/g" \
+		-e "s/$O[0-9a-f]*/<COMMIT-O>/g" \
+		-e "s/$P[0-9a-f]*/<COMMIT-P>/g" \
+		-e "s/$TAG1[0-9a-f]*/<TAG-1>/g" \
+		-e "s/$TAG2[0-9a-f]*/<TAG-2>/g" \
+		-e "s/$TAG3[0-9a-f]*/<TAG-3>/g" \
 		-e "s/ *\$//"
 }
 
Signature · 1 line
2.32.0.rc0
author
#28 lore raw

PATCH v4 2/2 t6020: do not mangle trailing spaces in output

In reply to Jiang Xin

From: Jiang Xin <[email protected]>

We used to call make_user_friendly_and_stable_output to mangle trailing spaces in output before comparing with the expect file. Ævar recommends generating expect file using pattern "'s/Z$//'" to compare expect file with raw output.

---
 t/t6020-bundle-misc.sh | 89 ++++++++++++++++++++++--------------------
 1 file changed, 46 insertions(+), 43 deletions(-)
t/t6020-bundle-misc.sh +46 -43
index c6a8ea7f76..7ee43fb1df 100755
@@ -83,7 +83,7 @@ test_commit_setvar () {
 
 # Format the output of git commands to make a user-friendly and stable
 # text.  We can easily prepare the expect text without having to worry
-# about future changes of the commit ID and spaces of the output.
+# about future changes of the commit ID.
 make_user_friendly_and_stable_output () {
 	sed \
 		-e "s/$A[0-9a-f]*/<COMMIT-A>/g" \
@@ -104,8 +104,11 @@ make_user_friendly_and_stable_output () {
 		-e "s/$P[0-9a-f]*/<COMMIT-P>/g" \
 		-e "s/$TAG1[0-9a-f]*/<TAG-1>/g" \
 		-e "s/$TAG2[0-9a-f]*/<TAG-2>/g" \
-		-e "s/$TAG3[0-9a-f]*/<TAG-3>/g" \
-		-e "s/ *\$//"
+		-e "s/$TAG3[0-9a-f]*/<TAG-3>/g"
+}
+
+format_and_save_expect () {
+	sed -e 's/^> //' -e 's/Z$//' >expect
 }
 
 #            (C)   (D, pull/1/head, topic/1)
@@ -180,11 +183,11 @@ test_expect_success 'create bundle from special rev: main^!' '
 
 	git bundle verify special-rev.bdl |
 		make_user_friendly_and_stable_output >actual &&
-	cat >expect <<-\EOF &&
-	The bundle contains this ref:
-	<COMMIT-P> refs/heads/main
-	The bundle requires this ref:
-	<COMMIT-O>
+	format_and_save_expect <<-\EOF &&
+	> The bundle contains this ref:
+	> <COMMIT-P> refs/heads/main
+	> The bundle requires this ref:
+	> <COMMIT-O> Z
 	EOF
 	test_cmp expect actual &&
 
@@ -201,12 +204,12 @@ test_expect_success 'create bundle with --max-count option' '
 
 	git bundle verify max-count.bdl |
 		make_user_friendly_and_stable_output >actual &&
-	cat >expect <<-\EOF &&
-	The bundle contains these 2 refs:
-	<COMMIT-P> refs/heads/main
-	<TAG-1> refs/tags/v1
-	The bundle requires this ref:
-	<COMMIT-O>
+	format_and_save_expect <<-\EOF &&
+	> The bundle contains these 2 refs:
+	> <COMMIT-P> refs/heads/main
+	> <TAG-1> refs/tags/v1
+	> The bundle requires this ref:
+	> <COMMIT-O> Z
 	EOF
 	test_cmp expect actual &&
 
@@ -226,16 +229,16 @@ test_expect_success 'create bundle with --since option' '
 
 	git bundle verify since.bdl |
 		make_user_friendly_and_stable_output >actual &&
-	cat >expect <<-\EOF &&
-	The bundle contains these 5 refs:
-	<COMMIT-P> refs/heads/main
-	<COMMIT-N> refs/heads/release
-	<TAG-2> refs/tags/v2
-	<TAG-3> refs/tags/v3
-	<COMMIT-P> HEAD
-	The bundle requires these 2 refs:
-	<COMMIT-M>
-	<COMMIT-K>
+	format_and_save_expect <<-\EOF &&
+	> The bundle contains these 5 refs:
+	> <COMMIT-P> refs/heads/main
+	> <COMMIT-N> refs/heads/release
+	> <TAG-2> refs/tags/v2
+	> <TAG-3> refs/tags/v3
+	> <COMMIT-P> HEAD
+	> The bundle requires these 2 refs:
+	> <COMMIT-M> Z
+	> <COMMIT-K> Z
 	EOF
 	test_cmp expect actual &&
 
@@ -294,13 +297,13 @@ test_expect_success 'create bundle 2 - has prerequisites' '
 		--stdin \
 		release <input &&
 
-	cat >expect <<-\EOF &&
-	The bundle contains this ref:
-	<COMMIT-N> refs/heads/release
-	The bundle requires these 3 refs:
-	<COMMIT-D>
-	<COMMIT-E>
-	<COMMIT-G>
+	format_and_save_expect <<-\EOF &&
+	> The bundle contains this ref:
+	> <COMMIT-N> refs/heads/release
+	> The bundle requires these 3 refs:
+	> <COMMIT-D> Z
+	> <COMMIT-E> Z
+	> <COMMIT-G> Z
 	EOF
 
 	git bundle verify 2.bdl |
@@ -318,11 +321,11 @@ test_expect_success 'create bundle 2 - has prerequisites' '
 test_expect_success 'fail to verify bundle without prerequisites' '
 	git init --bare test1.git &&
 
-	cat >expect <<-\EOF &&
-	error: Repository lacks these prerequisite commits:
-	error: <COMMIT-D>
-	error: <COMMIT-E>
-	error: <COMMIT-G>
+	format_and_save_expect <<-\EOF &&
+	> error: Repository lacks these prerequisite commits:
+	> error: <COMMIT-D> Z
+	> error: <COMMIT-E> Z
+	> error: <COMMIT-G> Z
 	EOF
 
 	test_must_fail git -C test1.git bundle verify ../2.bdl 2>&1 |
@@ -353,13 +356,13 @@ test_expect_success 'create bundle 3 - two refs, same object' '
 		--stdin \
 		main HEAD <input &&
 
-	cat >expect <<-\EOF &&
-	The bundle contains these 2 refs:
-	<COMMIT-P> refs/heads/main
-	<COMMIT-P> HEAD
-	The bundle requires these 2 refs:
-	<COMMIT-M>
-	<COMMIT-K>
+	format_and_save_expect <<-\EOF &&
+	> The bundle contains these 2 refs:
+	> <COMMIT-P> refs/heads/main
+	> <COMMIT-P> HEAD
+	> The bundle requires these 2 refs:
+	> <COMMIT-M> Z
+	> <COMMIT-K> Z
 	EOF
 
 	git bundle verify 3.bdl |
Signature · 1 line
2.32.0.rc0
#29 lore raw

Re: [PATCH 2/2] t6020: do not mangle trailing spaces in output

Quoted 6 lines · Jiang Xin

From: Jiang Xin <[email protected]>

We used to call make_user_friendly_and_stable_output to mangle trailing spaces in output before comparing with the expect file. Ævar recommends generating expect file using pattern "'s/Z$//'" to compare expect file with raw output.

I've tested both of these patches and they fix the reported issue on that gcc45 machine.

Quoted 39 lines
---
 t/t6020-bundle-misc.sh | 89 ++++++++++++++++++++++--------------------
 1 file changed, 46 insertions(+), 43 deletions(-)
t/t6020-bundle-misc.sh +6 -5
index c6a8ea7f76..7ee43fb1df 100755
@@ -83,7 +83,7 @@ test_commit_setvar () {
 
 # Format the output of git commands to make a user-friendly and stable
 # text.  We can easily prepare the expect text without having to worry
-# about future changes of the commit ID and spaces of the output.
+# about future changes of the commit ID.
 make_user_friendly_and_stable_output () {
 	sed \
 		-e "s/$A[0-9a-f]*/<COMMIT-A>/g" \
@@ -104,8 +104,11 @@ make_user_friendly_and_stable_output () {
 		-e "s/$P[0-9a-f]*/<COMMIT-P>/g" \
 		-e "s/$TAG1[0-9a-f]*/<TAG-1>/g" \
 		-e "s/$TAG2[0-9a-f]*/<TAG-2>/g" \
-		-e "s/$TAG3[0-9a-f]*/<TAG-3>/g" \
-		-e "s/ *\$//"
+		-e "s/$TAG3[0-9a-f]*/<TAG-3>/g"
+}
+
+format_and_save_expect () {
+	sed -e 's/^> //' -e 's/Z$//' >expect
 }
 
 #            (C)   (D, pull/1/head, topic/1)
@@ -180,11 +183,11 @@ test_expect_success 'create bundle from special rev: main^!' '
 
 	git bundle verify special-rev.bdl |
 		make_user_friendly_and_stable_output >actual &&
-	cat >expect <<-\EOF &&
-	The bundle contains this ref:

t/t6020-bundle-misc.sh test_expect_success 'create bundle from special rev: main^!' ' · line 192

Quoted 10 lines
t/t6020-bundle-misc.sh +5 -3
-	<COMMIT-P> refs/heads/main
-	The bundle requires this ref:
-	<COMMIT-O>
+	format_and_save_expect <<-\EOF &&
+	> The bundle contains this ref:
+	> <COMMIT-P> refs/heads/main
+	> The bundle requires this ref:
+	> <COMMIT-O> Z
 	EOF
 	test_cmp expect actual &&

I think for this test it would be better to just have the "Z" suffix and leave off prefixing all the lines with ">". That's done in t0000.sh because we have various empty lines, but here that's not the case.

I don't think that nit is worth a re-roll, and as you noted you're doing some larger changes to these tests in follow-up patches.

It's just convenient to have the test_cmp expect input as close to copy/paste-able as possible.

author
#30 lore raw

v2 cover Fixed t6020 bash compatible issue and fixed wrong sideband suffix issue

From: Jiang Xin <[email protected]>

In addition to fix the bash incompatible issue of t6020, find another issue when try to rewrite t5411 to compare raw command output.

Function "demultiplex_sideband()" will try to split the sideband-2 message by line breaks, and append a suffix to each nonempty line to clear the end of the screen line. But in the following example, there will be no suffix (8 spaces) for "<message-3>":

        PKT-LINE(\2 <message-1> CR <message-2> CR <message-3>)
        PKT-LINE(\2 CR <message-4> CR <message-5> CR)

This is because the line break of "<message-3>" is placed in the next pktline message.

With this fix, we could rewrite t5411 to test raw output of "git push" which has a stable suffix for each remote sideband-2 message.

Jiang Xin (4):
t6020: fix bash incompatible issue
test: refactor create_commits_in() for t5411 and t5548
sideband: append suffix for message whose CR in next pktline
test: compare raw output, not mangle tabs and spaces

 sideband.c                                    |   4 +
 t/t5411/common-functions.sh                   |  54 +++--
 t/t5411/test-0000-standard-git-push.sh        |  82 +++----
 .../test-0001-standard-git-push--porcelain.sh |  90 ++++----
 ...st-0003-pre-receive-declined--porcelain.sh |   8 +-
 t/t5411/test-0011-no-hook-error.sh            |  40 ++--
 t/t5411/test-0012-no-hook-error--porcelain.sh |  42 ++--
 t/t5411/test-0013-bad-protocol.sh             |  62 +++---
 t/t5411/test-0014-bad-protocol--porcelain.sh  |  80 +++----
 t/t5411/test-0020-report-ng.sh                |  32 +--
 t/t5411/test-0021-report-ng--porcelain.sh     |  36 ++--
 t/t5411/test-0022-report-unexpect-ref.sh      |  26 +--
 ...est-0023-report-unexpect-ref--porcelain.sh |  28 +--
 t/t5411/test-0024-report-unknown-ref.sh       |  18 +-
 ...test-0025-report-unknown-ref--porcelain.sh |  20 +-
 t/t5411/test-0026-push-options.sh             |  58 ++---
 t/t5411/test-0027-push-options--porcelain.sh  |  62 +++---
 t/t5411/test-0030-report-ok.sh                |  20 +-
 t/t5411/test-0031-report-ok--porcelain.sh     |  22 +-
 t/t5411/test-0032-report-with-options.sh      | 186 ++++++++--------
 ...est-0033-report-with-options--porcelain.sh | 200 +++++++++---------
 t/t5411/test-0034-report-ft.sh                |  22 +-
 t/t5411/test-0035-report-ft--porcelain.sh     |  24 +--
 ...t-0036-report-multi-rewrite-for-one-ref.sh | 132 ++++++------
 ...rt-multi-rewrite-for-one-ref--porcelain.sh | 138 ++++++------
 t/t5411/test-0038-report-mixed-refs.sh        |  74 +++----
 .../test-0039-report-mixed-refs--porcelain.sh |  76 +++----
 t/t5411/test-0040-process-all-refs.sh         |  80 +++----
 .../test-0041-process-all-refs--porcelain.sh  |  82 +++----
 ...t-0050-proc-receive-refs-with-modifiers.sh |  90 ++++----
 t/t5548-push-porcelain.sh                     |  97 +++++----
 t/t6020-bundle-misc.sh                        |  93 ++++----
 32 files changed, 1047 insertions(+), 1031 deletions(-)
Signature · 1 line
2.32.0.rc0.27.g7b1e85181b
#31 lore raw

Re: [PATCH v2 0/4] Fixed t6020 bash compatible issue and fixed wrong sideband suffix issue

Quoted 4 lines · Jiang Xin

From: Jiang Xin <[email protected]>

In addition to fix the bash incompatible issue of t6020, find another issue when try to rewrite t5411 to compare raw command output.

Do the three later patches depend on the t6020 fix, or is this made a 4-patch series only for the convenience of sending them out?

It's not like get_abbrev_oid() used in t6020 is defined in a common part of the test library and later used by other tests (instead, the patches duplicate this helper function into yet two more files).

author
#32 lore raw

Re: [PATCH v2 0/4] Fixed t6020 bash compatible issue and fixed wrong sideband suffix issue

Quoted 8 lines · Junio C Hamano <[email protected]> 于2021年6月14日周一 下午12:10写道:
Quoted 4 lines · Jiang Xin

From: Jiang Xin <[email protected]>

In addition to fix the bash incompatible issue of t6020, find another issue when try to rewrite t5411 to compare raw command output.

Quoted 7 lines

Do the three later patches depend on the t6020 fix, or is this made a 4-patch series only for the convenience of sending them out?

It's not like get_abbrev_oid() used in t6020 is defined in a common part of the test library and later used by other tests (instead, the patches duplicate this helper function into yet two more files).

Will split it into two patch series. One will fix bash incompatible parameter expansion in t6020, another will fix clear-to-eol at packet boundary issue of sideband and try to test raw output in t6020, t5548 and t5411.

I also queue another patch series, which add "--bare" support to "test_create_repo", and replace "git init" command in test cases to adapt to variable GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME.

--
Jiang Xin

author
#33 lore raw

PATCH v3 1/1 t6020: fix incompatible parameter expansion

Ævar reported that the function make_user_friendly_and_stable_output() failed on a i386 box (gcc45) in the gcc farm boxes with error:

sed: couldn't re-allocate memory

It turns out that older versions of bash (4.3) or dash (0.5.7) cannot evaluate expression like ${A%${A#???????}} used to get the leading 7 characters of variable A.

Replace the incompatible parameter expansion so that t6020 works on older version of bash or dash.

---
 t/t6020-bundle-misc.sh | 50 ++++++++++++++++++++++++++----------------
 1 file changed, 31 insertions(+), 19 deletions(-)
t/t6020-bundle-misc.sh +31 -19
index 881f72fd44..3140ca4fdc 100755
@@ -80,30 +80,42 @@ test_commit_setvar () {
 	eval $var=$oid
 }
 
+get_abbrev_oid () {
+	oid=$1 &&
+	suffix=${oid#???????} &&
+	oid=${oid%$suffix} &&
+	if test -n "$oid"
+	then
+		echo "$oid"
+	else
+		echo "undefined-oid"
+	fi
+}
+
 # Format the output of git commands to make a user-friendly and stable
 # text.  We can easily prepare the expect text without having to worry
 # about future changes of the commit ID and spaces of the output.
 make_user_friendly_and_stable_output () {
 	sed \
-		-e "s/${A%${A#???????}}[0-9a-f]*/<COMMIT-A>/g" \
-		-e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" \
-		-e "s/${C%${C#???????}}[0-9a-f]*/<COMMIT-C>/g" \
-		-e "s/${D%${D#???????}}[0-9a-f]*/<COMMIT-D>/g" \
-		-e "s/${E%${E#???????}}[0-9a-f]*/<COMMIT-E>/g" \
-		-e "s/${F%${F#???????}}[0-9a-f]*/<COMMIT-F>/g" \
-		-e "s/${G%${G#???????}}[0-9a-f]*/<COMMIT-G>/g" \
-		-e "s/${H%${H#???????}}[0-9a-f]*/<COMMIT-H>/g" \
-		-e "s/${I%${I#???????}}[0-9a-f]*/<COMMIT-I>/g" \
-		-e "s/${J%${J#???????}}[0-9a-f]*/<COMMIT-J>/g" \
-		-e "s/${K%${K#???????}}[0-9a-f]*/<COMMIT-K>/g" \
-		-e "s/${L%${L#???????}}[0-9a-f]*/<COMMIT-L>/g" \
-		-e "s/${M%${M#???????}}[0-9a-f]*/<COMMIT-M>/g" \
-		-e "s/${N%${N#???????}}[0-9a-f]*/<COMMIT-N>/g" \
-		-e "s/${O%${O#???????}}[0-9a-f]*/<COMMIT-O>/g" \
-		-e "s/${P%${P#???????}}[0-9a-f]*/<COMMIT-P>/g" \
-		-e "s/${TAG1%${TAG1#???????}}[0-9a-f]*/<TAG-1>/g" \
-		-e "s/${TAG2%${TAG2#???????}}[0-9a-f]*/<TAG-2>/g" \
-		-e "s/${TAG3%${TAG3#???????}}[0-9a-f]*/<TAG-3>/g" \
+		-e "s/$(get_abbrev_oid $A)[0-9a-f]*/<COMMIT-A>/g" \
+		-e "s/$(get_abbrev_oid $B)[0-9a-f]*/<COMMIT-B>/g" \
+		-e "s/$(get_abbrev_oid $C)[0-9a-f]*/<COMMIT-C>/g" \
+		-e "s/$(get_abbrev_oid $D)[0-9a-f]*/<COMMIT-D>/g" \
+		-e "s/$(get_abbrev_oid $E)[0-9a-f]*/<COMMIT-E>/g" \
+		-e "s/$(get_abbrev_oid $F)[0-9a-f]*/<COMMIT-F>/g" \
+		-e "s/$(get_abbrev_oid $G)[0-9a-f]*/<COMMIT-G>/g" \
+		-e "s/$(get_abbrev_oid $H)[0-9a-f]*/<COMMIT-H>/g" \
+		-e "s/$(get_abbrev_oid $I)[0-9a-f]*/<COMMIT-I>/g" \
+		-e "s/$(get_abbrev_oid $J)[0-9a-f]*/<COMMIT-J>/g" \
+		-e "s/$(get_abbrev_oid $K)[0-9a-f]*/<COMMIT-K>/g" \
+		-e "s/$(get_abbrev_oid $L)[0-9a-f]*/<COMMIT-L>/g" \
+		-e "s/$(get_abbrev_oid $M)[0-9a-f]*/<COMMIT-M>/g" \
+		-e "s/$(get_abbrev_oid $N)[0-9a-f]*/<COMMIT-N>/g" \
+		-e "s/$(get_abbrev_oid $O)[0-9a-f]*/<COMMIT-O>/g" \
+		-e "s/$(get_abbrev_oid $P)[0-9a-f]*/<COMMIT-P>/g" \
+		-e "s/$(get_abbrev_oid $TAG1)[0-9a-f]*/<TAG-1>/g" \
+		-e "s/$(get_abbrev_oid $TAG2)[0-9a-f]*/<TAG-2>/g" \
+		-e "s/$(get_abbrev_oid $TAG3)[0-9a-f]*/<TAG-3>/g" \
 		-e "s/ *\$//"
 }
 
Signature · 1 line
2.32.0.rc0.27.g7b1e85181b
#34 lore raw

Re: [PATCH v3] t6020: fix incompatible parameter expansion

Quoted 9 lines · Jiang Xin

Ævar reported that the function make_user_friendly_and_stable_output() failed on a i386 box (gcc45) in the gcc farm boxes with error:

sed: couldn't re-allocate memory

It turns out that older versions of bash (4.3) or dash (0.5.7) cannot evaluate expression like ${A%${A#???????}} used to get the leading 7 characters of variable A.

Quoted 6 lines

Replace the incompatible parameter expansion so that t6020 works on older version of bash or dash.

---

For what it's worth I've also tested this v3 on gcc45, it works too.

author
#35 lore raw

PATCH v2 1/4 t6020: fix bash incompatible issue

In reply to Ævar Arnfjörð Bjarmason

From: Jiang Xin <[email protected]>

Ævar reported that the function make_user_friendly_and_stable_output() failed on a i386 box (gcc45) in the gcc farm boxes with error:

sed: couldn't re-allocate memory

It turns out that older versions of bash (4.3) or dash (0.5.7) cannot evaluate expression like ${A%${A#???????}} used to get the leading 7 characters of variable A.

Replace the complex expressions so that t6020 works on older version of bash or dash.

---
 t/t6020-bundle-misc.sh | 50 ++++++++++++++++++++++++++----------------
 1 file changed, 31 insertions(+), 19 deletions(-)
t/t6020-bundle-misc.sh +31 -19
index 881f72fd44..3140ca4fdc 100755
@@ -80,30 +80,42 @@ test_commit_setvar () {
 	eval $var=$oid
 }
 
+get_abbrev_oid () {
+	oid=$1 &&
+	suffix=${oid#???????} &&
+	oid=${oid%$suffix} &&
+	if test -n "$oid"
+	then
+		echo "$oid"
+	else
+		echo "undefined-oid"
+	fi
+}
+
 # Format the output of git commands to make a user-friendly and stable
 # text.  We can easily prepare the expect text without having to worry
 # about future changes of the commit ID and spaces of the output.
 make_user_friendly_and_stable_output () {
 	sed \
-		-e "s/${A%${A#???????}}[0-9a-f]*/<COMMIT-A>/g" \
-		-e "s/${B%${B#???????}}[0-9a-f]*/<COMMIT-B>/g" \
-		-e "s/${C%${C#???????}}[0-9a-f]*/<COMMIT-C>/g" \
-		-e "s/${D%${D#???????}}[0-9a-f]*/<COMMIT-D>/g" \
-		-e "s/${E%${E#???????}}[0-9a-f]*/<COMMIT-E>/g" \
-		-e "s/${F%${F#???????}}[0-9a-f]*/<COMMIT-F>/g" \
-		-e "s/${G%${G#???????}}[0-9a-f]*/<COMMIT-G>/g" \
-		-e "s/${H%${H#???????}}[0-9a-f]*/<COMMIT-H>/g" \
-		-e "s/${I%${I#???????}}[0-9a-f]*/<COMMIT-I>/g" \
-		-e "s/${J%${J#???????}}[0-9a-f]*/<COMMIT-J>/g" \
-		-e "s/${K%${K#???????}}[0-9a-f]*/<COMMIT-K>/g" \
-		-e "s/${L%${L#???????}}[0-9a-f]*/<COMMIT-L>/g" \
-		-e "s/${M%${M#???????}}[0-9a-f]*/<COMMIT-M>/g" \
-		-e "s/${N%${N#???????}}[0-9a-f]*/<COMMIT-N>/g" \
-		-e "s/${O%${O#???????}}[0-9a-f]*/<COMMIT-O>/g" \
-		-e "s/${P%${P#???????}}[0-9a-f]*/<COMMIT-P>/g" \
-		-e "s/${TAG1%${TAG1#???????}}[0-9a-f]*/<TAG-1>/g" \
-		-e "s/${TAG2%${TAG2#???????}}[0-9a-f]*/<TAG-2>/g" \
-		-e "s/${TAG3%${TAG3#???????}}[0-9a-f]*/<TAG-3>/g" \
+		-e "s/$(get_abbrev_oid $A)[0-9a-f]*/<COMMIT-A>/g" \
+		-e "s/$(get_abbrev_oid $B)[0-9a-f]*/<COMMIT-B>/g" \
+		-e "s/$(get_abbrev_oid $C)[0-9a-f]*/<COMMIT-C>/g" \
+		-e "s/$(get_abbrev_oid $D)[0-9a-f]*/<COMMIT-D>/g" \
+		-e "s/$(get_abbrev_oid $E)[0-9a-f]*/<COMMIT-E>/g" \
+		-e "s/$(get_abbrev_oid $F)[0-9a-f]*/<COMMIT-F>/g" \
+		-e "s/$(get_abbrev_oid $G)[0-9a-f]*/<COMMIT-G>/g" \
+		-e "s/$(get_abbrev_oid $H)[0-9a-f]*/<COMMIT-H>/g" \
+		-e "s/$(get_abbrev_oid $I)[0-9a-f]*/<COMMIT-I>/g" \
+		-e "s/$(get_abbrev_oid $J)[0-9a-f]*/<COMMIT-J>/g" \
+		-e "s/$(get_abbrev_oid $K)[0-9a-f]*/<COMMIT-K>/g" \
+		-e "s/$(get_abbrev_oid $L)[0-9a-f]*/<COMMIT-L>/g" \
+		-e "s/$(get_abbrev_oid $M)[0-9a-f]*/<COMMIT-M>/g" \
+		-e "s/$(get_abbrev_oid $N)[0-9a-f]*/<COMMIT-N>/g" \
+		-e "s/$(get_abbrev_oid $O)[0-9a-f]*/<COMMIT-O>/g" \
+		-e "s/$(get_abbrev_oid $P)[0-9a-f]*/<COMMIT-P>/g" \
+		-e "s/$(get_abbrev_oid $TAG1)[0-9a-f]*/<TAG-1>/g" \
+		-e "s/$(get_abbrev_oid $TAG2)[0-9a-f]*/<TAG-2>/g" \
+		-e "s/$(get_abbrev_oid $TAG3)[0-9a-f]*/<TAG-3>/g" \
 		-e "s/ *\$//"
 }
 
Signature · 1 line
2.32.0.rc0.27.g7b1e85181b
author
#36 lore raw

PATCH v2 2/4 test: refactor create_commits_in() for t5411 and t5548

In reply to Ævar Arnfjörð Bjarmason

From: Jiang Xin <[email protected]>

Use "test_commit" in "create_commits_in" to create commit and get abbrev object ID by using function "get_abbrev_oid".

---
 t/t5411/common-functions.sh                   | 45 ++++++++---------
 t/t5411/test-0000-standard-git-push.sh        |  6 +--
 .../test-0001-standard-git-push--porcelain.sh |  6 +--
 t/t5411/test-0022-report-unexpect-ref.sh      |  2 +-
 ...est-0023-report-unexpect-ref--porcelain.sh |  2 +-
 t/t5411/test-0032-report-with-options.sh      |  8 +--
 ...est-0033-report-with-options--porcelain.sh |  8 +--
 ...t-0036-report-multi-rewrite-for-one-ref.sh | 10 ++--
 ...rt-multi-rewrite-for-one-ref--porcelain.sh | 10 ++--
 t/t5411/test-0038-report-mixed-refs.sh        |  4 +-
 .../test-0039-report-mixed-refs--porcelain.sh |  4 +-
 t/t5411/test-0040-process-all-refs.sh         |  8 +--
 .../test-0041-process-all-refs--porcelain.sh  |  8 +--
 ...t-0050-proc-receive-refs-with-modifiers.sh |  4 +-
 t/t5548-push-porcelain.sh                     | 49 +++++++++----------
 15 files changed, 84 insertions(+), 90 deletions(-)
t/t5411/common-functions.sh +21 -24
index 6694858e18..6398f5f2a3 100644
@@ -6,29 +6,28 @@
 # NOTE: Never calling this function from a subshell since variable
 # assignments will disappear when subshell exits.
 create_commits_in () {
-	repo="$1" &&
-	if ! parent=$(git -C "$repo" rev-parse HEAD^{} --)
-	then
-		parent=
-	fi &&
-	T=$(git -C "$repo" write-tree) &&
+	repo="$1" && test -d "$repo" ||
+	error "Repository $repo does not exist."
 	shift &&
 	while test $# -gt 0
 	do
 		name=$1 &&
-		test_tick &&
-		if test -z "$parent"
-		then
-			oid=$(echo $name | git -C "$repo" commit-tree $T)
-		else
-			oid=$(echo $name | git -C "$repo" commit-tree -p $parent $T)
-		fi &&
-		eval $name=$oid &&
-		parent=$oid &&
-		shift ||
-		return 1
-	done &&
-	git -C "$repo" update-ref refs/heads/main $oid
+		shift &&
+		test_commit -C "$repo" --no-tag "$name" &&
+		eval $name=$(git -C "$repo" rev-parse HEAD)
+	done
+}
+
+get_abbrev_oid () {
+	oid=$1 &&
+	suffix=${oid#???????} &&
+	oid=${oid%$suffix} &&
+	if test -n "$oid"
+	then
+		echo "$oid"
+	else
+		echo "undefined-oid"
+	fi
 }
 
 # Format the output of git-push, git-show-ref and other commands to make a
@@ -44,12 +43,10 @@ make_user_friendly_and_stable_output () {
 		-e "s/  */ /g" \
 		-e "s/'/\"/g" \
 		-e "s/	/    /g" \
-		-e "s/$A/<COMMIT-A>/g" \
-		-e "s/$B/<COMMIT-B>/g" \
-		-e "s/$TAG/<TAG-v123>/g" \
+		-e "s/$(get_abbrev_oid $A)[0-9a-f]*/<COMMIT-A>/g" \
+		-e "s/$(get_abbrev_oid $B)[0-9a-f]*/<COMMIT-B>/g" \
+		-e "s/$(get_abbrev_oid $TAG)[0-9a-f]*/<TAG-v123>/g" \
 		-e "s/$ZERO_OID/<ZERO-OID>/g" \
-		-e "s/$(echo $A | cut -c1-7)[0-9a-f]*/<OID-A>/g" \
-		-e "s/$(echo $B | cut -c1-7)[0-9a-f]*/<OID-B>/g" \
 		-e "s#To $URL_PREFIX/upstream.git#To <URL/of/upstream.git>#" \
 		-e "/^error: / d"
 }
t/t5411/test-0000-standard-git-push.sh +3 -3
index e1e0175c12..2c69cf60d4 100644
@@ -15,7 +15,7 @@ test_expect_success "git-push ($PROTOCOL)" '
 	remote: post-receive< <COMMIT-A> <COMMIT-B> refs/heads/main
 	remote: post-receive< <ZERO-OID> <COMMIT-A> refs/heads/next
 	To <URL/of/upstream.git>
-	 <OID-A>..<OID-B> <COMMIT-B> -> main
+	 <COMMIT-A>..<COMMIT-B> <COMMIT-B> -> main
 	 * [new branch] HEAD -> next
 	EOF
 	test_cmp expect actual &&
@@ -69,7 +69,7 @@ test_expect_success "non-fast-forward git-push ($PROTOCOL)" '
 	remote: # post-receive hook
 	remote: post-receive< <COMMIT-A> <COMMIT-B> refs/heads/next
 	To <URL/of/upstream.git>
-	 <OID-A>..<OID-B> <COMMIT-B> -> next
+	 <COMMIT-A>..<COMMIT-B> <COMMIT-B> -> next
 	 ! [rejected] main -> main (non-fast-forward)
 	EOF
 	test_cmp expect actual &&
@@ -106,7 +106,7 @@ test_expect_success "git-push -f ($PROTOCOL)" '
 	remote: post-receive< <ZERO-OID> <COMMIT-A> refs/review/main/topic
 	remote: post-receive< <ZERO-OID> <COMMIT-A> refs/heads/a/b/c
 	To <URL/of/upstream.git>
-	 + <OID-B>...<OID-A> main -> main (forced update)
+	 + <COMMIT-B>...<COMMIT-A> main -> main (forced update)
 	 - [deleted] next
 	 * [new tag] v123 -> v123
 	 * [new reference] main -> refs/review/main/topic
t/t5411/test-0001-standard-git-push--porcelain.sh +3 -3
index bcbda72341..7b982c8395 100644
@@ -15,7 +15,7 @@ test_expect_success "git-push ($PROTOCOL/porcelain)" '
 	remote: post-receive< <COMMIT-A> <COMMIT-B> refs/heads/main
 	remote: post-receive< <ZERO-OID> <COMMIT-A> refs/heads/next
 	To <URL/of/upstream.git>
-	     <COMMIT-B>:refs/heads/main    <OID-A>..<OID-B>
+	     <COMMIT-B>:refs/heads/main    <COMMIT-A>..<COMMIT-B>
 	*    HEAD:refs/heads/next    [new branch]
 	Done
 	EOF
@@ -71,7 +71,7 @@ test_expect_success "non-fast-forward git-push ($PROTOCOL/porcelain)" '
 	remote: # post-receive hook
 	remote: post-receive< <COMMIT-A> <COMMIT-B> refs/heads/next
 	To <URL/of/upstream.git>
-	     <COMMIT-B>:refs/heads/next    <OID-A>..<OID-B>
+	     <COMMIT-B>:refs/heads/next    <COMMIT-A>..<COMMIT-B>
 	!    refs/heads/main:refs/heads/main    [rejected] (non-fast-forward)
 	Done
 	EOF
@@ -109,7 +109,7 @@ test_expect_success "git-push -f ($PROTOCOL/porcelain)" '
 	remote: post-receive< <ZERO-OID> <COMMIT-A> refs/review/main/topic
 	remote: post-receive< <ZERO-OID> <COMMIT-A> refs/heads/a/b/c
 	To <URL/of/upstream.git>
-	+    refs/heads/main:refs/heads/main    <OID-B>...<OID-A> (forced update)
+	+    refs/heads/main:refs/heads/main    <COMMIT-B>...<COMMIT-A> (forced update)
 	-    :refs/heads/next    [deleted]
 	*    refs/tags/v123:refs/tags/v123    [new tag]
 	*    refs/heads/main:refs/review/main/topic    [new reference]
t/t5411/test-0022-report-unexpect-ref.sh +1 -1
index dbed467186..a482ff931a 100644
@@ -26,7 +26,7 @@ test_expect_success "proc-receive: report unexpected ref ($PROTOCOL)" '
 	remote: # post-receive hook
 	remote: post-receive< <COMMIT-A> <COMMIT-B> refs/heads/main
 	To <URL/of/upstream.git>
-	 <OID-A>..<OID-B> <COMMIT-B> -> main
+	 <COMMIT-A>..<COMMIT-B> <COMMIT-B> -> main
 	 ! [remote rejected] HEAD -> refs/for/main/topic (proc-receive failed to report status)
 	EOF
 	test_cmp expect actual &&
t/t5411/test-0023-report-unexpect-ref--porcelain.sh +1 -1
index e89096fa13..c586cd70a0 100644
@@ -26,7 +26,7 @@ test_expect_success "proc-receive: report unexpected ref ($PROTOCOL/porcelain)"
 	remote: # post-receive hook
 	remote: post-receive< <COMMIT-A> <COMMIT-B> refs/heads/main
 	To <URL/of/upstream.git>
-	     <COMMIT-B>:refs/heads/main    <OID-A>..<OID-B>
+	     <COMMIT-B>:refs/heads/main    <COMMIT-A>..<COMMIT-B>
 	!    HEAD:refs/for/main/topic    [remote rejected] (proc-receive failed to report status)
 	Done
 	EOF
t/t5411/test-0032-report-with-options.sh +4 -4
index 437ade012d..bbd09ecdfb 100644
@@ -123,7 +123,7 @@ test_expect_success "proc-receive: report option refname and old-oid ($PROTOCOL)
 	remote: # post-receive hook
 	remote: post-receive< <COMMIT-B> <COMMIT-A> refs/pull/123/head
 	To <URL/of/upstream.git>
-	 <OID-B>..<OID-A> HEAD -> refs/pull/123/head
+	 <COMMIT-B>..<COMMIT-A> HEAD -> refs/pull/123/head
 	EOF
 	test_cmp expect actual
 '
@@ -155,7 +155,7 @@ test_expect_success "proc-receive: report option old-oid ($PROTOCOL)" '
 	remote: # post-receive hook
 	remote: post-receive< <COMMIT-B> <COMMIT-A> refs/for/main/topic
 	To <URL/of/upstream.git>
-	 <OID-B>..<OID-A> HEAD -> refs/for/main/topic
+	 <COMMIT-B>..<COMMIT-A> HEAD -> refs/for/main/topic
 	EOF
 	test_cmp expect actual
 '
@@ -189,7 +189,7 @@ test_expect_success "proc-receive: report option old-oid and new-oid ($PROTOCOL)
 	remote: # post-receive hook
 	remote: post-receive< <COMMIT-A> <COMMIT-B> refs/for/main/topic
 	To <URL/of/upstream.git>
-	 <OID-A>..<OID-B> HEAD -> refs/for/main/topic
+	 <COMMIT-A>..<COMMIT-B> HEAD -> refs/for/main/topic
 	EOF
 	test_cmp expect actual
 '
@@ -243,7 +243,7 @@ test_expect_success "proc-receive: report with multiple rewrites ($PROTOCOL)" '
 	To <URL/of/upstream.git>
 	 * [new reference] HEAD -> refs/pull/123/head
 	 * [new reference] HEAD -> refs/for/a/b/c/topic
-	 + <OID-B>...<OID-A> HEAD -> refs/pull/124/head (forced update)
+	 + <COMMIT-B>...<COMMIT-A> HEAD -> refs/pull/124/head (forced update)
 	EOF
 	test_cmp expect actual &&
 
t/t5411/test-0033-report-with-options--porcelain.sh +4 -4
index 11486720ee..d6a24d60ff 100644
@@ -127,7 +127,7 @@ test_expect_success "proc-receive: report option refname and old-oid ($PROTOCOL/
 	remote: # post-receive hook
 	remote: post-receive< <COMMIT-B> <COMMIT-A> refs/pull/123/head
 	To <URL/of/upstream.git>
-	     HEAD:refs/pull/123/head    <OID-B>..<OID-A>
+	     HEAD:refs/pull/123/head    <COMMIT-B>..<COMMIT-A>
 	Done
 	EOF
 	test_cmp expect actual
@@ -160,7 +160,7 @@ test_expect_success "proc-receive: report option old-oid ($PROTOCOL/porcelain)"
 	remote: # post-receive hook
 	remote: post-receive< <COMMIT-B> <COMMIT-A> refs/for/main/topic
 	To <URL/of/upstream.git>
-	     HEAD:refs/for/main/topic    <OID-B>..<OID-A>
+	     HEAD:refs/for/main/topic    <COMMIT-B>..<COMMIT-A>
 	Done
 	EOF
 	test_cmp expect actual
@@ -195,7 +195,7 @@ test_expect_success "proc-receive: report option old-oid and new-oid ($PROTOCOL/
 	remote: # post-receive hook
 	remote: post-receive< <COMMIT-A> <COMMIT-B> refs/for/main/topic
 	To <URL/of/upstream.git>
-	     HEAD:refs/for/main/topic    <OID-A>..<OID-B>
+	     HEAD:refs/for/main/topic    <COMMIT-A>..<COMMIT-B>
 	Done
 	EOF
 	test_cmp expect actual
@@ -251,7 +251,7 @@ test_expect_success "proc-receive: report with multiple rewrites ($PROTOCOL/porc
 	To <URL/of/upstream.git>
 	*    HEAD:refs/pull/123/head    [new reference]
 	*    HEAD:refs/for/a/b/c/topic    [new reference]
-	+    HEAD:refs/pull/124/head    <OID-B>...<OID-A> (forced update)
+	+    HEAD:refs/pull/124/head    <COMMIT-B>...<COMMIT-A> (forced update)
 	Done
 	EOF
 	test_cmp expect actual &&
t/t5411/test-0036-report-multi-rewrite-for-one-ref.sh +5 -5
index be9b18b2b6..604656824b 100644
@@ -60,9 +60,9 @@ test_expect_success "proc-receive: multiple rewrite for one ref, no refname for
 	remote: post-receive< <ZERO-OID> <COMMIT-A> refs/changes/24/124/1
 	remote: post-receive< <COMMIT-A> <COMMIT-B> refs/changes/25/125/1
 	To <URL/of/upstream.git>
-	 <OID-A>..<OID-B> HEAD -> refs/for/main/topic
+	 <COMMIT-A>..<COMMIT-B> HEAD -> refs/for/main/topic
 	 * [new reference] HEAD -> refs/changes/24/124/1
-	 <OID-A>..<OID-B> HEAD -> refs/changes/25/125/1
+	 <COMMIT-A>..<COMMIT-B> HEAD -> refs/changes/25/125/1
 	EOF
 	test_cmp expect actual &&
 
@@ -136,8 +136,8 @@ test_expect_success "proc-receive: multiple rewrites for one ref, no refname for
 	remote: post-receive< <COMMIT-B> <COMMIT-A> refs/changes/25/125/1
 	To <URL/of/upstream.git>
 	 * [new reference] HEAD -> refs/changes/24/124/1
-	 <OID-A>..<OID-B> HEAD -> refs/for/main/topic
-	 + <OID-B>...<OID-A> HEAD -> refs/changes/25/125/1 (forced update)
+	 <COMMIT-A>..<COMMIT-B> HEAD -> refs/for/main/topic
+	 + <COMMIT-B>...<COMMIT-A> HEAD -> refs/changes/25/125/1 (forced update)
 	EOF
 	test_cmp expect actual &&
 
@@ -198,7 +198,7 @@ test_expect_success "proc-receive: multiple rewrites for one ref ($PROTOCOL)" '
 	remote: post-receive< <COMMIT-A> <COMMIT-B> refs/changes/24/124/2
 	To <URL/of/upstream.git>
 	 * [new reference] HEAD -> refs/changes/23/123/1
-	 <OID-A>..<OID-B> HEAD -> refs/changes/24/124/2
+	 <COMMIT-A>..<COMMIT-B> HEAD -> refs/changes/24/124/2
 	EOF
 	test_cmp expect actual &&
 
t/t5411/test-0037-report-multi-rewrite-for-one-ref--porcelain.sh +5 -5
index 95fb89c031..6cc0c78a2a 100644
@@ -45,9 +45,9 @@ test_expect_success "proc-receive: multiple rewrite for one ref, no refname for
 	remote: post-receive< <ZERO-OID> <COMMIT-A> refs/changes/24/124/1
 	remote: post-receive< <COMMIT-A> <COMMIT-B> refs/changes/25/125/1
 	To <URL/of/upstream.git>
-	     HEAD:refs/for/main/topic    <OID-A>..<OID-B>
+	     HEAD:refs/for/main/topic    <COMMIT-A>..<COMMIT-B>
 	*    HEAD:refs/changes/24/124/1    [new reference]
-	     HEAD:refs/changes/25/125/1    <OID-A>..<OID-B>
+	     HEAD:refs/changes/25/125/1    <COMMIT-A>..<COMMIT-B>
 	Done
 	EOF
 	test_cmp expect actual &&
@@ -107,8 +107,8 @@ test_expect_success "proc-receive: multiple rewrites for one ref, no refname for
 	remote: post-receive< <COMMIT-B> <COMMIT-A> refs/changes/25/125/1
 	To <URL/of/upstream.git>
 	*    HEAD:refs/changes/24/124/1    [new reference]
-	     HEAD:refs/for/main/topic    <OID-A>..<OID-B>
-	+    HEAD:refs/changes/25/125/1    <OID-B>...<OID-A> (forced update)
+	     HEAD:refs/for/main/topic    <COMMIT-A>..<COMMIT-B>
+	+    HEAD:refs/changes/25/125/1    <COMMIT-B>...<COMMIT-A> (forced update)
 	Done
 	EOF
 	test_cmp expect actual &&
@@ -155,7 +155,7 @@ test_expect_success "proc-receive: multiple rewrites for one ref ($PROTOCOL/porc
 	remote: post-receive< <COMMIT-A> <COMMIT-B> refs/changes/24/124/2
 	To <URL/of/upstream.git>
 	*    HEAD:refs/changes/23/123/1    [new reference]
-	     HEAD:refs/changes/24/124/2    <OID-A>..<OID-B>
+	     HEAD:refs/changes/24/124/2    <COMMIT-A>..<COMMIT-B>
 	Done
 	EOF
 	test_cmp expect actual &&
t/t5411/test-0038-report-mixed-refs.sh +2 -2
index 5e005299cc..9260644814 100644
@@ -55,12 +55,12 @@ test_expect_success "proc-receive: report update of mixed refs ($PROTOCOL)" '
 	remote: post-receive< <ZERO-OID> <COMMIT-A> refs/heads/foo
 	remote: post-receive< <COMMIT-A> <COMMIT-B> refs/for/main/topic
 	To <URL/of/upstream.git>
-	 <OID-A>..<OID-B> <COMMIT-B> -> main
+	 <COMMIT-A>..<COMMIT-B> <COMMIT-B> -> main
 	 * [new branch] HEAD -> bar
 	 * [new branch] HEAD -> baz
 	 * [new reference] HEAD -> refs/for/next/topic2
 	 * [new branch] HEAD -> foo
-	 <OID-A>..<OID-B> HEAD -> refs/for/main/topic
+	 <COMMIT-A>..<COMMIT-B> HEAD -> refs/for/main/topic
 	 ! [remote rejected] HEAD -> refs/for/next/topic1 (fail to call Web API)
 	 ! [remote rejected] HEAD -> refs/for/next/topic3 (proc-receive failed to report status)
 	EOF
t/t5411/test-0039-report-mixed-refs--porcelain.sh +2 -2
index 8f891c5385..4fe37683f8 100644
@@ -55,12 +55,12 @@ test_expect_success "proc-receive: report update of mixed refs ($PROTOCOL/porcel
 	remote: post-receive< <ZERO-OID> <COMMIT-A> refs/heads/foo
 	remote: post-receive< <COMMIT-A> <COMMIT-B> refs/for/main/topic
 	To <URL/of/upstream.git>
-	     <COMMIT-B>:refs/heads/main    <OID-A>..<OID-B>
+	     <COMMIT-B>:refs/heads/main    <COMMIT-A>..<COMMIT-B>
 	*    HEAD:refs/heads/bar    [new branch]
 	*    HEAD:refs/heads/baz    [new branch]
 	*    HEAD:refs/for/next/topic2    [new reference]
 	*    HEAD:refs/heads/foo    [new branch]
-	     HEAD:refs/for/main/topic    <OID-A>..<OID-B>
+	     HEAD:refs/for/main/topic    <COMMIT-A>..<COMMIT-B>
 	!    HEAD:refs/for/next/topic1    [remote rejected] (fail to call Web API)
 	!    HEAD:refs/for/next/topic3    [remote rejected] (proc-receive failed to report status)
 	Done
t/t5411/test-0040-process-all-refs.sh +4 -4
index fdcdcc7c2e..33a7f49a50 100644
@@ -85,11 +85,11 @@ test_expect_success "proc-receive: process all refs ($PROTOCOL)" '
 	remote: post-receive< <COMMIT-A> <COMMIT-B> refs/pull/123/head
 	remote: post-receive< <COMMIT-B> <COMMIT-A> refs/pull/124/head
 	To <URL/of/upstream.git>
-	 <OID-A>..<OID-B> <COMMIT-B> -> bar
+	 <COMMIT-A>..<COMMIT-B> <COMMIT-B> -> bar
 	 - [deleted] foo
-	 + <OID-B>...<OID-A> HEAD -> main (forced update)
-	 <OID-A>..<OID-B> HEAD -> refs/pull/123/head
-	 + <OID-B>...<OID-A> HEAD -> refs/pull/124/head (forced update)
+	 + <COMMIT-B>...<COMMIT-A> HEAD -> main (forced update)
+	 <COMMIT-A>..<COMMIT-B> HEAD -> refs/pull/123/head
+	 + <COMMIT-B>...<COMMIT-A> HEAD -> refs/pull/124/head (forced update)
 	EOF
 	test_cmp expect actual &&
 
t/t5411/test-0041-process-all-refs--porcelain.sh +4 -4
index 73b35fe0aa..07dce47a7d 100644
@@ -85,11 +85,11 @@ test_expect_success "proc-receive: process all refs ($PROTOCOL/porcelain)" '
 	remote: post-receive< <COMMIT-A> <COMMIT-B> refs/pull/123/head
 	remote: post-receive< <COMMIT-B> <COMMIT-A> refs/pull/124/head
 	To <URL/of/upstream.git>
-	     <COMMIT-B>:refs/heads/bar    <OID-A>..<OID-B>
+	     <COMMIT-B>:refs/heads/bar    <COMMIT-A>..<COMMIT-B>
 	-    :refs/heads/foo    [deleted]
-	+    HEAD:refs/heads/main    <OID-B>...<OID-A> (forced update)
-	     HEAD:refs/pull/123/head    <OID-A>..<OID-B>
-	+    HEAD:refs/pull/124/head    <OID-B>...<OID-A> (forced update)
+	+    HEAD:refs/heads/main    <COMMIT-B>...<COMMIT-A> (forced update)
+	     HEAD:refs/pull/123/head    <COMMIT-A>..<COMMIT-B>
+	+    HEAD:refs/pull/124/head    <COMMIT-B>...<COMMIT-A> (forced update)
 	Done
 	EOF
 	test_cmp expect actual &&
t/t5411/test-0050-proc-receive-refs-with-modifiers.sh +2 -2
index 7214647ada..906d75e62d 100644
@@ -46,7 +46,7 @@ test_expect_success "proc-receive: update branch and new tag ($PROTOCOL)" '
 	remote: post-receive< <COMMIT-A> <COMMIT-B> refs/pull/123/head
 	remote: post-receive< <ZERO-OID> <TAG-v123> refs/pull/124/head
 	To <URL/of/upstream.git>
-	 <OID-A>..<OID-B> <COMMIT-B> -> refs/pull/123/head
+	 <COMMIT-A>..<COMMIT-B> <COMMIT-B> -> refs/pull/123/head
 	 * [new reference] v123 -> refs/pull/124/head
 	EOF
 	test_cmp expect actual &&
@@ -116,7 +116,7 @@ test_expect_success "proc-receive: create/delete branch, and delete tag ($PROTOC
 	remote: post-receive< <ZERO-OID> <COMMIT-A> refs/pull/124/head
 	To <URL/of/upstream.git>
 	 - [deleted] refs/pull/123/head
-	 <OID-A>..<OID-B> <COMMIT-B> -> topic
+	 <COMMIT-A>..<COMMIT-B> <COMMIT-B> -> topic
 	 - [deleted] v123
 	 * [new reference] <COMMIT-A> -> refs/pull/124/head
 	EOF
t/t5548-push-porcelain.sh +23 -26

79 lines not on this page · raw message

Signature · 1 line
2.32.0.rc0.27.g7b1e85181b
author
#37 lore raw

PATCH v2 3/4 sideband: append suffix for message whose CR in next pktline

In reply to Ævar Arnfjörð Bjarmason

From: Jiang Xin <[email protected]>

When calling "demultiplex_sideband" on a sideband-2 message, will try to split the message by line breaks, and append a suffix to each nonempty line to clear the end of the screen line. But in the following example, there will be no suffix (8 spaces) for "<message-3>":

    PKT-LINE(\2 <message-1> CR <message-2> CR <message-3>)
    PKT-LINE(\2 CR <message-4> CR <message-5> CR)

This is because the line break of "<message-3>" is placed in the next pktline message.

Without this fix, t5411 must remove trailing spaces of the actual output of "git-push" command before comparing.

---
 sideband.c | 4 ++++
 1 file changed, 4 insertions(+)
sideband.c +4 -0
index 6f9e026732..abf2be98e1 100644
@@ -185,6 +185,10 @@ int demultiplex_sideband(const char *me, int status,
 
 			if (!scratch->len)
 				strbuf_addstr(scratch, DISPLAY_PREFIX);
+			else if (!linelen)
+				/* buf has a leading CR which ends the remaining
+				 * scratch of last round of "demultiplex_sideband" */
+				strbuf_addstr(scratch, suffix);
 			if (linelen > 0) {
 				maybe_colorize_sideband(scratch, b, linelen);
 				strbuf_addstr(scratch, suffix);
Signature · 1 line
2.32.0.rc0.27.g7b1e85181b
#38 lore raw

Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline

Quoted 7 lines · Jiang Xin

From: Jiang Xin <[email protected]>

When calling "demultiplex_sideband" on a sideband-2 message, will try to split the message by line breaks, and append a suffix to each nonempty line to clear the end of the screen line. But in the following example, there will be no suffix (8 spaces) for "<message-3>":

Quoted 8 lines
    PKT-LINE(\2 <message-1> CR <message-2> CR <message-3>)
    PKT-LINE(\2 CR <message-4> CR <message-5> CR)

This is because the line break of "<message-3>" is placed in the next pktline message.

Without this fix, t5411 must remove trailing spaces of the actual output of "git-push" command before comparing.

Nice, i.e. let's generally fix the output instead.

Quoted 10 lines
---
 sideband.c | 4 ++++
 1 file changed, 4 insertions(+)
sideband.c +0 -0
index 6f9e026732..abf2be98e1 100644

sideband.c int demultiplex_sideband(const char *me, int status, · line 192

Quoted 8 lines
sideband.c +4 -0
@@ -185,6 +185,10 @@ int demultiplex_sideband(const char *me, int status,
 
 			if (!scratch->len)
 				strbuf_addstr(scratch, DISPLAY_PREFIX);
+			else if (!linelen)
+				/* buf has a leading CR which ends the remaining
+				 * scratch of last round of "demultiplex_sideband" */
+				strbuf_addstr(scratch, suffix);
 			if (linelen > 0) {

I haven't thought about this carefully but isn't there some way to combine these if/else if/if statementsn that's clearer?

I.e. here we're doing an "if" check for a !lineline and then an "if" that can't be true if !linelen.

Isn't this the same as:

if (!scratch->len) {
    ...
} else {
        if (!linelen)
            ...
        else if (linelen > 0)
            ...
}

Or are there cases where we take that "linelen > 0" arm if !scratch->len?

Quoted 2 lines
sideband.c +0 -0
 				maybe_colorize_sideband(scratch, b, linelen);
 				strbuf_addstr(scratch, suffix);
#39 lore raw

Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline

In reply to Jiang Xin
Quoted 5 lines · Jiang Xin

From: Jiang Xin <[email protected]>

When calling "demultiplex_sideband" on a sideband-2 message, will try to split the message by line breaks, and append a suffix to each nonempty line to clear the end of the screen line.

Subject of "will try" and "append" is missing. Do you mean that the helper function in question does these two things? I.e.

	demultiplex_sideband() used on a sideband #2 will try
	to... and appends ...
Quoted 5 lines

But in the following example,
there will be no suffix (8 spaces) for "<message-3>":

    PKT-LINE(\2 <message-1> CR <message-2> CR <message-3>)
    PKT-LINE(\2 CR <message-4> CR <message-5> CR)

That description may mechanically correct, but

after <message-3>, we fail to clear to the end of line

may make it easier to understand what the problem we are trying to solve for those who do not remember what these suffix games are about.

Quoted 16 lines

This is because the line break of "<message-3>" is placed in the next pktline message.

Without this fix, t5411 must remove trailing spaces of the actual output of "git-push" command before comparing.

---
 sideband.c | 4 ++++
 1 file changed, 4 insertions(+)
sideband.c +0 -0
index 6f9e026732..abf2be98e1 100644

sideband.c int demultiplex_sideband(const char *me, int status, · line 191

Quoted 7 lines
sideband.c +4 -0
@@ -185,6 +185,10 @@ int demultiplex_sideband(const char *me, int status,
 
 			if (!scratch->len)
 				strbuf_addstr(scratch, DISPLAY_PREFIX);
+			else if (!linelen)
+				/* buf has a leading CR which ends the remaining
+				 * scratch of last round of "demultiplex_sideband" */
+				strbuf_addstr(scratch, suffix);

The style of multi-line comment needs fixing, but the contents of the comment is a bit hard to grok.

sideband.c int demultiplex_sideband(const char *me, int status, · line 194

Quoted 3 lines
sideband.c +0 -0
 			if (linelen > 0) {
 				maybe_colorize_sideband(scratch, b, linelen);
 				strbuf_addstr(scratch, suffix);

I wonder if the following is simpler to read, though.

-- >8 --
Subject: [PATCH] sideband: don't lose clear-to-eol at packet boundary

When demultiplex_sideband() sees a CR or LF on the sideband #2, it adds "suffix" string to clear to the end of the current line, which helps when relaying a progress display whose records are terminated with CRs.

The code however forgot that depending on the length of the payload line, such a CR may fall exactly at the packet boundary and the number of bytes before the CR from the beginning of the packet could be zero. In such a case, the message that was terminated by the CR were leftover in the "scratch" buffer in the previous call to the function and we still need to clear to the end of the current line.

Just remove the unnecessary check on linelen; maybe_colorize_sideband() on 0-byte payload turns into a no-op, and we should be adding clear-to-eol for each and every CR/LF anyway.

 sideband.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)
sideband.c +3 -4
index 6f9e026732..1575bf16dd 100644
@@ -185,10 +185,9 @@ int demultiplex_sideband(const char *me, int status,
 
 			if (!scratch->len)
 				strbuf_addstr(scratch, DISPLAY_PREFIX);
-			if (linelen > 0) {
-				maybe_colorize_sideband(scratch, b, linelen);
-				strbuf_addstr(scratch, suffix);
-			}
+
+			maybe_colorize_sideband(scratch, b, linelen);
+			strbuf_addstr(scratch, suffix);
 
 			strbuf_addch(scratch, *brk);
 			xwrite(2, scratch->buf, scratch->len);
author
#40 lore raw

Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline

Quoted 92 lines · Junio C Hamano <[email protected]> 于2021年6月14日周一 上午11:50写道:
Quoted 5 lines · Jiang Xin

From: Jiang Xin <[email protected]>

When calling "demultiplex_sideband" on a sideband-2 message, will try to split the message by line breaks, and append a suffix to each nonempty line to clear the end of the screen line.

Subject of "will try" and "append" is missing. Do you mean that the helper function in question does these two things? I.e.

demultiplex_sideband() used on a sideband #2 will try to... and appends ...

Quoted 5 lines

But in the following example,
there will be no suffix (8 spaces) for "<message-3>":

    PKT-LINE(\2 <message-1> CR <message-2> CR <message-3>)
    PKT-LINE(\2 CR <message-4> CR <message-5> CR)

That description may mechanically correct, but

after <message-3>, we fail to clear to the end of line

may make it easier to understand what the problem we are trying to solve for those who do not remember what these suffix games are about.

Quoted 16 lines

This is because the line break of "<message-3>" is placed in the next pktline message.

Without this fix, t5411 must remove trailing spaces of the actual output of "git-push" command before comparing.

---
 sideband.c | 4 ++++
 1 file changed, 4 insertions(+)
sideband.c +0 -0
index 6f9e026732..abf2be98e1 100644

sideband.c int demultiplex_sideband(const char *me, int status, · line 191

Quoted 7 lines
sideband.c +4 -0
@@ -185,6 +185,10 @@ int demultiplex_sideband(const char *me, int status,
 
                      if (!scratch->len)
                              strbuf_addstr(scratch, DISPLAY_PREFIX);
+                     else if (!linelen)
+                             /* buf has a leading CR which ends the remaining
+                              * scratch of last round of "demultiplex_sideband" */
+                             strbuf_addstr(scratch, suffix);

The style of multi-line comment needs fixing, but the contents of the comment is a bit hard to grok.

sideband.c int demultiplex_sideband(const char *me, int status, · line 194

Quoted 3 lines
sideband.c +0 -0
                      if (linelen > 0) {
                              maybe_colorize_sideband(scratch, b, linelen);
                              strbuf_addstr(scratch, suffix);

I wonder if the following is simpler to read, though.

-- >8 --
Subject: [PATCH] sideband: don't lose clear-to-eol at packet boundary

When demultiplex_sideband() sees a CR or LF on the sideband #2, it adds "suffix" string to clear to the end of the current line, which helps when relaying a progress display whose records are terminated with CRs.

The code however forgot that depending on the length of the payload line, such a CR may fall exactly at the packet boundary and the number of bytes before the CR from the beginning of the packet could be zero. In such a case, the message that was terminated by the CR were leftover in the "scratch" buffer in the previous call to the function and we still need to clear to the end of the current line.

Just remove the unnecessary check on linelen; maybe_colorize_sideband() on 0-byte payload turns into a no-op, and we should be adding clear-to-eol for each and every CR/LF anyway.

 sideband.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)
sideband.c +0 -0
index 6f9e026732..1575bf16dd 100644
@@ -185,10 +185,9 @@ int demultiplex_sideband(const char *me, int status,
 
                        if (!scratch->len)
                                strbuf_addstr(scratch, DISPLAY_PREFIX);

sideband.c int demultiplex_sideband(const char *me, int status, · line 193

Quoted 10 lines
sideband.c +3 -4
-                       if (linelen > 0) {
-                               maybe_colorize_sideband(scratch, b, linelen);
-                               strbuf_addstr(scratch, suffix);
-                       }
+
+                       maybe_colorize_sideband(scratch, b, linelen);
+                       strbuf_addstr(scratch, suffix);
 
                        strbuf_addch(scratch, *brk);
                        xwrite(2, scratch->buf, scratch->len);

The above changes will add suffix to the end of each line, and even an empty lines. However, according to the comment in commit ebe8fa738d (fix display overlap between remote and local progress, 2007-11-04) which introduced the suffix implementation for the first time, no suffix should be appended for empty lines.

/*
* Let's insert a suffix to clear the end
* of the screen line, but only if current
* line data actually contains something.
*/

So my implementation is to try not to break the original implementation, and keep the linelen unchanged.

The strbuf "scratch" will be reset at line 18th in the while block, so the nonempty scratch at line 7 indicates the parameter scratch of demultiplex_sideband() is not empty. With the following patch, additional suffix is only added before a leading CR in a packet which is seperated with its message by packet boundary.

```
01    while ((brk = strpbrk(b, "\n\r"))) {
02            int linelen = brk - b;
03
04 +         /* Has no empty scratch from last call of "demultiplex_sideband"
05 +          * and has a leading CR in buf.
06 +          */
07 +         if (scratch->len && !linelen)
08 +                   strbuf_addstr(scratch, suffix);
09            if (!scratch->len)
10                    strbuf_addstr(scratch, DISPLAY_PREFIX);
11            if (linelen > 0) {
12                    maybe_colorize_sideband(scratch, b, linelen);
13                    strbuf_addstr(scratch, suffix);
14            }
15
16            strbuf_addch(scratch, *brk);
17            xwrite(2, scratch->buf, scratch->len);
18            strbuf_reset(scratch);
19
20            b = brk + 1;
21    }
```
#41 lore raw

Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline

Quoted 8 lines · Jiang Xin

/*
* Let's insert a suffix to clear the end
* of the screen line, but only if current
* line data actually contains something.
*/

So my implementation is to try not to break the original implementation, and keep the linelen unchanged.

I knew what you wanted to do from your code---I am questioning if that "only when something is there" was really sensible, or if it was just attracting bugs.

Thanks.

author
#42 lore raw

Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline

Quoted 9 lines · Junio C Hamano <[email protected]> 于2021年6月15日周二 上午9:17写道:
Quoted 6 lines · Jiang Xin

/*
* Let's insert a suffix to clear the end
* of the screen line, but only if current
* line data actually contains something.
*/

Quoted 7 lines
Quoted 2 lines

So my implementation is to try not to break the original implementation, and keep the linelen unchanged.

I knew what you wanted to do from your code---I am questioning if that "only when something is there" was really sensible, or if it was just attracting bugs.

@Nicolas, what's your opinion? Is it ok to add clear-to-eol suffix to each line even empty ones?

--
Jiang Xin

#43 lore raw

Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline

Quoted 13 lines · Jiang Xin
Quoted 12 lines · Junio C Hamano <[email protected]> 于2021年6月15日周二 上午9:17写道:
Quoted 8 lines · Jiang Xin

/*
* Let's insert a suffix to clear the end
* of the screen line, but only if current
* line data actually contains something.
*/

So my implementation is to try not to break the original implementation, and keep the linelen unchanged.

Quoted 7 lines
Quoted 4 lines

I knew what you wanted to do from your code---I am questioning if that "only when something is there" was really sensible, or if it was just attracting bugs.

@Nicolas, what's your opinion? Is it ok to add clear-to-eol suffix to each line even empty ones?

That would be the simplest thing to do.

But there must have been a reason for doing it otherwise. I just don't remember anymore.

Maybe it had to do with progress reporting that does a bunch of percentage updates followed by '\r' to remain on the same line, and at the end a single '\n' to move to the next line without erasing the final status report line. That would be a case for not clearing empty lines.

Nicolas

author
#44 lore raw

Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline

Quoted 26 lines · Nicolas Pitre <[email protected]> 于2021年6月15日周二 上午10:11写道:
Quoted 13 lines · Jiang Xin
Quoted 12 lines · Junio C Hamano <[email protected]> 于2021年6月15日周二 上午9:17写道:
Quoted 8 lines · Jiang Xin

/*
* Let's insert a suffix to clear the end
* of the screen line, but only if current
* line data actually contains something.
*/

So my implementation is to try not to break the original implementation, and keep the linelen unchanged.

Quoted 7 lines
Quoted 4 lines

I knew what you wanted to do from your code---I am questioning if that "only when something is there" was really sensible, or if it was just attracting bugs.

@Nicolas, what's your opinion? Is it ok to add clear-to-eol suffix to each line even empty ones?

That would be the simplest thing to do.

Quoted 8 lines

But there must have been a reason for doing it otherwise. I just don't remember anymore.

Maybe it had to do with progress reporting that does a bunch of percentage updates followed by '\r' to remain on the same line, and at the end a single '\n' to move to the next line without erasing the final status report line. That would be a case for not clearing empty lines.

Thank @Nicolas for helping me understand the story behinds the code.

If there are two sideband #2 packets like this:

    PKTLINE(\2 "<progress-1>" CR "<progress-2>" CR)
    PKTLINE(\2 "<message-3>" LF "<message-4>" LF)

We should append clear-to-eol suffix to "<progress-1>", "<progess-2>" and "<message-3>" to erase the last message displayed on the same line. Even though there is no need to add the clear-to-eol suffix to "<message-4>", always adding suffix before line breaks (CR or LF) of nonempty message make it simple to program.

If there are empty messages in sideband #2 packets like this:

    PKTLINE(\2 "<progress-1>" CR LF "<message-2>" LF)
    PKTLINE(\2 "<message-3>" LF)

For the empty message between "<progress-1>" and "<message-2>", nothing to display and no need to add clear-to-eol suffix.

The issue this patch try to fix is like the following example:

    PKTLINE(\2 "<progress-1>" CR "<progress-2>")
    PKTLINE(\2 CR "<message-3>" LF)

The message "<progress-2>" is displayed without a proper clear-to-eol suffix, because it's eol (CR) is in another pktline.

Since we can distinguished this case by checking the size of "scratch", IMHO, it better not add suffix before all line breaks.

--
Jiang Xin

#45 lore raw

Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline

Quoted 7 lines · Jiang Xin

The issue this patch try to fix is like the following example:

    PKTLINE(\2 "<progress-1>" CR "<progress-2>")
    PKTLINE(\2 CR "<message-3>" LF)

The message "<progress-2>" is displayed without a proper clear-to-eol suffix, because it's eol (CR) is in another pktline.

I'd fix this issue with the following logic:

bool pending_clear_to_eol;

my_putchar(c) {
	switch (c) {
	case '\r':
	case '\n':
		pending_clear_to_eol = true;
		break;
	default:
		if (pending_clear_to_eol) {
			clear_to_eol();
			pending_clear_to_eol = false;
		}
		break;
	}
	putchar(c);
}

In other words, you clear the line after printing "remote:" but only if there is a non \n or \r coming next.

Nicolas

#46 lore raw

Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline

Quoted 30 lines · Nicolas Pitre
Quoted 7 lines · Jiang Xin

The issue this patch try to fix is like the following example:

    PKTLINE(\2 "<progress-1>" CR "<progress-2>")
    PKTLINE(\2 CR "<message-3>" LF)

The message "<progress-2>" is displayed without a proper clear-to-eol suffix, because it's eol (CR) is in another pktline.

I'd fix this issue with the following logic:

bool pending_clear_to_eol;

my_putchar(c) {
	switch (c) {
	case '\r':
	case '\n':
		pending_clear_to_eol = true;
		break;
	default:
		if (pending_clear_to_eol) {
			clear_to_eol();
			pending_clear_to_eol = false;
		}
		break;
	}
	putchar(c);
}
Quoted 2 lines

In other words, you clear the line after printing "remote:" but only if there is a non \n or \r coming next.

What puzzles me the most in this discussion is why we do this for LF. I do understand why we need it for CR---the line we are going to show message on after emitting CR would be full of leftover letters we previously have written before emitting CR, so we'd show the message (to overwrite the initial part enough to show our own message) and then clear to the end with either ANSI sequence of sufficient number of whitespaces. But line feed would take us to a fresh and blank line---there is nothing to clear, no?

Thanks.

author
#47 lore raw

Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline

Quoted 36 lines · Junio C Hamano <[email protected]> 于2021年6月15日周二 下午12:46写道:
Quoted 30 lines · Nicolas Pitre
Quoted 7 lines · Jiang Xin

The issue this patch try to fix is like the following example:

    PKTLINE(\2 "<progress-1>" CR "<progress-2>")
    PKTLINE(\2 CR "<message-3>" LF)

The message "<progress-2>" is displayed without a proper clear-to-eol suffix, because it's eol (CR) is in another pktline.

I'd fix this issue with the following logic:

bool pending_clear_to_eol;

my_putchar(c) {
      switch (c) {
      case '\r':
      case '\n':
              pending_clear_to_eol = true;
              break;
      default:
              if (pending_clear_to_eol) {
                      clear_to_eol();
                      pending_clear_to_eol = false;
              }
              break;
      }
      putchar(c);
}
Quoted 2 lines

In other words, you clear the line after printing "remote:" but only if there is a non \n or \r coming next.

Quoted 8 lines

What puzzles me the most in this discussion is why we do this for LF. I do understand why we need it for CR---the line we are going to show message on after emitting CR would be full of leftover letters we previously have written before emitting CR, so we'd show the message (to overwrite the initial part enough to show our own message) and then clear to the end with either ANSI sequence of sufficient number of whitespaces. But line feed would take us to a fresh and blank line---there is nothing to clear, no?

I guess this may because sideband #2 messages are printed on the screen in a background process, it never know a line where it starts to print has characters on the right. So it is safe to write an additional clear-to-eol suffix no matter the message ends with CR or LF.

--
Jiang Xin

#48 lore raw

Re: [PATCH v2 3/4] sideband: append suffix for message whose CR in next pktline

In reply to Junio C Hamano
Quoted 35 lines · Junio C Hamano
Quoted 30 lines · Nicolas Pitre
Quoted 7 lines · Jiang Xin

The issue this patch try to fix is like the following example:

    PKTLINE(\2 "<progress-1>" CR "<progress-2>")
    PKTLINE(\2 CR "<message-3>" LF)

The message "<progress-2>" is displayed without a proper clear-to-eol suffix, because it's eol (CR) is in another pktline.

I'd fix this issue with the following logic:

bool pending_clear_to_eol;

my_putchar(c) {
	switch (c) {
	case '\r':
	case '\n':
		pending_clear_to_eol = true;
		break;
	default:
		if (pending_clear_to_eol) {
			clear_to_eol();
			pending_clear_to_eol = false;
		}
		break;
	}
	putchar(c);
}
Quoted 2 lines

In other words, you clear the line after printing "remote:" but only if there is a non \n or \r coming next.

Quoted 8 lines

What puzzles me the most in this discussion is why we do this for LF. I do understand why we need it for CR---the line we are going to show message on after emitting CR would be full of leftover letters we previously have written before emitting CR, so we'd show the message (to overwrite the initial part enough to show our own message) and then clear to the end with either ANSI sequence of sufficient number of whitespaces. But line feed would take us to a fresh and blank line---there is nothing to clear, no?

Depends. Suppose the local process is doing the progress report with CR.

Then the remote sends a single line with LF.

You expects the remote line to be displayed over the local progress report and the local progress report to be resumed on the following line. Without the line clearing you might have leftover garbage on the remote message line.

Nicolas

author
#49 lore raw

PATCH v2 4/4 test: compare raw output, not mangle tabs and spaces

In reply to Ævar Arnfjörð Bjarmason

From: Jiang Xin <[email protected]>

We used to call make_user_friendly_and_stable_output to mangle trailing spaces in output before comparing with the expect file. Ævar recommends generating expect file using pattern "'s/Z$//'" to compare expect file with raw output.

---
 t/t5411/common-functions.sh                   |   9 +-
 t/t5411/test-0000-standard-git-push.sh        |  82 +++----
 .../test-0001-standard-git-push--porcelain.sh |  90 ++++----
 ...st-0003-pre-receive-declined--porcelain.sh |   8 +-
 t/t5411/test-0011-no-hook-error.sh            |  40 ++--
 t/t5411/test-0012-no-hook-error--porcelain.sh |  42 ++--
 t/t5411/test-0013-bad-protocol.sh             |  62 +++---
 t/t5411/test-0014-bad-protocol--porcelain.sh  |  80 +++----
 t/t5411/test-0020-report-ng.sh                |  32 +--
 t/t5411/test-0021-report-ng--porcelain.sh     |  36 ++--
 t/t5411/test-0022-report-unexpect-ref.sh      |  26 +--
 ...est-0023-report-unexpect-ref--porcelain.sh |  28 +--
 t/t5411/test-0024-report-unknown-ref.sh       |  18 +-
 ...test-0025-report-unknown-ref--porcelain.sh |  20 +-
 t/t5411/test-0026-push-options.sh             |  58 ++---
 t/t5411/test-0027-push-options--porcelain.sh  |  62 +++---
 t/t5411/test-0030-report-ok.sh                |  20 +-
 t/t5411/test-0031-report-ok--porcelain.sh     |  22 +-
 t/t5411/test-0032-report-with-options.sh      | 186 ++++++++--------
 ...est-0033-report-with-options--porcelain.sh | 200 +++++++++---------
 t/t5411/test-0034-report-ft.sh                |  22 +-
 t/t5411/test-0035-report-ft--porcelain.sh     |  24 +--
 ...t-0036-report-multi-rewrite-for-one-ref.sh | 132 ++++++------
 ...rt-multi-rewrite-for-one-ref--porcelain.sh | 138 ++++++------
 t/t5411/test-0038-report-mixed-refs.sh        |  74 +++----
 .../test-0039-report-mixed-refs--porcelain.sh |  76 +++----
 t/t5411/test-0040-process-all-refs.sh         |  80 +++----
 .../test-0041-process-all-refs--porcelain.sh  |  82 +++----
 ...t-0050-proc-receive-refs-with-modifiers.sh |  90 ++++----
 t/t5548-push-porcelain.sh                     |  54 ++---
 t/t6020-bundle-misc.sh                        |  45 ++--
 31 files changed, 972 insertions(+), 966 deletions(-)
t/t5411/common-functions.sh +5 -4
index 6398f5f2a3..3c747782c1 100644
@@ -32,17 +32,14 @@ get_abbrev_oid () {
 
 # Format the output of git-push, git-show-ref and other commands to make a
 # user-friendly and stable text.  We can easily prepare the expect text
-# without having to worry about future changes of the commit ID and spaces
+# without having to worry about changes of the commit ID (full or abbrev.)
 # of the output.  Single quotes are replaced with double quotes, because
 # it is boring to prepare unquoted single quotes in expect text.  We also
 # remove some locale error messages. The emitted human-readable errors are
 # redundant to the more machine-readable output the tests already assert.
 make_user_friendly_and_stable_output () {
 	sed \
-		-e "s/  *\$//" \
-		-e "s/  */ /g" \
 		-e "s/'/\"/g" \
-		-e "s/	/    /g" \
 		-e "s/$(get_abbrev_oid $A)[0-9a-f]*/<COMMIT-A>/g" \
 		-e "s/$(get_abbrev_oid $B)[0-9a-f]*/<COMMIT-B>/g" \
 		-e "s/$(get_abbrev_oid $TAG)[0-9a-f]*/<TAG-v123>/g" \
@@ -56,6 +53,10 @@ filter_out_user_friendly_and_stable_output () {
 		sed -n ${1+"$@"}
 }
 
+format_and_save_expect () {
+	sed -e 's/^> //' -e 's/Z$//' >expect
+}
+
 test_cmp_refs () {
 	indir=
 	if test "$1" = "-C"
t/t5411/test-0000-standard-git-push.sh +41 -41

106 lines not on this page · raw message

t/t5411/test-0001-standard-git-push--porcelain.sh +45 -45

115 lines not on this page · raw message

t/t5411/test-0003-pre-receive-declined--porcelain.sh +4 -4
index e9c9db5d1f..2393b04ad9 100644
@@ -14,10 +14,10 @@ test_expect_success "git-push is declined ($PROTOCOL/porcelain)" '
 		HEAD:refs/heads/next \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output <out-$test_count >actual &&
-	cat >expect <<-EOF &&
-	To <URL/of/upstream.git>
-	!    <COMMIT-B>:refs/heads/main    [remote rejected] (pre-receive hook declined)
-	!    HEAD:refs/heads/next    [remote rejected] (pre-receive hook declined)
+	format_and_save_expect <<-EOF &&
+	> To <URL/of/upstream.git>
+	> !	<COMMIT-B>:refs/heads/main	[remote rejected] (pre-receive hook declined)
+	> !	HEAD:refs/heads/next	[remote rejected] (pre-receive hook declined)
 	Done
 	EOF
 	test_cmp expect actual &&
t/t5411/test-0011-no-hook-error.sh +20 -20
index 3ef136e6ef..d35002b1f0 100644
@@ -7,16 +7,16 @@ test_expect_success "proc-receive: no hook, fail to push special ref ($PROTOCOL)
 		HEAD:refs/for/main/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output <out-$test_count >actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/heads/next
-	remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic
-	remote: error: cannot find hook "proc-receive"
-	remote: # post-receive hook
-	remote: post-receive< <ZERO-OID> <COMMIT-A> refs/heads/next
-	To <URL/of/upstream.git>
-	 * [new branch] HEAD -> next
-	 ! [remote rejected] HEAD -> refs/for/main/topic (fail to run proc-receive hook)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/heads/next        Z
+	> remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic        Z
+	> remote: error: cannot find hook "proc-receive"        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive< <ZERO-OID> <COMMIT-A> refs/heads/next        Z
+	> To <URL/of/upstream.git>
+	>  * [new branch]      HEAD -> next
+	>  ! [remote rejected] HEAD -> refs/for/main/topic (fail to run proc-receive hook)
 	EOF
 	test_cmp expect actual &&
 
@@ -41,16 +41,16 @@ test_expect_success "proc-receive: no hook, all failed for atomic push ($PROTOCO
 		HEAD:next \
 		HEAD:refs/for/main/topic >out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output <out-$test_count >actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive< <COMMIT-A> <COMMIT-B> refs/heads/main
-	remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/heads/next
-	remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic
-	remote: error: cannot find hook "proc-receive"
-	To <URL/of/upstream.git>
-	 ! [remote rejected] <COMMIT-B> -> main (fail to run proc-receive hook)
-	 ! [remote rejected] HEAD -> next (fail to run proc-receive hook)
-	 ! [remote rejected] HEAD -> refs/for/main/topic (fail to run proc-receive hook)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive< <COMMIT-A> <COMMIT-B> refs/heads/main        Z
+	> remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/heads/next        Z
+	> remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic        Z
+	> remote: error: cannot find hook "proc-receive"        Z
+	> To <URL/of/upstream.git>
+	>  ! [remote rejected] <COMMIT-B> -> main (fail to run proc-receive hook)
+	>  ! [remote rejected] HEAD -> next (fail to run proc-receive hook)
+	>  ! [remote rejected] HEAD -> refs/for/main/topic (fail to run proc-receive hook)
 	EOF
 	test_cmp expect actual &&
 
t/t5411/test-0012-no-hook-error--porcelain.sh +21 -21

54 lines not on this page · raw message

t/t5411/test-0013-bad-protocol.sh +31 -31

86 lines not on this page · raw message

t/t5411/test-0014-bad-protocol--porcelain.sh +40 -40

140 lines not on this page · raw message

t/t5411/test-0020-report-ng.sh +16 -16
index ad2c8f6535..e915dbc28d 100644
@@ -14,14 +14,14 @@ test_expect_success "proc-receive: fail to update (ng, no message, $PROTOCOL)" '
 		HEAD:refs/for/main/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output <out-$test_count >actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic
-	remote: proc-receive> ng refs/for/main/topic
-	To <URL/of/upstream.git>
-	 ! [remote rejected] HEAD -> refs/for/main/topic (failed)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic        Z
+	> remote: proc-receive> ng refs/for/main/topic        Z
+	> To <URL/of/upstream.git>
+	>  ! [remote rejected] HEAD -> refs/for/main/topic (failed)
 	EOF
 	test_cmp expect actual &&
 
@@ -46,14 +46,14 @@ test_expect_success "proc-receive: fail to update (ng, with message, $PROTOCOL)"
 		HEAD:refs/for/main/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output <out-$test_count >actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic
-	remote: proc-receive> ng refs/for/main/topic error msg
-	To <URL/of/upstream.git>
-	 ! [remote rejected] HEAD -> refs/for/main/topic (error msg)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic        Z
+	> remote: proc-receive> ng refs/for/main/topic error msg        Z
+	> To <URL/of/upstream.git>
+	>  ! [remote rejected] HEAD -> refs/for/main/topic (error msg)
 	EOF
 	test_cmp expect actual &&
 
t/t5411/test-0021-report-ng--porcelain.sh +18 -18
index d8ae9d3414..2a392e099b 100644
@@ -14,15 +14,15 @@ test_expect_success "proc-receive: fail to update (ng, no message, $PROTOCOL/por
 		HEAD:refs/for/main/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output <out-$test_count >actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic
-	remote: proc-receive> ng refs/for/main/topic
-	To <URL/of/upstream.git>
-	!    HEAD:refs/for/main/topic    [remote rejected] (failed)
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic        Z
+	> remote: proc-receive> ng refs/for/main/topic        Z
+	> To <URL/of/upstream.git>
+	> !	HEAD:refs/for/main/topic	[remote rejected] (failed)
+	> Done
 	EOF
 	test_cmp expect actual &&
 
@@ -47,15 +47,15 @@ test_expect_success "proc-receive: fail to update (ng, with message, $PROTOCOL/p
 		HEAD:refs/for/main/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output <out-$test_count >actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic
-	remote: proc-receive> ng refs/for/main/topic error msg
-	To <URL/of/upstream.git>
-	!    HEAD:refs/for/main/topic    [remote rejected] (error msg)
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic        Z
+	> remote: proc-receive> ng refs/for/main/topic error msg        Z
+	> To <URL/of/upstream.git>
+	> !	HEAD:refs/for/main/topic	[remote rejected] (error msg)
+	> Done
 	EOF
 	test_cmp expect actual &&
 
t/t5411/test-0022-report-unexpect-ref.sh +13 -13
index a482ff931a..f7a494bdb9 100644
@@ -15,19 +15,19 @@ test_expect_success "proc-receive: report unexpected ref ($PROTOCOL)" '
 		HEAD:refs/for/main/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output <out-$test_count >actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive< <COMMIT-A> <COMMIT-B> refs/heads/main
-	remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic
-	remote: proc-receive> ok refs/heads/main
-	remote: error: proc-receive reported status on unexpected ref: refs/heads/main
-	remote: # post-receive hook
-	remote: post-receive< <COMMIT-A> <COMMIT-B> refs/heads/main
-	To <URL/of/upstream.git>
-	 <COMMIT-A>..<COMMIT-B> <COMMIT-B> -> main
-	 ! [remote rejected] HEAD -> refs/for/main/topic (proc-receive failed to report status)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive< <COMMIT-A> <COMMIT-B> refs/heads/main        Z
+	> remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/heads/main        Z
+	> remote: error: proc-receive reported status on unexpected ref: refs/heads/main        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive< <COMMIT-A> <COMMIT-B> refs/heads/main        Z
+	> To <URL/of/upstream.git>
+	>    <COMMIT-A>..<COMMIT-B>  <COMMIT-B> -> main
+	>  ! [remote rejected] HEAD -> refs/for/main/topic (proc-receive failed to report status)
 	EOF
 	test_cmp expect actual &&
 
t/t5411/test-0023-report-unexpect-ref--porcelain.sh +14 -14
index c586cd70a0..63c479e975 100644
@@ -15,20 +15,20 @@ test_expect_success "proc-receive: report unexpected ref ($PROTOCOL/porcelain)"
 		HEAD:refs/for/main/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output <out-$test_count >actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive< <COMMIT-A> <COMMIT-B> refs/heads/main
-	remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic
-	remote: proc-receive> ok refs/heads/main
-	remote: error: proc-receive reported status on unexpected ref: refs/heads/main
-	remote: # post-receive hook
-	remote: post-receive< <COMMIT-A> <COMMIT-B> refs/heads/main
-	To <URL/of/upstream.git>
-	     <COMMIT-B>:refs/heads/main    <COMMIT-A>..<COMMIT-B>
-	!    HEAD:refs/for/main/topic    [remote rejected] (proc-receive failed to report status)
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive< <COMMIT-A> <COMMIT-B> refs/heads/main        Z
+	> remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/heads/main        Z
+	> remote: error: proc-receive reported status on unexpected ref: refs/heads/main        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive< <COMMIT-A> <COMMIT-B> refs/heads/main        Z
+	> To <URL/of/upstream.git>
+	>  	<COMMIT-B>:refs/heads/main	<COMMIT-A>..<COMMIT-B>
+	> !	HEAD:refs/for/main/topic	[remote rejected] (proc-receive failed to report status)
+	> Done
 	EOF
 	test_cmp expect actual &&
 
t/t5411/test-0024-report-unknown-ref.sh +9 -9
index 77204244b8..af055aa086 100644
@@ -14,15 +14,15 @@ test_expect_success "proc-receive: report unknown reference ($PROTOCOL)" '
 		HEAD:refs/for/a/b/c/my/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output <out-$test_count >actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/a/b/c/my/topic
-	remote: # proc-receive hook
-	remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/a/b/c/my/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: error: proc-receive reported status on unknown ref: refs/for/main/topic
-	To <URL/of/upstream.git>
-	 ! [remote rejected] HEAD -> refs/for/a/b/c/my/topic (proc-receive failed to report status)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/a/b/c/my/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/a/b/c/my/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: error: proc-receive reported status on unknown ref: refs/for/main/topic        Z
+	> To <URL/of/upstream.git>
+	>  ! [remote rejected] HEAD -> refs/for/a/b/c/my/topic (proc-receive failed to report status)
 	EOF
 	test_cmp expect actual &&
 
t/t5411/test-0025-report-unknown-ref--porcelain.sh +10 -10
index eeb1ce6b2c..99601ca321 100644
@@ -14,16 +14,16 @@ test_expect_success "proc-receive: report unknown reference ($PROTOCOL/porcelain
 		HEAD:refs/for/a/b/c/my/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output <out-$test_count >actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/a/b/c/my/topic
-	remote: # proc-receive hook
-	remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/a/b/c/my/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: error: proc-receive reported status on unknown ref: refs/for/main/topic
-	To <URL/of/upstream.git>
-	!    HEAD:refs/for/a/b/c/my/topic    [remote rejected] (proc-receive failed to report status)
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/a/b/c/my/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/a/b/c/my/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: error: proc-receive reported status on unknown ref: refs/for/main/topic        Z
+	> To <URL/of/upstream.git>
+	> !	HEAD:refs/for/a/b/c/my/topic	[remote rejected] (proc-receive failed to report status)
+	> Done
 	EOF
 	test_cmp expect actual &&
 
t/t5411/test-0026-push-options.sh +29 -29

70 lines not on this page · raw message

t/t5411/test-0027-push-options--porcelain.sh +31 -31

74 lines not on this page · raw message

t/t5411/test-0030-report-ok.sh +10 -10
index 8acb4f204f..a3a6278213 100644
@@ -14,16 +14,16 @@ test_expect_success "proc-receive: ok ($PROTOCOL)" '
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output <out >actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: # post-receive hook
-	remote: post-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic
-	To <URL/of/upstream.git>
-	 * [new reference] HEAD -> refs/for/main/topic
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic        Z
+	> To <URL/of/upstream.git>
+	>  * [new reference]   HEAD -> refs/for/main/topic
 	EOF
 	test_cmp expect actual &&
 
t/t5411/test-0031-report-ok--porcelain.sh +11 -11
index a967718046..0e175388b6 100644
@@ -14,17 +14,17 @@ test_expect_success "proc-receive: ok ($PROTOCOL/porcelain)" '
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output <out >actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: # post-receive hook
-	remote: post-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic
-	To <URL/of/upstream.git>
-	*    HEAD:refs/for/main/topic    [new reference]
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive< <ZERO-OID> <COMMIT-A> refs/for/main/topic        Z
+	> To <URL/of/upstream.git>
+	> *	HEAD:refs/for/main/topic	[new reference]
+	> Done
 	EOF
 	test_cmp expect actual &&
 
t/t5411/test-0032-report-with-options.sh +93 -93

228 lines not on this page · raw message

t/t5411/test-0033-report-with-options--porcelain.sh +100 -100

242 lines not on this page · raw message

t/t5411/test-0034-report-ft.sh +11 -11
index 6e0d08b327..73a47d1ffd 100644
@@ -15,17 +15,17 @@ test_expect_success "proc-receive: fall throught, let receive-pack to execute ($
 		$B:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output <out >actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option fall-through
-	remote: # post-receive hook
-	remote: post-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic
-	To <URL/of/upstream.git>
-	 * [new reference] <COMMIT-B> -> refs/for/main/topic
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option fall-through        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic        Z
+	> To <URL/of/upstream.git>
+	>  * [new reference]   <COMMIT-B> -> refs/for/main/topic
 	EOF
 	test_cmp expect actual &&
 
t/t5411/test-0035-report-ft--porcelain.sh +12 -12
index 81bae9f2ec..c350201107 100644
@@ -15,18 +15,18 @@ test_expect_success "proc-receive: fall throught, let receive-pack to execute ($
 		$B:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output <out >actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option fall-through
-	remote: # post-receive hook
-	remote: post-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic
-	To <URL/of/upstream.git>
-	*    <COMMIT-B>:refs/for/main/topic    [new reference]
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option fall-through        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive< <ZERO-OID> <COMMIT-B> refs/for/main/topic        Z
+	> To <URL/of/upstream.git>
+	> *	<COMMIT-B>:refs/for/main/topic	[new reference]
+	> Done
 	EOF
 	test_cmp expect actual &&
 
t/t5411/test-0036-report-multi-rewrite-for-one-ref.sh +66 -66

150 lines not on this page · raw message

t/t5411/test-0037-report-multi-rewrite-for-one-ref--porcelain.sh +69 -69

156 lines not on this page · raw message

t/t5411/test-0038-report-mixed-refs.sh +37 -37

80 lines not on this page · raw message

t/t5411/test-0039-report-mixed-refs--porcelain.sh +38 -38

82 lines not on this page · raw message

t/t5411/test-0040-process-all-refs.sh +40 -40

86 lines not on this page · raw message

t/t5411/test-0041-process-all-refs--porcelain.sh +41 -41

88 lines not on this page · raw message

t/t5411/test-0050-proc-receive-refs-with-modifiers.sh +45 -45

102 lines not on this page · raw message

t/t5548-push-porcelain.sh +28 -26

99 lines not on this page · raw message

t/t6020-bundle-misc.sh +24 -21

114 lines not on this page · raw message

Signature · 1 line
2.32.0.rc0.27.g7b1e85181b
author
#50 lore raw

PATCH v6 2/3 bundle: lost objects when removing duplicate pendings

In reply to Junio C Hamano

From: Jiang Xin <[email protected]>

git rev-list will list one commit for the following command:

    $ git rev-list 'main^!'
    <tip-commit-of-main-branch>

But providing the same rev-list args to git bundle, fail to create a bundle file.

    $ git bundle create - 'main^!'
    # v2 git bundle
    -<OID> <one-line-message>

fatal: Refusing to create empty bundle.

This is because when removing duplicate objects in function object_array_remove_duplicates(), one unique pending object which has the same name is deleted by mistake. The revision arg 'main^!' in the above example is parsed by handle_revision_arg(), and at lease two different objects will be appended to revs.pending, one points to the parent commit of the "main" branch, and the other points to the tip commit of the "main" branch. These two objects have the same name "main". Only one object is left with the name "main" after calling the function object_array_remove_duplicates().

And what's worse, when adding boundary commits into pending list, we use one-line commit message as names, and the arbitory names may surprise git-bundle.

Only comparing objects themselves (".item") is also not good enough, because user may want to create a bundle with two identical objects but with different reference names, such as: "HEAD" and "refs/heads/main".

Add new function contains_object() which compare both the address and the name of the object.

---
 object.c               | 10 ++++++----
 t/t6020-bundle-misc.sh |  2 +-
 2 files changed, 7 insertions(+), 5 deletions(-)
object.c +6 -4
index 68f80b0b3d..98017bed8e 100644
@@ -412,15 +412,16 @@ void object_array_clear(struct object_array *array)
 }
 
 /*
- * Return true iff array already contains an entry with name.
+ * Return true if array already contains an entry.
  */
-static int contains_name(struct object_array *array, const char *name)
+static int contains_object(struct object_array *array,
+			   const struct object *item, const char *name)
 {
 	unsigned nr = array->nr, i;
 	struct object_array_entry *object = array->objects;
 
 	for (i = 0; i < nr; i++, object++)
-		if (!strcmp(object->name, name))
+		if (item == object->item && !strcmp(object->name, name))
 			return 1;
 	return 0;
 }
@@ -432,7 +433,8 @@ void object_array_remove_duplicates(struct object_array *array)
 
 	array->nr = 0;
 	for (src = 0; src < nr; src++) {
-		if (!contains_name(array, objects[src].name)) {
+		if (!contains_object(array, objects[src].item,
+				     objects[src].name)) {
 			if (src != array->nr)
 				objects[array->nr] = objects[src];
 			array->nr++;
t/t6020-bundle-misc.sh +1 -1
index 201f34b5c3..b554538e00 100755
@@ -167,7 +167,7 @@ test_expect_success 'setup' '
 	test_commit_setvar P "Commit P" main.txt
 '
 
-test_expect_failure 'create bundle from special rev: main^!' '
+test_expect_success 'create bundle from special rev: main^!' '
 	git bundle create special-rev.bdl "main^!" &&
 
 	git bundle list-heads special-rev.bdl |
Signature · 1 line
2.28.0.15.gba9e81f0bd
author
#51 lore raw

PATCH v6 3/3 bundle: arguments can be read from stdin

In reply to Junio C Hamano

From: Jiang Xin <[email protected]>

In order to create an incremental bundle, we need to pass many arguments to let git-bundle ignore some already packed commits. It will be more convenient to pass args via stdin. But the current implementation does not allow us to do this.

This is because args are parsed twice when creating bundle. The first time for parsing args is in compute_and_write_prerequisites() by running git-rev-list command to write prerequisites in bundle file, and stdin is consumed in this step if "--stdin" option is provided for git-bundle. Later nothing can be read from stdin when running setup_revisions() in create_bundle().

The solution is to parse args once by removing the entire function compute_and_write_prerequisites() and then calling function setup_revisions(). In order to write prerequisites for bundle, will call prepare_revision_walk() and traverse_commit_list(). But after calling prepare_revision_walk(), the object array revs.pending is left empty, and the following steps could not work properly with the empty object array (revs.pending). Therefore, make a copy of revs to revs_copy for later use right after calling setup_revisions().

The copy of revs_copy is not a deep copy, it shares the same objects with revs. The object array of revs has been cleared, but objects themselves are still kept. Flags of objects may change after calling prepare_revision_walk(), we can use these changed flags without calling the git rev-list command and parsing its output like the former implementation.

Also add testcases for git bundle in t6020, which read args from stdin.

---
 bundle.c                | 109 ++++++++++++++++++++++------------------
 t/t5607-clone-bundle.sh |   4 +-
 t/t6020-bundle-misc.sh  |  77 ++++++++++++++++++++++++++--
 3 files changed, 134 insertions(+), 56 deletions(-)
bundle.c +59 -50
index cb0e5931ac..693d619551 100644
@@ -338,48 +338,6 @@ static int write_pack_data(int bundle_fd, struct rev_info *revs, struct strvec *
 	return 0;
 }
 
-static int compute_and_write_prerequisites(int bundle_fd,
-					   struct rev_info *revs,
-					   int argc, const char **argv)
-{
-	struct child_process rls = CHILD_PROCESS_INIT;
-	struct strbuf buf = STRBUF_INIT;
-	FILE *rls_fout;
-	int i;
-
-	strvec_pushl(&rls.args,
-		     "rev-list", "--boundary", "--pretty=oneline",
-		     NULL);
-	for (i = 1; i < argc; i++)
-		strvec_push(&rls.args, argv[i]);
-	rls.out = -1;
-	rls.git_cmd = 1;
-	if (start_command(&rls))
-		return -1;
-	rls_fout = xfdopen(rls.out, "r");
-	while (strbuf_getwholeline(&buf, rls_fout, '\n') != EOF) {
-		struct object_id oid;
-		if (buf.len > 0 && buf.buf[0] == '-') {
-			write_or_die(bundle_fd, buf.buf, buf.len);
-			if (!get_oid_hex(buf.buf + 1, &oid)) {
-				struct object *object = parse_object_or_die(&oid,
-									    buf.buf);
-				object->flags |= UNINTERESTING;
-				add_pending_object(revs, object, buf.buf);
-			}
-		} else if (!get_oid_hex(buf.buf, &oid)) {
-			struct object *object = parse_object_or_die(&oid,
-								    buf.buf);
-			object->flags |= SHOWN;
-		}
-	}
-	strbuf_release(&buf);
-	fclose(rls_fout);
-	if (finish_command(&rls))
-		return error(_("rev-list died"));
-	return 0;
-}
-
 /*
  * Write out bundle refs based on the tips already
  * parsed into revs.pending. As a side effect, may
@@ -474,6 +432,38 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs)
 	return ref_count;
 }
 
+struct bundle_prerequisites_info {
+	struct object_array *pending;
+	int fd;
+};
+
+static void write_bundle_prerequisites(struct commit *commit, void *data)
+{
+	struct bundle_prerequisites_info *bpi = data;
+	struct object *object;
+	struct pretty_print_context ctx = { 0 };
+	struct strbuf buf = STRBUF_INIT;
+
+	if (!(commit->object.flags & BOUNDARY))
+		return;
+	strbuf_addf(&buf, "-%s ", oid_to_hex(&commit->object.oid));
+	write_or_die(bpi->fd, buf.buf, buf.len);
+
+	ctx.fmt = CMIT_FMT_ONELINE;
+	ctx.output_encoding = get_log_output_encoding();
+	strbuf_reset(&buf);
+	pretty_print_commit(&ctx, commit, &buf);
+	strbuf_trim(&buf);
+
+	object = (struct object *)commit;
+	object->flags |= UNINTERESTING;
+	add_object_array_with_path(object, buf.buf, bpi->pending, S_IFINVALID,
+				   NULL);
+	strbuf_addch(&buf, '\n');
+	write_or_die(bpi->fd, buf.buf, buf.len);
+	strbuf_release(&buf);
+}
+
 int create_bundle(struct repository *r, const char *path,
 		  int argc, const char **argv, struct strvec *pack_options, int version)
 {
@@ -481,8 +471,10 @@ int create_bundle(struct repository *r, const char *path,
 	int bundle_fd = -1;
 	int bundle_to_stdout;
 	int ref_count = 0;
-	struct rev_info revs;
+	struct rev_info revs, revs_copy;
 	int min_version = the_hash_algo == &hash_algos[GIT_HASH_SHA1] ? 2 : 3;
+	struct bundle_prerequisites_info bpi;
+	int i;
 
 	bundle_to_stdout = !strcmp(path, "-");
 	if (bundle_to_stdout)
@@ -512,10 +504,6 @@ int create_bundle(struct repository *r, const char *path,
 	save_commit_buffer = 0;
 	repo_init_revisions(r, &revs, NULL);
 
-	/* write prerequisites */
-	if (compute_and_write_prerequisites(bundle_fd, &revs, argc, argv))
-		goto err;
-
 	argc = setup_revisions(argc, argv, &revs, NULL);
 
 	if (argc > 1) {
@@ -523,16 +511,37 @@ int create_bundle(struct repository *r, const char *path,
 		goto err;
 	}
 
-	object_array_remove_duplicates(&revs.pending);
+	/* save revs.pending in revs_copy for later use */
+	memcpy(&revs_copy, &revs, sizeof(revs));
+	revs_copy.pending.nr = 0;
+	revs_copy.pending.alloc = 0;
+	revs_copy.pending.objects = NULL;
+	for (i = 0; i < revs.pending.nr; i++) {
+		struct object_array_entry *e = revs.pending.objects + i;
+		if (e)
+			add_object_array_with_path(e->item, e->name,
+						   &revs_copy.pending,
+						   e->mode, e->path);
+	}
 
-	ref_count = write_bundle_refs(bundle_fd, &revs);
+	/* write prerequisites */
+	revs.boundary = 1;
+	if (prepare_revision_walk(&revs))
+		die("revision walk setup failed");
+	bpi.fd = bundle_fd;
+	bpi.pending = &revs_copy.pending;
+	traverse_commit_list(&revs, write_bundle_prerequisites, NULL, &bpi);
+	object_array_remove_duplicates(&revs_copy.pending);
+
+	/* write bundle refs */
+	ref_count = write_bundle_refs(bundle_fd, &revs_copy);
 	if (!ref_count)
 		die(_("Refusing to create empty bundle."));
 	else if (ref_count < 0)
 		goto err;
 
 	/* write pack */
-	if (write_pack_data(bundle_fd, &revs, pack_options))
+	if (write_pack_data(bundle_fd, &revs_copy, pack_options))
 		goto err;
 
 	if (!bundle_to_stdout) {
t/t5607-clone-bundle.sh +2 -2
index 26985f4b44..425258767d 100755
@@ -38,13 +38,13 @@ test_expect_success 'die if bundle file cannot be created' '
 	test_must_fail git bundle create adir --all
 '
 
-test_expect_failure 'bundle --stdin' '
+test_expect_success 'bundle --stdin' '
 	echo master | git bundle create stdin-bundle.bdl --stdin &&
 	git ls-remote stdin-bundle.bdl >output &&
 	grep master output
 '
 
-test_expect_failure 'bundle --stdin <rev-list options>' '
+test_expect_success 'bundle --stdin <rev-list options>' '
 	echo master | git bundle create hybrid-bundle.bdl --stdin tag &&
 	git ls-remote hybrid-bundle.bdl >output &&
 	grep master output
t/t6020-bundle-misc.sh +73 -4

143 lines not on this page · raw message

Signature · 1 line
2.28.0.15.gba9e81f0bd
author
#52 lore raw

PATCH v5 2/3 bundle: lost objects when removing duplicate pendings

In reply to Junio C Hamano

From: Jiang Xin <[email protected]>

git rev-list will list one commit for the following command:

    $ git rev-list 'main^!'
    <tip-commit-of-main-branch>

But providing the same rev-list args to git bundle, fail to create a bundle file.

    $ git bundle create - 'main^!'
    # v2 git bundle
    -<OID> <one-line-message>

fatal: Refusing to create empty bundle.

This is because when removing duplicate objects in function object_array_remove_duplicates(), one unique pending object which has the same name is deleted by mistake. The revision arg 'main^!' in the above example is parsed by handle_revision_arg(), and at lease two different objects will be appended to revs.pending, one points to the parent commit of the "main" branch, and the other points to the tip commit of the "main" branch. These two objects have the same name "main". Only one object is left with the name "main" after calling the function object_array_remove_duplicates().

And what's worse, when adding boundary commits into pending list, we use one-line commit message as names, and the arbitory names may surprise git-bundle.

Only comparing objects themselves (".item") is also not good enough, because user may want to create a bundle with two identical objects but with different reference names, such as: "HEAD" and "refs/heads/main".

Add new function contains_object() which compare both the address and the name of the object.

---
 object.c               | 10 ++++++----
 t/t6020-bundle-misc.sh |  2 +-
 2 files changed, 7 insertions(+), 5 deletions(-)
object.c +6 -4
index 68f80b0b3d..98017bed8e 100644
@@ -412,15 +412,16 @@ void object_array_clear(struct object_array *array)
 }
 
 /*
- * Return true iff array already contains an entry with name.
+ * Return true if array already contains an entry.
  */
-static int contains_name(struct object_array *array, const char *name)
+static int contains_object(struct object_array *array,
+			   const struct object *item, const char *name)
 {
 	unsigned nr = array->nr, i;
 	struct object_array_entry *object = array->objects;
 
 	for (i = 0; i < nr; i++, object++)
-		if (!strcmp(object->name, name))
+		if (item == object->item && !strcmp(object->name, name))
 			return 1;
 	return 0;
 }
@@ -432,7 +433,8 @@ void object_array_remove_duplicates(struct object_array *array)
 
 	array->nr = 0;
 	for (src = 0; src < nr; src++) {
-		if (!contains_name(array, objects[src].name)) {
+		if (!contains_object(array, objects[src].item,
+				     objects[src].name)) {
 			if (src != array->nr)
 				objects[array->nr] = objects[src];
 			array->nr++;
t/t6020-bundle-misc.sh +1 -1
index 637cdb5a8e..5c9d5ed94d 100755
@@ -169,7 +169,7 @@ test_expect_success 'setup' '
 	test_commit_setvar P "Commit P" main.txt
 '
 
-test_expect_failure 'create bundle from special rev: main^!' '
+test_expect_success 'create bundle from special rev: main^!' '
 	git bundle create special-rev.bdl "main^!" &&
 
 	git bundle list-heads special-rev.bdl |
Signature · 1 line
2.30.0.2.g06d2f50715
#53 lore raw

Re: [PATCH v5 2/3] bundle: lost objects when removing duplicate pendings

Quoted 4 lines · Jiang Xin

/*
- * Return true iff array already contains an entry with name.
+ * Return true if array already contains an entry. */

I think the original deliberately spells "if and only if", but a natural reading of "return true if X" would be "this would never return true if not X", so perhaps the above change is OK.

Quoted 28 lines
object.c +5 -3
-static int contains_name(struct object_array *array, const char *name)
+static int contains_object(struct object_array *array,
+			   const struct object *item, const char *name)
 {
 	unsigned nr = array->nr, i;
 	struct object_array_entry *object = array->objects;
 
 	for (i = 0; i < nr; i++, object++)
-		if (!strcmp(object->name, name))
+		if (item == object->item && !strcmp(object->name, name))
 			return 1;
 	return 0;
 }
@@ -432,7 +433,8 @@ void object_array_remove_duplicates(struct object_array *array)
 
 	array->nr = 0;
 	for (src = 0; src < nr; src++) {
-		if (!contains_name(array, objects[src].name)) {
+		if (!contains_object(array, objects[src].item,
+				     objects[src].name)) {
 			if (src != array->nr)
 				objects[array->nr] = objects[src];
 			array->nr++;
t/t6020-bundle-misc.sh +0 -0
index 637cdb5a8e..5c9d5ed94d 100755

t/t6020-bundle-misc.sh test_expect_success 'setup' ' · line 175

Quoted 8 lines
t/t6020-bundle-misc.sh +1 -1
@@ -169,7 +169,7 @@ test_expect_success 'setup' '
 	test_commit_setvar P "Commit P" main.txt
 '
 
-test_expect_failure 'create bundle from special rev: main^!' '
+test_expect_success 'create bundle from special rev: main^!' '
 	git bundle create special-rev.bdl "main^!" &&
 
 	git bundle list-heads special-rev.bdl |

n

author
#54 lore raw

PATCH v5 3/3 bundle: arguments can be read from stdin

In reply to Junio C Hamano

From: Jiang Xin <[email protected]>

In order to create an incremental bundle, we need to pass many arguments to let git-bundle ignore some already packed commits. It will be more convenient to pass args via stdin. But the current implementation does not allow us to do this.

This is because args are parsed twice when creating bundle. The first time for parsing args is in compute_and_write_prerequisites() by running git-rev-list command to write prerequisites in bundle file, and stdin is consumed in this step if "--stdin" option is provided for git-bundle. Later nothing can be read from stdin when running setup_revisions() in create_bundle().

The solution is to parse args once by removing the entire function compute_and_write_prerequisites() and then calling function setup_revisions(). In order to write prerequisites for bundle, will call prepare_revision_walk() and traverse_commit_list(). But after calling prepare_revision_walk(), the object array revs.pending is left empty, and the following steps could not work properly with the empty object array (revs.pending). Therefore, make a copy of revs to revs_copy for later use right after calling setup_revisions().

The copy of revs_copy is not a deep copy, it shares the same objects with revs. The object array of revs has been cleared, but objects themselves are still kept. Flags of objects may change after calling prepare_revision_walk(), we can use these changed flags without calling the git rev-list command and parsing its output like the former implementation.

Also add testcases for git bundle in t6020, which read args from stdin.

---
 bundle.c                | 109 ++++++++++++++++++++++------------------
 t/t5607-clone-bundle.sh |   4 +-
 t/t6020-bundle-misc.sh  |  77 ++++++++++++++++++++++++++--
 3 files changed, 134 insertions(+), 56 deletions(-)
bundle.c +59 -50
index cb0e5931ac..693d619551 100644
@@ -338,48 +338,6 @@ static int write_pack_data(int bundle_fd, struct rev_info *revs, struct strvec *
 	return 0;
 }
 
-static int compute_and_write_prerequisites(int bundle_fd,
-					   struct rev_info *revs,
-					   int argc, const char **argv)
-{
-	struct child_process rls = CHILD_PROCESS_INIT;
-	struct strbuf buf = STRBUF_INIT;
-	FILE *rls_fout;
-	int i;
-
-	strvec_pushl(&rls.args,
-		     "rev-list", "--boundary", "--pretty=oneline",
-		     NULL);
-	for (i = 1; i < argc; i++)
-		strvec_push(&rls.args, argv[i]);
-	rls.out = -1;
-	rls.git_cmd = 1;
-	if (start_command(&rls))
-		return -1;
-	rls_fout = xfdopen(rls.out, "r");
-	while (strbuf_getwholeline(&buf, rls_fout, '\n') != EOF) {
-		struct object_id oid;
-		if (buf.len > 0 && buf.buf[0] == '-') {
-			write_or_die(bundle_fd, buf.buf, buf.len);
-			if (!get_oid_hex(buf.buf + 1, &oid)) {
-				struct object *object = parse_object_or_die(&oid,
-									    buf.buf);
-				object->flags |= UNINTERESTING;
-				add_pending_object(revs, object, buf.buf);
-			}
-		} else if (!get_oid_hex(buf.buf, &oid)) {
-			struct object *object = parse_object_or_die(&oid,
-								    buf.buf);
-			object->flags |= SHOWN;
-		}
-	}
-	strbuf_release(&buf);
-	fclose(rls_fout);
-	if (finish_command(&rls))
-		return error(_("rev-list died"));
-	return 0;
-}
-
 /*
  * Write out bundle refs based on the tips already
  * parsed into revs.pending. As a side effect, may
@@ -474,6 +432,38 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs)
 	return ref_count;
 }
 
+struct bundle_prerequisites_info {
+	struct object_array *pending;
+	int fd;
+};
+
+static void write_bundle_prerequisites(struct commit *commit, void *data)
+{
+	struct bundle_prerequisites_info *bpi = data;
+	struct object *object;
+	struct pretty_print_context ctx = { 0 };
+	struct strbuf buf = STRBUF_INIT;
+
+	if (!(commit->object.flags & BOUNDARY))
+		return;
+	strbuf_addf(&buf, "-%s ", oid_to_hex(&commit->object.oid));
+	write_or_die(bpi->fd, buf.buf, buf.len);
+
+	ctx.fmt = CMIT_FMT_ONELINE;
+	ctx.output_encoding = get_log_output_encoding();
+	strbuf_reset(&buf);
+	pretty_print_commit(&ctx, commit, &buf);
+	strbuf_trim(&buf);
+
+	object = (struct object *)commit;
+	object->flags |= UNINTERESTING;
+	add_object_array_with_path(object, buf.buf, bpi->pending, S_IFINVALID,
+				   NULL);
+	strbuf_addch(&buf, '\n');
+	write_or_die(bpi->fd, buf.buf, buf.len);
+	strbuf_release(&buf);
+}
+
 int create_bundle(struct repository *r, const char *path,
 		  int argc, const char **argv, struct strvec *pack_options, int version)
 {
@@ -481,8 +471,10 @@ int create_bundle(struct repository *r, const char *path,
 	int bundle_fd = -1;
 	int bundle_to_stdout;
 	int ref_count = 0;
-	struct rev_info revs;
+	struct rev_info revs, revs_copy;
 	int min_version = the_hash_algo == &hash_algos[GIT_HASH_SHA1] ? 2 : 3;
+	struct bundle_prerequisites_info bpi;
+	int i;
 
 	bundle_to_stdout = !strcmp(path, "-");
 	if (bundle_to_stdout)
@@ -512,10 +504,6 @@ int create_bundle(struct repository *r, const char *path,
 	save_commit_buffer = 0;
 	repo_init_revisions(r, &revs, NULL);
 
-	/* write prerequisites */
-	if (compute_and_write_prerequisites(bundle_fd, &revs, argc, argv))
-		goto err;
-
 	argc = setup_revisions(argc, argv, &revs, NULL);
 
 	if (argc > 1) {
@@ -523,16 +511,37 @@ int create_bundle(struct repository *r, const char *path,
 		goto err;
 	}
 
-	object_array_remove_duplicates(&revs.pending);
+	/* save revs.pending in revs_copy for later use */
+	memcpy(&revs_copy, &revs, sizeof(revs));
+	revs_copy.pending.nr = 0;
+	revs_copy.pending.alloc = 0;
+	revs_copy.pending.objects = NULL;
+	for (i = 0; i < revs.pending.nr; i++) {
+		struct object_array_entry *e = revs.pending.objects + i;
+		if (e)
+			add_object_array_with_path(e->item, e->name,
+						   &revs_copy.pending,
+						   e->mode, e->path);
+	}
 
-	ref_count = write_bundle_refs(bundle_fd, &revs);
+	/* write prerequisites */
+	revs.boundary = 1;
+	if (prepare_revision_walk(&revs))
+		die("revision walk setup failed");
+	bpi.fd = bundle_fd;
+	bpi.pending = &revs_copy.pending;
+	traverse_commit_list(&revs, write_bundle_prerequisites, NULL, &bpi);
+	object_array_remove_duplicates(&revs_copy.pending);
+
+	/* write bundle refs */
+	ref_count = write_bundle_refs(bundle_fd, &revs_copy);
 	if (!ref_count)
 		die(_("Refusing to create empty bundle."));
 	else if (ref_count < 0)
 		goto err;
 
 	/* write pack */
-	if (write_pack_data(bundle_fd, &revs, pack_options))
+	if (write_pack_data(bundle_fd, &revs_copy, pack_options))
 		goto err;
 
 	if (!bundle_to_stdout) {
t/t5607-clone-bundle.sh +2 -2
index 26985f4b44..425258767d 100755
@@ -38,13 +38,13 @@ test_expect_success 'die if bundle file cannot be created' '
 	test_must_fail git bundle create adir --all
 '
 
-test_expect_failure 'bundle --stdin' '
+test_expect_success 'bundle --stdin' '
 	echo master | git bundle create stdin-bundle.bdl --stdin &&
 	git ls-remote stdin-bundle.bdl >output &&
 	grep master output
 '
 
-test_expect_failure 'bundle --stdin <rev-list options>' '
+test_expect_success 'bundle --stdin <rev-list options>' '
 	echo master | git bundle create hybrid-bundle.bdl --stdin tag &&
 	git ls-remote hybrid-bundle.bdl >output &&
 	grep master output
t/t6020-bundle-misc.sh +73 -4

143 lines not on this page · raw message

Signature · 1 line
2.30.0.2.g06d2f50715
author
#55 lore raw

Re: [PATCH v4 1/2] bundle: lost objects when removing duplicate pendings

In reply to Junio C Hamano
Quoted 5 lines · Junio C Hamano <[email protected]> 于2021年1月9日周六 上午10:11写道:
Quoted 5 lines
t/t6020-bundle-misc.sh new file +5 -0
+# Display the pack data contained in the bundle file, bypassing the
+# header that contains the signature, prerequisites and references.
+convert_bundle_to_pack () {
+     while read x && test -n "$x"
+     do
Quoted 8 lines

t/t6020-bundle-misc.sh line 60

Quoted 4 lines
t/t6020-bundle-misc.sh new file +4 -0
+             :;
+     done
+     cat
+}

This looks somewhat familiar. Perhaps extract out necessary helpers including this one into t/test-bundle-lib or something similar in a preparatory step before this patch?

Will add a new helper "t/test-bundle-functions.sh". But as how to include this helper, there are several solutions, which is better?

1. For those scripts which use these shared functions include this helper like:

. ./test-lib.sh
# current directory changed, so add path prefix . "$TEST_DIRECTORY"/test-bundle-functions.sh

2. Include "test-bundle-functions.sh" in "test-lib.sh" like this:

. "$TEST_DIRECTORY/test-lib-functions.sh" . "$TEST_DIRECTORY/test-bundle-functions.sh"

3. Create a "t/test-lib.d/" directory, add add “test-bundle-functions.sh” inside "t/test-lib.d/" as an extension, just like chriscool introduced in sharness project.

----8<-----

 # test_perf subshells can have them too
 . "$TEST_DIRECTORY/test-lib-functions.sh"

+if test -d "$TEST_DIRECTORY/test-lib.d"
+then
+ for file in "$TEST_DIRECTORY/test-lib.d/"*.sh
+ do
+ # Ensure glob was not an empty match:
+ test -e "${file}" || break
+
+ if test -n "$debug"
+ then
+ echo >&5 "test-lib: loading extensions from ${file}"

+               fi
+               . "${file}"
+               if test $? != 0
+               then

+ echo >&5 "test-lib: Error loading ${file}. Aborting."
+ exit 1
+ fi
+ done
+fi
+
----->8-----

#56 lore raw

Re: [PATCH v4 1/2] bundle: lost objects when removing duplicate pendings

Quoted 5 lines · Jiang Xin

1. For those scripts which use these shared functions include this helper like:

. ./test-lib.sh
# current directory changed, so add path prefix . "$TEST_DIRECTORY"/test-bundle-functions.sh

This one. But without double spaces after the dot.

author
#57 lore raw

PATCH v4 2/2 bundle: arguments can be read from stdin

In reply to Đoàn Trần Công Danh

From: Jiang Xin <[email protected]>

In order to create an incremental bundle, we need to pass many arguments to let git-bundle ignore some already packed commits. It will be more convenient to pass args via stdin. But the current implementation does not allow us to do this.

This is because args are parsed twice when creating bundle. The first time for parsing args is in compute_and_write_prerequisites() by running git-rev-list command to write prerequisites in bundle file, and stdin is consumed in this step if "--stdin" option is provided for git-bundle. Later nothing can be read from stdin when running setup_revisions() in create_bundle().

The solution is to parse args once by removing the entire function compute_and_write_prerequisites() and then calling function setup_revisions(). In order to write prerequisites for bundle, will call prepare_revision_walk() and traverse_commit_list(). But after calling prepare_revision_walk(), the object array revs.pending is left empty, and the following steps could not work properly with the empty object array (revs.pending). Therefore, make a copy of revs to revs_copy for later use right after calling setup_revisions().

The copy of revs_copy is not a deep copy, it shares the same objects with revs. The object array of revs has been cleared, but objects themselves are still kept. Flags of objects may change after calling prepare_revision_walk(), we can use these changed flags without calling the git rev-list command and parsing its output like the former implementation.

Also add testcases for git bundle in t6020, which read args from stdin.

---
 bundle.c                | 111 ++++++++++++++++++++++------------------
 t/t5607-clone-bundle.sh |   4 +-
 t/t6020-bundle-misc.sh  |  77 ++++++++++++++++++++++++++--
 3 files changed, 135 insertions(+), 57 deletions(-)
bundle.c +60 -51
index cb0e5931ac..72970d962a 100644
@@ -338,48 +338,6 @@ static int write_pack_data(int bundle_fd, struct rev_info *revs, struct strvec *
 	return 0;
 }
 
-static int compute_and_write_prerequisites(int bundle_fd,
-					   struct rev_info *revs,
-					   int argc, const char **argv)
-{
-	struct child_process rls = CHILD_PROCESS_INIT;
-	struct strbuf buf = STRBUF_INIT;
-	FILE *rls_fout;
-	int i;
-
-	strvec_pushl(&rls.args,
-		     "rev-list", "--boundary", "--pretty=oneline",
-		     NULL);
-	for (i = 1; i < argc; i++)
-		strvec_push(&rls.args, argv[i]);
-	rls.out = -1;
-	rls.git_cmd = 1;
-	if (start_command(&rls))
-		return -1;
-	rls_fout = xfdopen(rls.out, "r");
-	while (strbuf_getwholeline(&buf, rls_fout, '\n') != EOF) {
-		struct object_id oid;
-		if (buf.len > 0 && buf.buf[0] == '-') {
-			write_or_die(bundle_fd, buf.buf, buf.len);
-			if (!get_oid_hex(buf.buf + 1, &oid)) {
-				struct object *object = parse_object_or_die(&oid,
-									    buf.buf);
-				object->flags |= UNINTERESTING;
-				add_pending_object(revs, object, buf.buf);
-			}
-		} else if (!get_oid_hex(buf.buf, &oid)) {
-			struct object *object = parse_object_or_die(&oid,
-								    buf.buf);
-			object->flags |= SHOWN;
-		}
-	}
-	strbuf_release(&buf);
-	fclose(rls_fout);
-	if (finish_command(&rls))
-		return error(_("rev-list died"));
-	return 0;
-}
-
 /*
  * Write out bundle refs based on the tips already
  * parsed into revs.pending. As a side effect, may
@@ -425,7 +383,7 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs)
 		 * constraints.
 		 */
 		if (!(e->item->flags & SHOWN) && e->item->type == OBJ_COMMIT) {
-			warning(_("ref '%s' is excluded by the rev-list options"),
+			warning(_("ref '%s' is excluded by the limiting options"),
 				e->name);
 			goto skip_write_ref;
 		}
@@ -474,6 +432,38 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs)
 	return ref_count;
 }
 
+struct bundle_prerequisites_info {
+	struct object_array *pending;
+	int fd;
+};
+
+static void write_bundle_prerequisites(struct commit *commit, void *data)
+{
+	struct bundle_prerequisites_info *bpi = data;
+	struct object *object;
+	struct pretty_print_context ctx = { 0 };
+	struct strbuf buf = STRBUF_INIT;
+
+	if (!(commit->object.flags & BOUNDARY))
+		return;
+	strbuf_addf(&buf, "-%s ", oid_to_hex(&commit->object.oid));
+	write_or_die(bpi->fd, buf.buf, buf.len);
+
+	ctx.fmt = CMIT_FMT_ONELINE;
+	ctx.output_encoding = get_log_output_encoding();
+	strbuf_reset(&buf);
+	pretty_print_commit(&ctx, commit, &buf);
+	strbuf_trim(&buf);
+
+	object = (struct object *)commit;
+	object->flags |= UNINTERESTING;
+	add_object_array_with_path(object, buf.buf, bpi->pending, S_IFINVALID,
+				   NULL);
+	strbuf_addch(&buf, '\n');
+	write_or_die(bpi->fd, buf.buf, buf.len);
+	strbuf_release(&buf);
+}
+
 int create_bundle(struct repository *r, const char *path,
 		  int argc, const char **argv, struct strvec *pack_options, int version)
 {
@@ -481,8 +471,10 @@ int create_bundle(struct repository *r, const char *path,
 	int bundle_fd = -1;
 	int bundle_to_stdout;
 	int ref_count = 0;
-	struct rev_info revs;
+	struct rev_info revs, revs_copy;
 	int min_version = the_hash_algo == &hash_algos[GIT_HASH_SHA1] ? 2 : 3;
+	struct bundle_prerequisites_info bpi;
+	int i;
 
 	bundle_to_stdout = !strcmp(path, "-");
 	if (bundle_to_stdout)
@@ -512,10 +504,6 @@ int create_bundle(struct repository *r, const char *path,
 	save_commit_buffer = 0;
 	repo_init_revisions(r, &revs, NULL);
 
-	/* write prerequisites */
-	if (compute_and_write_prerequisites(bundle_fd, &revs, argc, argv))
-		goto err;
-
 	argc = setup_revisions(argc, argv, &revs, NULL);
 
 	if (argc > 1) {
@@ -523,16 +511,37 @@ int create_bundle(struct repository *r, const char *path,
 		goto err;
 	}
 
-	object_array_remove_duplicates(&revs.pending);
+	/* save revs.pending in revs_copy for later use */
+	memcpy(&revs_copy, &revs, sizeof(revs));
+	revs_copy.pending.nr = 0;
+	revs_copy.pending.alloc = 0;
+	revs_copy.pending.objects = NULL;
+	for (i = 0; i < revs.pending.nr; i++) {
+		struct object_array_entry *e = revs.pending.objects + i;
+		if (e)
+			add_object_array_with_path(e->item, e->name,
+						   &revs_copy.pending,
+						   e->mode, e->path);
+	}
 
-	ref_count = write_bundle_refs(bundle_fd, &revs);
+	/* write prerequisites */
+	revs.boundary = 1;
+	if (prepare_revision_walk(&revs))
+		die("revision walk setup failed");
+	bpi.fd = bundle_fd;
+	bpi.pending = &revs_copy.pending;
+	traverse_commit_list(&revs, write_bundle_prerequisites, NULL, &bpi);
+	object_array_remove_duplicates(&revs_copy.pending);
+
+	/* write bundle refs */
+	ref_count = write_bundle_refs(bundle_fd, &revs_copy);
 	if (!ref_count)
 		die(_("Refusing to create empty bundle."));
 	else if (ref_count < 0)
 		goto err;
 
 	/* write pack */
-	if (write_pack_data(bundle_fd, &revs, pack_options))
+	if (write_pack_data(bundle_fd, &revs_copy, pack_options))
 		goto err;
 
 	if (!bundle_to_stdout) {
t/t5607-clone-bundle.sh +2 -2
index 26985f4b44..425258767d 100755
@@ -38,13 +38,13 @@ test_expect_success 'die if bundle file cannot be created' '
 	test_must_fail git bundle create adir --all
 '
 
-test_expect_failure 'bundle --stdin' '
+test_expect_success 'bundle --stdin' '
 	echo master | git bundle create stdin-bundle.bdl --stdin &&
 	git ls-remote stdin-bundle.bdl >output &&
 	grep master output
 '
 
-test_expect_failure 'bundle --stdin <rev-list options>' '
+test_expect_success 'bundle --stdin <rev-list options>' '
 	echo master | git bundle create hybrid-bundle.bdl --stdin tag &&
 	git ls-remote hybrid-bundle.bdl >output &&
 	grep master output
t/t6020-bundle-misc.sh +73 -4

143 lines not on this page · raw message

Signature · 1 line
2.30.0.2.g06d2f50715
#58 lore raw
Quoted 10 lines · Jiang Xin
t/t5607-clone-bundle.sh +1 -1
index 26985f4b44..425258767d 100755
@@ -38,13 +38,13 @@ test_expect_success 'die if bundle file cannot be created' '
 	test_must_fail git bundle create adir --all
 '
 
-test_expect_failure 'bundle --stdin' '
+test_expect_success 'bundle --stdin' '

t/t5607-clone-bundle.sh test_expect_success 'die if bundle file cannot be created' ' · line 50

Quoted 10 lines
t/t5607-clone-bundle.sh +1 -1
 	echo master | git bundle create stdin-bundle.bdl --stdin &&
 	git ls-remote stdin-bundle.bdl >output &&
 	grep master output
 '
 
-test_expect_failure 'bundle --stdin <rev-list options>' '
+test_expect_success 'bundle --stdin <rev-list options>' '
 	echo master | git bundle create hybrid-bundle.bdl --stdin tag &&
 	git ls-remote hybrid-bundle.bdl >output &&
 	grep master output

I think these tests that documented the known limitation came from f62e0a39 (t5704 (bundle): add tests for bundle --stdin, 2010-04-19).

The author(s) of the original test deserves to be notified ;-)

author
#59 lore raw

PATCH v3 2/2 bundle: arguments can be read from stdin

In reply to Junio C Hamano

From: Jiang Xin <[email protected]>

In order to create an incremental bundle, we need to pass many arguments to let git-bundle ignore some already packed commits. It will be more convenient to pass args via stdin. But the current implementation does not allow us to do this.

This is because args are parsed twice when creating bundle. The first time for parsing args is in compute_and_write_prerequisites() by running git-rev-list command to write prerequisites in bundle file, and stdin is consumed in this step if "--stdin" option is provided for git-bundle. Later nothing can be read from stdin when running setup_revisions() in create_bundle().

The solution is to parse args once by removing the entire function compute_and_write_prerequisites() and then calling function setup_revisions(). In order to write prerequisites for bundle, will call prepare_revision_walk() and traverse_commit_list(). But after calling prepare_revision_walk(), the object array revs.pending is left empty, and the following steps could not work properly with the empty object array (revs.pending). Therefore, make a copy of revs to revs_copy for later use right after calling setup_revisions().

The copy of revs_copy is not a deep copy, it shares the same objects with revs. The object array of revs has been cleared, but objects themselves are still kept. Flags of objects may change after calling prepare_revision_walk(), we can use these changed flags without calling the git rev-list command and parsing its output like the former implementation.

Also add testcases for git bundle in t6020, which read args from stdin.

---
 bundle.c                | 111 ++++++++++++++++++++++------------------
 t/t5607-clone-bundle.sh |   4 +-
 t/t6020-bundle-misc.sh  |  77 ++++++++++++++++++++++++++--
 3 files changed, 135 insertions(+), 57 deletions(-)
bundle.c +60 -51
index cb0e5931ac..72970d962a 100644
@@ -338,48 +338,6 @@ static int write_pack_data(int bundle_fd, struct rev_info *revs, struct strvec *
 	return 0;
 }
 
-static int compute_and_write_prerequisites(int bundle_fd,
-					   struct rev_info *revs,
-					   int argc, const char **argv)
-{
-	struct child_process rls = CHILD_PROCESS_INIT;
-	struct strbuf buf = STRBUF_INIT;
-	FILE *rls_fout;
-	int i;
-
-	strvec_pushl(&rls.args,
-		     "rev-list", "--boundary", "--pretty=oneline",
-		     NULL);
-	for (i = 1; i < argc; i++)
-		strvec_push(&rls.args, argv[i]);
-	rls.out = -1;
-	rls.git_cmd = 1;
-	if (start_command(&rls))
-		return -1;
-	rls_fout = xfdopen(rls.out, "r");
-	while (strbuf_getwholeline(&buf, rls_fout, '\n') != EOF) {
-		struct object_id oid;
-		if (buf.len > 0 && buf.buf[0] == '-') {
-			write_or_die(bundle_fd, buf.buf, buf.len);
-			if (!get_oid_hex(buf.buf + 1, &oid)) {
-				struct object *object = parse_object_or_die(&oid,
-									    buf.buf);
-				object->flags |= UNINTERESTING;
-				add_pending_object(revs, object, buf.buf);
-			}
-		} else if (!get_oid_hex(buf.buf, &oid)) {
-			struct object *object = parse_object_or_die(&oid,
-								    buf.buf);
-			object->flags |= SHOWN;
-		}
-	}
-	strbuf_release(&buf);
-	fclose(rls_fout);
-	if (finish_command(&rls))
-		return error(_("rev-list died"));
-	return 0;
-}
-
 /*
  * Write out bundle refs based on the tips already
  * parsed into revs.pending. As a side effect, may
@@ -425,7 +383,7 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs)
 		 * constraints.
 		 */
 		if (!(e->item->flags & SHOWN) && e->item->type == OBJ_COMMIT) {
-			warning(_("ref '%s' is excluded by the rev-list options"),
+			warning(_("ref '%s' is excluded by the limiting options"),
 				e->name);
 			goto skip_write_ref;
 		}
@@ -474,6 +432,38 @@ static int write_bundle_refs(int bundle_fd, struct rev_info *revs)
 	return ref_count;
 }
 
+struct bundle_prerequisites_info {
+	struct object_array *pending;
+	int fd;
+};
+
+static void write_bundle_prerequisites(struct commit *commit, void *data)
+{
+	struct bundle_prerequisites_info *bpi = data;
+	struct object *object;
+	struct pretty_print_context ctx = { 0 };
+	struct strbuf buf = STRBUF_INIT;
+
+	if (!(commit->object.flags & BOUNDARY))
+		return;
+	strbuf_addf(&buf, "-%s ", oid_to_hex(&commit->object.oid));
+	write_or_die(bpi->fd, buf.buf, buf.len);
+
+	ctx.fmt = CMIT_FMT_ONELINE;
+	ctx.output_encoding = get_log_output_encoding();
+	strbuf_reset(&buf);
+	pretty_print_commit(&ctx, commit, &buf);
+	strbuf_trim(&buf);
+
+	object = (struct object *)commit;
+	object->flags |= UNINTERESTING;
+	add_object_array_with_path(object, buf.buf, bpi->pending, S_IFINVALID,
+				   NULL);
+	strbuf_addch(&buf, '\n');
+	write_or_die(bpi->fd, buf.buf, buf.len);
+	strbuf_release(&buf);
+}
+
 int create_bundle(struct repository *r, const char *path,
 		  int argc, const char **argv, struct strvec *pack_options, int version)
 {
@@ -481,8 +471,10 @@ int create_bundle(struct repository *r, const char *path,
 	int bundle_fd = -1;
 	int bundle_to_stdout;
 	int ref_count = 0;
-	struct rev_info revs;
+	struct rev_info revs, revs_copy;
 	int min_version = the_hash_algo == &hash_algos[GIT_HASH_SHA1] ? 2 : 3;
+	struct bundle_prerequisites_info bpi;
+	int i;
 
 	bundle_to_stdout = !strcmp(path, "-");
 	if (bundle_to_stdout)
@@ -512,10 +504,6 @@ int create_bundle(struct repository *r, const char *path,
 	save_commit_buffer = 0;
 	repo_init_revisions(r, &revs, NULL);
 
-	/* write prerequisites */
-	if (compute_and_write_prerequisites(bundle_fd, &revs, argc, argv))
-		goto err;
-
 	argc = setup_revisions(argc, argv, &revs, NULL);
 
 	if (argc > 1) {
@@ -523,16 +511,37 @@ int create_bundle(struct repository *r, const char *path,
 		goto err;
 	}
 
-	object_array_remove_duplicates(&revs.pending);
+	/* save revs.pending in revs_copy for later use */
+	memcpy(&revs_copy, &revs, sizeof(revs));
+	revs_copy.pending.nr = 0;
+	revs_copy.pending.alloc = 0;
+	revs_copy.pending.objects = NULL;
+	for (i = 0; i < revs.pending.nr; i++) {
+		struct object_array_entry *e = revs.pending.objects + i;
+		if (e)
+			add_object_array_with_path(e->item, e->name,
+						   &revs_copy.pending,
+						   e->mode, e->path);
+	}
 
-	ref_count = write_bundle_refs(bundle_fd, &revs);
+	/* write prerequisites */
+	revs.boundary = 1;
+	if (prepare_revision_walk(&revs))
+		die("revision walk setup failed");
+	bpi.fd = bundle_fd;
+	bpi.pending = &revs_copy.pending;
+	traverse_commit_list(&revs, write_bundle_prerequisites, NULL, &bpi);
+	object_array_remove_duplicates(&revs_copy.pending);
+
+	/* write bundle refs */
+	ref_count = write_bundle_refs(bundle_fd, &revs_copy);
 	if (!ref_count)
 		die(_("Refusing to create empty bundle."));
 	else if (ref_count < 0)
 		goto err;
 
 	/* write pack */
-	if (write_pack_data(bundle_fd, &revs, pack_options))
+	if (write_pack_data(bundle_fd, &revs_copy, pack_options))
 		goto err;
 
 	if (!bundle_to_stdout) {
t/t5607-clone-bundle.sh +2 -2
index 26985f4b44..425258767d 100755
@@ -38,13 +38,13 @@ test_expect_success 'die if bundle file cannot be created' '
 	test_must_fail git bundle create adir --all
 '
 
-test_expect_failure 'bundle --stdin' '
+test_expect_success 'bundle --stdin' '
 	echo master | git bundle create stdin-bundle.bdl --stdin &&
 	git ls-remote stdin-bundle.bdl >output &&
 	grep master output
 '
 
-test_expect_failure 'bundle --stdin <rev-list options>' '
+test_expect_success 'bundle --stdin <rev-list options>' '
 	echo master | git bundle create hybrid-bundle.bdl --stdin tag &&
 	git ls-remote hybrid-bundle.bdl >output &&
 	grep master output
t/t6020-bundle-misc.sh +73 -4

143 lines not on this page · raw message

Signature · 1 line
2.30.0.2.g06d2f50715