Skip to contentSkip to conversation

pack-redundant: new algorithm to find min packs

PATCH Superseded Jiang Xin 8 participants 83 messages Dec 18, 2018 – Feb 2, 2019

lore
Share & tools
lore

Jiang Xin

author
#0 lore raw

PATCH 1/2 pack-redundant: new algorithm to find min packs

From: Sun Chao <[email protected]>

When calling git pack-redundant --all, if there are too many local packs and too many redundant objects within them, the too deep iteration of get_permutations will exhaust all the resources, and the process of git pack-redundant will be killed.

The following script could create a repository with too many redundant packs, and running git pack-redundant --all in the test.git repo will die soon.

    #!/bin/sh

    repo="$(pwd)/test.git"
    work="$(pwd)/test"
    i=1
    max=199

    if test -d "$repo" || test -d "$work"; then
    	echo >&2 "ERROR: '$repo' or '$work' already exist"
    	exit 1
    fi

    git init -q --bare "$repo"
    git --git-dir="$repo" config gc.auto 0
    git --git-dir="$repo" config transfer.unpackLimit 0
    git clone -q "$repo" "$work" 2>/dev/null

    while :; do
        cd "$work"
        echo "loop $i: $(date +%s)" >$i
        git add $i
        git commit -q -sm "loop $i"
        git push -q origin HEAD:master
        printf "\rCreate pack %4d/%d\t" $i $max
        if test $i -ge $max; then break; fi

        cd "$repo"
        git repack -q
        if test $(($i % 2)) -eq 0; then
            git repack -aq
            pack=$(ls -t $repo/objects/pack/*.pack | head -1)
            touch "${pack%.pack}.keep"
        fi
        i=$((i+1))
    done
    printf "\ndone\n"

To get the min unique pack list, we can replace the iteration in minimize function with a new algorithm, and this could solve this issue:

1. Get the unique and non_uniqe packs, add the unique packs to the min list.

2. Remove the objects of unique packs from non_unique packs, then each object left in the non_unique packs will have at least two copies.

3. Sort the non_unique packs by the objects' size, more objects first, and add the first non_unique pack to min list.

4. Drop the duplicated objects from other packs in the ordered non_unique pack list, and repeat step 3.

Original PR and discussions: https://github.com/jiangxin/git/pull/25

---
 builtin/pack-redundant.c | 116 ++++++++++++++++++++++++---------------
 1 file changed, 73 insertions(+), 43 deletions(-)
builtin/pack-redundant.c +73 -43
index cf9a9aabd4..19dcf74750 100644
@@ -421,14 +421,52 @@ static inline off_t pack_set_bytecount(struct pack_list *pl)
 	return ret;
 }
 
-static void minimize(struct pack_list **min)
+static int cmp_pack_list_reverse(const void *a, const void *b)
 {
-	struct pack_list *pl, *unique = NULL,
-		*non_unique = NULL, *min_perm = NULL;
-	struct pll *perm, *perm_all, *perm_ok = NULL, *new_perm;
-	struct llist *missing;
-	off_t min_perm_size = 0, perm_size;
-	int n;
+	struct pack_list *pl_a = *((struct pack_list **)a);
+	struct pack_list *pl_b = *((struct pack_list **)b);
+	size_t sz_a = pl_a->all_objects->size;
+	size_t sz_b = pl_b->all_objects->size;
+
+	if (sz_a == sz_b)
+		return 0;
+	else if (sz_a < sz_b)
+		return 1;
+	else
+		return -1;
+}
+
+/* Sort pack_list, greater size of all_objects first */
+static void sort_pack_list(struct pack_list **pl)
+{
+	struct pack_list **ary, *p;
+	int i;
+	size_t n = pack_list_size(*pl);
+
+	if (n < 2)
+		return;
+
+	/* prepare an array of packed_list for easier sorting */
+	ary = xcalloc(n, sizeof(struct pack_list *));
+	for (n = 0, p = *pl; p; p = p->next)
+		ary[n++] = p;
+
+	QSORT(ary, n, cmp_pack_list_reverse);
+
+	/* link them back again */
+	for (i = 0; i < n - 1; i++)
+		ary[i]->next = ary[i + 1];
+	ary[n - 1]->next = NULL;
+	*pl = ary[0];
+
+	free(ary);
+}
+
+
+static void minimize(struct pack_list **min, struct llist *ignore)
+{
+	struct pack_list *pl, *unique = NULL, *non_unique = NULL;
+	struct llist *missing, *unique_pack_objects;
 
 	pl = local_packs;
 	while (pl) {
@@ -446,49 +484,41 @@ static void minimize(struct pack_list **min)
 		pl = pl->next;
 	}
 
+	*min = unique;
+
 	/* return if there are no objects missing from the unique set */
 	if (missing->size == 0) {
-		*min = unique;
 		free(missing);
 		return;
 	}
 
-	/* find the permutations which contain all missing objects */
-	for (n = 1; n <= pack_list_size(non_unique) && !perm_ok; n++) {
-		perm_all = perm = get_permutations(non_unique, n);
-		while (perm) {
-			if (is_superset(perm->pl, missing)) {
-				new_perm = xmalloc(sizeof(struct pll));
-				memcpy(new_perm, perm, sizeof(struct pll));
-				new_perm->next = perm_ok;
-				perm_ok = new_perm;
-			}
-			perm = perm->next;
-		}
-		if (perm_ok)
-			break;
-		pll_free(perm_all);
-	}
-	if (perm_ok == NULL)
-		die("Internal error: No complete sets found!");
-
-	/* find the permutation with the smallest size */
-	perm = perm_ok;
-	while (perm) {
-		perm_size = pack_set_bytecount(perm->pl);
-		if (!min_perm_size || min_perm_size > perm_size) {
-			min_perm_size = perm_size;
-			min_perm = perm->pl;
-		}
-		perm = perm->next;
-	}
-	*min = min_perm;
-	/* add the unique packs to the list */
-	pl = unique;
+	unique_pack_objects = llist_copy(all_objects);
+	llist_sorted_difference_inplace(unique_pack_objects, missing);
+
+	/* remove all the ignored objects and unique pack objects from the non_unique packs */
+	pl = non_unique;
 	while (pl) {
-		pack_list_insert(min, pl);
+		llist_sorted_difference_inplace(pl->all_objects, ignore);
+		llist_sorted_difference_inplace(pl->all_objects, unique_pack_objects);
 		pl = pl->next;
 	}
+
+	while ((pl = non_unique)) {
+		/* sort the non_unique packs, greater size of all_objects first */
+		sort_pack_list(&non_unique);
+		if (non_unique->all_objects->size == 0)
+			break;
+
+		pack_list_insert(min, non_unique);
+
+		while ((pl = pl->next)) {
+			if (pl->all_objects->size == 0)
+				break;
+			llist_sorted_difference_inplace(pl->all_objects, non_unique->all_objects);
+		}
+
+		non_unique = non_unique->next;
+	}
 }
 
 static void load_all_objects(void)
@@ -603,7 +633,7 @@ static void load_all(void)
 int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 {
 	int i;
-	struct pack_list *min, *red, *pl;
+	struct pack_list *min = NULL, *red, *pl;
 	struct llist *ignore;
 	struct object_id *oid;
 	char buf[GIT_MAX_HEXSZ + 2]; /* hex hash + \n + \0 */
@@ -667,7 +697,7 @@ int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 		pl = pl->next;
 	}
 
-	minimize(&min);
+	minimize(&min, ignore);
 
 	if (verbose) {
 		fprintf(stderr, "There are %lu packs available in alt-odbs.\n",
Signature · 1 line
2.20.0.2.g660e9286fc
author
#1 lore raw

PATCH 2/2 pack-redundant: remove unused functions

From: Sun Chao <[email protected]>

Remove unused functions to find min packs, such as get_permutations, pll_free, etc.

---
 builtin/pack-redundant.c | 81 ----------------------------------------
 1 file changed, 81 deletions(-)
builtin/pack-redundant.c +0 -81
index 19dcf74750..d0ff2377f3 100644
@@ -63,15 +63,6 @@ static inline struct llist_item *llist_item_get(void)
 	return new_item;
 }
 
-static void llist_free(struct llist *list)
-{
-	while ((list->back = list->front)) {
-		list->front = list->front->next;
-		llist_item_put(list->back);
-	}
-	free(list);
-}
-
 static inline void llist_init(struct llist **list)
 {
 	*list = xmalloc(sizeof(struct llist));
@@ -285,78 +276,6 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
 	}
 }
 
-static void pll_free(struct pll *l)
-{
-	struct pll *old;
-	struct pack_list *opl;
-
-	while (l) {
-		old = l;
-		while (l->pl) {
-			opl = l->pl;
-			l->pl = opl->next;
-			free(opl);
-		}
-		l = l->next;
-		free(old);
-	}
-}
-
-/* all the permutations have to be free()d at the same time,
- * since they refer to each other
- */
-static struct pll * get_permutations(struct pack_list *list, int n)
-{
-	struct pll *subset, *ret = NULL, *new_pll = NULL;
-
-	if (list == NULL || pack_list_size(list) < n || n == 0)
-		return NULL;
-
-	if (n == 1) {
-		while (list) {
-			new_pll = xmalloc(sizeof(*new_pll));
-			new_pll->pl = NULL;
-			pack_list_insert(&new_pll->pl, list);
-			new_pll->next = ret;
-			ret = new_pll;
-			list = list->next;
-		}
-		return ret;
-	}
-
-	while (list->next) {
-		subset = get_permutations(list->next, n - 1);
-		while (subset) {
-			new_pll = xmalloc(sizeof(*new_pll));
-			new_pll->pl = subset->pl;
-			pack_list_insert(&new_pll->pl, list);
-			new_pll->next = ret;
-			ret = new_pll;
-			subset = subset->next;
-		}
-		list = list->next;
-	}
-	return ret;
-}
-
-static int is_superset(struct pack_list *pl, struct llist *list)
-{
-	struct llist *diff;
-
-	diff = llist_copy(list);
-
-	while (pl) {
-		llist_sorted_difference_inplace(diff, pl->all_objects);
-		if (diff->size == 0) { /* we're done */
-			llist_free(diff);
-			return 1;
-		}
-		pl = pl->next;
-	}
-	llist_free(diff);
-	return 0;
-}
-
 static size_t sizeof_union(struct packed_git *p1, struct packed_git *p2)
 {
 	size_t ret = 0;
Signature · 1 line
2.20.0.2.g660e9286fc
author
#2 lore raw

v2 cover pack-redundant: new algorithm to find min packs

Sun Chao is my former colleague at Huawei. He finds a bug of git-pack-redundant.

When I was in Huawei, I develop a program to manage fork tree of repositories, using alternate repo for forks to save disk spaces.

Sun Chao finds if there are too many packs and many of them overlap each other, running git pack-redundant --all will exhaust all memories and the process will be killed by kernel.

There is a script in commit log of commit 2/3, which can be used to create a repository with lots of redundant packs. Running git pack-redundant --all in it can reproduce this issue.

Updates of reroll v2:

* Add test cases in t5322.
* Fix a bug in patch 2/3.

--

Jiang Xin (1):
t5322: test cases for git-pack-redundant

Sun Chao (2):
pack-redundant: new algorithm to find min packs
pack-redundant: remove unused functions

 builtin/pack-redundant.c  | 181 ++++++++++++++------------------------
 t/t5322-pack-redundant.sh |  69 +++++++++++++++
 2 files changed, 137 insertions(+), 113 deletions(-)
 create mode 100755 t/t5322-pack-redundant.sh
Signature · 1 line
2.20.0.3.gc45e608566
author
#80 lore raw

PATCH v2 1/3 t5322: test cases for git-pack-redundant

In reply to Jiang Xin

From: Jiang Xin <[email protected]>

Add test cases for git pack-redundant to validate new algorithm for git pack-redundant.

---
 t/t5322-pack-redundant.sh | 69 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100755 t/t5322-pack-redundant.sh
t/t5322-pack-redundant.sh new file +69 -0
new file mode 100755
index 0000000000..4add9c2bb1
@@ -0,0 +1,69 @@
+#!/bin/sh
+#
+# Copyright (c) 2018 Jiang Xin
+#
+
+test_description='git redundant test'
+
+. ./test-lib.sh
+
+create_commits()
+{
+	set -e
+	parent=
+	for name in A B C D E F G H I J K L M
+	do
+		test_tick
+		T=$(git write-tree)
+		if test -z "$parent"
+		then
+			sha1=$(echo $name | git commit-tree $T)
+		else
+			sha1=$(echo $name | git commit-tree -p $parent $T)
+		fi
+		eval $name=$sha1
+		parent=$sha1
+	done
+	git update-ref refs/heads/master $M
+}
+
+create_redundant_packs()
+{
+	set -e
+	cd .git/objects/pack
+	P1=$(printf "$T\n$A\n" | git pack-objects pack 2>/dev/null)
+	P2=$(printf "$T\n$A\n$B\n$C\n$D\n$E\n" | git pack-objects pack 2>/dev/null)
+	P3=$(printf "$C\n$D\n$F\n$G\n$I\n$J\n" | git pack-objects pack 2>/dev/null)
+	P4=$(printf "$D\n$E\n$G\n$H\n$J\n$K\n" | git pack-objects pack 2>/dev/null)
+	P5=$(printf "$F\n$G\n$H\n" | git pack-objects pack 2>/dev/null)
+	P6=$(printf "$F\n$I\n$L\n" | git pack-objects pack 2>/dev/null)
+	P7=$(printf "$H\n$K\n$M\n" | git pack-objects pack 2>/dev/null)
+	P8=$(printf "$L\n$M\n" | git pack-objects pack 2>/dev/null)
+	cd -
+}
+
+# Create commits and packs
+create_commits
+create_redundant_packs
+
+test_expect_success 'clear loose objects' '
+	git prune-packed &&
+	test $(find .git/objects -type f | grep -v pack | wc -l) -eq 0
+'
+
+printf "$P1\n$P4\n$P5\n$P6\n" | sort >expected
+
+test_expect_success 'git pack-redundant --all' '
+	git pack-redundant --all | \
+		sed -e "s#^.*/pack-\(.*\)\.\(idx\|pack\)#\1#g" | \
+		sort -u >actual &&
+	test_cmp expected actual
+'
+
+test_expect_success 'remove redundant packs' '
+	git pack-redundant --all | xargs rm &&
+	git fsck &&
+	test $(git pack-redundant --all | wc -l) -eq 0
+'
+
+test_done
Signature · 1 line
2.20.0.3.gc45e608566
author
#81 lore raw

PATCH v2 2/3 pack-redundant: new algorithm to find min packs

In reply to Jiang Xin

From: Sun Chao <[email protected]>

When calling git pack-redundant --all, if there are too many local packs and too many redundant objects within them, the too deep iteration of get_permutations will exhaust all the resources, and the process of git pack-redundant will be killed.

The following script could create a repository with too many redundant packs, and running git pack-redundant --all in the test.git repo will die soon.

    #!/bin/sh

    repo="$(pwd)/test.git"
    work="$(pwd)/test"
    i=1
    max=199

    if test -d "$repo" || test -d "$work"; then
    	echo >&2 "ERROR: '$repo' or '$work' already exist"
    	exit 1
    fi

    git init -q --bare "$repo"
    git --git-dir="$repo" config gc.auto 0
    git --git-dir="$repo" config transfer.unpackLimit 0
    git clone -q "$repo" "$work" 2>/dev/null

    while :; do
        cd "$work"
        echo "loop $i: $(date +%s)" >$i
        git add $i
        git commit -q -sm "loop $i"
        git push -q origin HEAD:master
        printf "\rCreate pack %4d/%d\t" $i $max
        if test $i -ge $max; then break; fi

        cd "$repo"
        git repack -q
        if test $(($i % 2)) -eq 0; then
            git repack -aq
            pack=$(ls -t $repo/objects/pack/*.pack | head -1)
            touch "${pack%.pack}.keep"
        fi
        i=$((i+1))
    done
    printf "\ndone\n"

To get the min unique pack list, we can replace the iteration in minimize function with a new algorithm, and this could solve this issue:

1. Get the unique and non_uniqe packs, add the unique packs to the min list.

2. Remove the objects of unique packs from non_unique packs, then each object left in the non_unique packs will have at least two copies.

3. Sort the non_unique packs by the objects' size, more objects first, and add the first non_unique pack to min list.

4. Drop the duplicated objects from other packs in the ordered non_unique pack list, and repeat step 3.

Original PR and discussions: https://github.com/jiangxin/git/pull/25

---
 builtin/pack-redundant.c | 109 ++++++++++++++++++++++++---------------
 1 file changed, 68 insertions(+), 41 deletions(-)
builtin/pack-redundant.c +68 -41
index cf9a9aabd4..3655cc7dc6 100644
@@ -421,14 +421,52 @@ static inline off_t pack_set_bytecount(struct pack_list *pl)
 	return ret;
 }
 
+static int cmp_pack_list_reverse(const void *a, const void *b)
+{
+	struct pack_list *pl_a = *((struct pack_list **)a);
+	struct pack_list *pl_b = *((struct pack_list **)b);
+	size_t sz_a = pl_a->all_objects->size;
+	size_t sz_b = pl_b->all_objects->size;
+
+	if (sz_a == sz_b)
+		return 0;
+	else if (sz_a < sz_b)
+		return 1;
+	else
+		return -1;
+}
+
+/* Sort pack_list, greater size of all_objects first */
+static void sort_pack_list(struct pack_list **pl)
+{
+	struct pack_list **ary, *p;
+	int i;
+	size_t n = pack_list_size(*pl);
+
+	if (n < 2)
+		return;
+
+	/* prepare an array of packed_list for easier sorting */
+	ary = xcalloc(n, sizeof(struct pack_list *));
+	for (n = 0, p = *pl; p; p = p->next)
+		ary[n++] = p;
+
+	QSORT(ary, n, cmp_pack_list_reverse);
+
+	/* link them back again */
+	for (i = 0; i < n - 1; i++)
+		ary[i]->next = ary[i + 1];
+	ary[n - 1]->next = NULL;
+	*pl = ary[0];
+
+	free(ary);
+}
+
+
 static void minimize(struct pack_list **min)
 {
-	struct pack_list *pl, *unique = NULL,
-		*non_unique = NULL, *min_perm = NULL;
-	struct pll *perm, *perm_all, *perm_ok = NULL, *new_perm;
-	struct llist *missing;
-	off_t min_perm_size = 0, perm_size;
-	int n;
+	struct pack_list *pl, *unique = NULL, *non_unique = NULL;
+	struct llist *missing, *unique_pack_objects;
 
 	pl = local_packs;
 	while (pl) {
@@ -446,49 +484,37 @@ static void minimize(struct pack_list **min)
 		pl = pl->next;
 	}
 
+	*min = unique;
+
 	/* return if there are no objects missing from the unique set */
 	if (missing->size == 0) {
-		*min = unique;
 		free(missing);
 		return;
 	}
 
-	/* find the permutations which contain all missing objects */
-	for (n = 1; n <= pack_list_size(non_unique) && !perm_ok; n++) {
-		perm_all = perm = get_permutations(non_unique, n);
-		while (perm) {
-			if (is_superset(perm->pl, missing)) {
-				new_perm = xmalloc(sizeof(struct pll));
-				memcpy(new_perm, perm, sizeof(struct pll));
-				new_perm->next = perm_ok;
-				perm_ok = new_perm;
-			}
-			perm = perm->next;
-		}
-		if (perm_ok)
-			break;
-		pll_free(perm_all);
-	}
-	if (perm_ok == NULL)
-		die("Internal error: No complete sets found!");
-
-	/* find the permutation with the smallest size */
-	perm = perm_ok;
-	while (perm) {
-		perm_size = pack_set_bytecount(perm->pl);
-		if (!min_perm_size || min_perm_size > perm_size) {
-			min_perm_size = perm_size;
-			min_perm = perm->pl;
-		}
-		perm = perm->next;
-	}
-	*min = min_perm;
-	/* add the unique packs to the list */
-	pl = unique;
+	unique_pack_objects = llist_copy(all_objects);
+	llist_sorted_difference_inplace(unique_pack_objects, missing);
+
+	/* remove unique pack objects from the non_unique packs */
+	pl = non_unique;
 	while (pl) {
-		pack_list_insert(min, pl);
+		llist_sorted_difference_inplace(pl->all_objects, unique_pack_objects);
 		pl = pl->next;
 	}
+
+	while (non_unique) {
+		/* sort the non_unique packs, greater size of all_objects first */
+		sort_pack_list(&non_unique);
+		if (non_unique->all_objects->size == 0)
+			break;
+
+		pack_list_insert(min, non_unique);
+
+		for (pl = non_unique->next; pl && pl->all_objects->size > 0;  pl = pl->next)
+			llist_sorted_difference_inplace(pl->all_objects, non_unique->all_objects);
+
+		non_unique = non_unique->next;
+	}
 }
 
 static void load_all_objects(void)
@@ -603,7 +629,7 @@ static void load_all(void)
 int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 {
 	int i;
-	struct pack_list *min, *red, *pl;
+	struct pack_list *min = NULL, *red, *pl;
 	struct llist *ignore;
 	struct object_id *oid;
 	char buf[GIT_MAX_HEXSZ + 2]; /* hex hash + \n + \0 */
@@ -664,6 +690,7 @@ int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 	pl = local_packs;
 	while (pl) {
 		llist_sorted_difference_inplace(pl->unique_objects, ignore);
+		llist_sorted_difference_inplace(pl->all_objects, ignore);
 		pl = pl->next;
 	}
 
Signature · 1 line
2.20.0.3.gc45e608566
author
#82 lore raw

PATCH v2 3/3 pack-redundant: remove unused functions

In reply to Jiang Xin

From: Sun Chao <[email protected]>

Remove unused functions to find min packs, such as get_permutations, pll_free, etc.

---
 builtin/pack-redundant.c | 72 ----------------------------------------
 1 file changed, 72 deletions(-)
builtin/pack-redundant.c +0 -72
index 3655cc7dc6..9630117c90 100644
@@ -285,78 +285,6 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
 	}
 }
 
-static void pll_free(struct pll *l)
-{
-	struct pll *old;
-	struct pack_list *opl;
-
-	while (l) {
-		old = l;
-		while (l->pl) {
-			opl = l->pl;
-			l->pl = opl->next;
-			free(opl);
-		}
-		l = l->next;
-		free(old);
-	}
-}
-
-/* all the permutations have to be free()d at the same time,
- * since they refer to each other
- */
-static struct pll * get_permutations(struct pack_list *list, int n)
-{
-	struct pll *subset, *ret = NULL, *new_pll = NULL;
-
-	if (list == NULL || pack_list_size(list) < n || n == 0)
-		return NULL;
-
-	if (n == 1) {
-		while (list) {
-			new_pll = xmalloc(sizeof(*new_pll));
-			new_pll->pl = NULL;
-			pack_list_insert(&new_pll->pl, list);
-			new_pll->next = ret;
-			ret = new_pll;
-			list = list->next;
-		}
-		return ret;
-	}
-
-	while (list->next) {
-		subset = get_permutations(list->next, n - 1);
-		while (subset) {
-			new_pll = xmalloc(sizeof(*new_pll));
-			new_pll->pl = subset->pl;
-			pack_list_insert(&new_pll->pl, list);
-			new_pll->next = ret;
-			ret = new_pll;
-			subset = subset->next;
-		}
-		list = list->next;
-	}
-	return ret;
-}
-
-static int is_superset(struct pack_list *pl, struct llist *list)
-{
-	struct llist *diff;
-
-	diff = llist_copy(list);
-
-	while (pl) {
-		llist_sorted_difference_inplace(diff, pl->all_objects);
-		if (diff->size == 0) { /* we're done */
-			llist_free(diff);
-			return 1;
-		}
-		pl = pl->next;
-	}
-	llist_free(diff);
-	return 0;
-}
-
 static size_t sizeof_union(struct packed_git *p1, struct packed_git *p2)
 {
 	size_t ret = 0;
Signature · 1 line
2.20.0.3.gc45e608566
author
#3 lore raw

v3 cover pack-redundant: new algorithm to find min packs

In reply to Jiang Xin

Sun Chao (my former colleague at Huawei) found a bug of git-pack-redundant. If there are too many packs and many of them overlap each other, running git pack-redundant --all will exhaust all memories and the process will be killed by kernel.

There is a script in commit log of commit 2/3, which can be used to create a repository with lots of redundant packs. Running git pack-redundant --all in it can reproduce this issue.

Updates of reroll v3:

* Rename test case file from t5322 to t5323, for I see t5322 exist in commit 404dead121: "pack-objects: add --sparse option".

Jiang Xin (1):
t5323: test cases for git-pack-redundant

Sun Chao (2):
pack-redundant: new algorithm to find min packs
pack-redundant: remove unused functions

 builtin/pack-redundant.c  | 181 +++++++++++++++++-----------------------------
 t/t5323-pack-redundant.sh |  84 +++++++++++++++++++++
 2 files changed, 152 insertions(+), 113 deletions(-)
 create mode 100755 t/t5323-pack-redundant.sh
Signature · 1 line
2.14.5.agit.2
author
#4 lore raw

PATCH v3 1/3 t5323: test cases for git-pack-redundant

In reply to Jiang Xin

From: Jiang Xin <[email protected]>

Add test cases for git pack-redundant to validate new algorithm for git pack-redundant.

---
 t/t5323-pack-redundant.sh | 84 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)
 create mode 100755 t/t5323-pack-redundant.sh
t/t5323-pack-redundant.sh new file +84 -0
new file mode 100755
index 0000000000..ef6076f065
@@ -0,0 +1,84 @@
+#!/bin/sh
+#
+# Copyright (c) 2018 Jiang Xin
+#
+
+test_description='git redundant test'
+
+. ./test-lib.sh
+
+create_commits()
+{
+	set -e
+	parent=
+	for name in A B C D E F G H I J K L M
+	do
+		test_tick
+		T=$(git write-tree)
+		if test -z "$parent"
+		then
+			sha1=$(echo $name | git commit-tree $T)
+		else
+			sha1=$(echo $name | git commit-tree -p $parent $T)
+		fi
+		eval $name=$sha1
+		parent=$sha1
+	done
+	git update-ref refs/heads/master $M
+}
+
+create_redundant_packs()
+{
+	set -e
+	cd .git/objects/pack
+	P1=$(printf "$T\n$A\n" | git pack-objects pack 2>/dev/null)
+	P2=$(printf "$T\n$A\n$B\n$C\n$D\n$E\n" | git pack-objects pack 2>/dev/null)
+	P3=$(printf "$C\n$D\n$F\n$G\n$I\n$J\n" | git pack-objects pack 2>/dev/null)
+	P4=$(printf "$D\n$E\n$G\n$H\n$J\n$K\n" | git pack-objects pack 2>/dev/null)
+	P5=$(printf "$F\n$G\n$H\n" | git pack-objects pack 2>/dev/null)
+	P6=$(printf "$F\n$I\n$L\n" | git pack-objects pack 2>/dev/null)
+	P7=$(printf "$H\n$K\n$M\n" | git pack-objects pack 2>/dev/null)
+	P8=$(printf "$L\n$M\n" | git pack-objects pack 2>/dev/null)
+	cd -
+	eval P$P1=P1:$P1
+	eval P$P2=P2:$P2
+	eval P$P3=P3:$P3
+	eval P$P4=P4:$P4
+	eval P$P5=P5:$P5
+	eval P$P6=P6:$P6
+	eval P$P7=P7:$P7
+	eval P$P8=P8:$P8
+}
+
+# Create commits and packs
+create_commits
+create_redundant_packs
+
+test_expect_success 'clear loose objects' '
+	git prune-packed &&
+	test $(find .git/objects -type f | grep -v pack | wc -l) -eq 0
+'
+
+cat >expected <<EOF
+P1:$P1
+P4:$P4
+P5:$P5
+P6:$P6
+EOF
+
+test_expect_success 'git pack-redundant --all' '
+	git pack-redundant --all | \
+		sed -e "s#^.*/pack-\(.*\)\.\(idx\|pack\)#\1#g" | \
+		sort -u | \
+		while read p; do eval echo "\${P$p}"; done | \
+		sort > actual && \
+	test_cmp expected actual
+'
+
+test_expect_success 'remove redundant packs' '
+	git pack-redundant --all | xargs rm &&
+	git fsck &&
+	test $(git pack-redundant --all | wc -l) -eq 0
+'
+
+test_done
Signature · 1 line
2.14.5.agit.2
author
#73 lore raw

PATCH v3 2/3 pack-redundant: new algorithm to find min packs

In reply to Jiang Xin

From: Sun Chao <[email protected]>

When calling git pack-redundant --all, if there are too many local packs and too many redundant objects within them, the too deep iteration of get_permutations will exhaust all the resources, and the process of git pack-redundant will be killed.

The following script could create a repository with too many redundant packs, and running git pack-redundant --all in the test.git repo will die soon.

    #!/bin/sh

    repo="$(pwd)/test.git"
    work="$(pwd)/test"
    i=1
    max=199

    if test -d "$repo" || test -d "$work"; then
    	echo >&2 "ERROR: '$repo' or '$work' already exist"
    	exit 1
    fi

    git init -q --bare "$repo"
    git --git-dir="$repo" config gc.auto 0
    git --git-dir="$repo" config transfer.unpackLimit 0
    git clone -q "$repo" "$work" 2>/dev/null

    while :; do
        cd "$work"
        echo "loop $i: $(date +%s)" >$i
        git add $i
        git commit -q -sm "loop $i"
        git push -q origin HEAD:master
        printf "\rCreate pack %4d/%d\t" $i $max
        if test $i -ge $max; then break; fi

        cd "$repo"
        git repack -q
        if test $(($i % 2)) -eq 0; then
            git repack -aq
            pack=$(ls -t $repo/objects/pack/*.pack | head -1)
            touch "${pack%.pack}.keep"
        fi
        i=$((i+1))
    done
    printf "\ndone\n"

To get the min unique pack list, we can replace the iteration in minimize function with a new algorithm, and this could solve this issue:

1. Get the unique and non_uniqe packs, add the unique packs to the min list.

2. Remove the objects of unique packs from non_unique packs, then each object left in the non_unique packs will have at least two copies.

3. Sort the non_unique packs by the objects' size, more objects first, and add the first non_unique pack to min list.

4. Drop the duplicated objects from other packs in the ordered non_unique pack list, and repeat step 3.

Original PR and discussions: https://github.com/jiangxin/git/pull/25

---
 builtin/pack-redundant.c | 109 +++++++++++++++++++++++++++++------------------
 1 file changed, 68 insertions(+), 41 deletions(-)
builtin/pack-redundant.c +68 -41
index cf9a9aabd4..3655cc7dc6 100644
@@ -421,14 +421,52 @@ static inline off_t pack_set_bytecount(struct pack_list *pl)
 	return ret;
 }
 
+static int cmp_pack_list_reverse(const void *a, const void *b)
+{
+	struct pack_list *pl_a = *((struct pack_list **)a);
+	struct pack_list *pl_b = *((struct pack_list **)b);
+	size_t sz_a = pl_a->all_objects->size;
+	size_t sz_b = pl_b->all_objects->size;
+
+	if (sz_a == sz_b)
+		return 0;
+	else if (sz_a < sz_b)
+		return 1;
+	else
+		return -1;
+}
+
+/* Sort pack_list, greater size of all_objects first */
+static void sort_pack_list(struct pack_list **pl)
+{
+	struct pack_list **ary, *p;
+	int i;
+	size_t n = pack_list_size(*pl);
+
+	if (n < 2)
+		return;
+
+	/* prepare an array of packed_list for easier sorting */
+	ary = xcalloc(n, sizeof(struct pack_list *));
+	for (n = 0, p = *pl; p; p = p->next)
+		ary[n++] = p;
+
+	QSORT(ary, n, cmp_pack_list_reverse);
+
+	/* link them back again */
+	for (i = 0; i < n - 1; i++)
+		ary[i]->next = ary[i + 1];
+	ary[n - 1]->next = NULL;
+	*pl = ary[0];
+
+	free(ary);
+}
+
+
 static void minimize(struct pack_list **min)
 {
-	struct pack_list *pl, *unique = NULL,
-		*non_unique = NULL, *min_perm = NULL;
-	struct pll *perm, *perm_all, *perm_ok = NULL, *new_perm;
-	struct llist *missing;
-	off_t min_perm_size = 0, perm_size;
-	int n;
+	struct pack_list *pl, *unique = NULL, *non_unique = NULL;
+	struct llist *missing, *unique_pack_objects;
 
 	pl = local_packs;
 	while (pl) {
@@ -446,49 +484,37 @@ static void minimize(struct pack_list **min)
 		pl = pl->next;
 	}
 
+	*min = unique;
+
 	/* return if there are no objects missing from the unique set */
 	if (missing->size == 0) {
-		*min = unique;
 		free(missing);
 		return;
 	}
 
-	/* find the permutations which contain all missing objects */
-	for (n = 1; n <= pack_list_size(non_unique) && !perm_ok; n++) {
-		perm_all = perm = get_permutations(non_unique, n);
-		while (perm) {
-			if (is_superset(perm->pl, missing)) {
-				new_perm = xmalloc(sizeof(struct pll));
-				memcpy(new_perm, perm, sizeof(struct pll));
-				new_perm->next = perm_ok;
-				perm_ok = new_perm;
-			}
-			perm = perm->next;
-		}
-		if (perm_ok)
-			break;
-		pll_free(perm_all);
-	}
-	if (perm_ok == NULL)
-		die("Internal error: No complete sets found!");
-
-	/* find the permutation with the smallest size */
-	perm = perm_ok;
-	while (perm) {
-		perm_size = pack_set_bytecount(perm->pl);
-		if (!min_perm_size || min_perm_size > perm_size) {
-			min_perm_size = perm_size;
-			min_perm = perm->pl;
-		}
-		perm = perm->next;
-	}
-	*min = min_perm;
-	/* add the unique packs to the list */
-	pl = unique;
+	unique_pack_objects = llist_copy(all_objects);
+	llist_sorted_difference_inplace(unique_pack_objects, missing);
+
+	/* remove unique pack objects from the non_unique packs */
+	pl = non_unique;
 	while (pl) {
-		pack_list_insert(min, pl);
+		llist_sorted_difference_inplace(pl->all_objects, unique_pack_objects);
 		pl = pl->next;
 	}
+
+	while (non_unique) {
+		/* sort the non_unique packs, greater size of all_objects first */
+		sort_pack_list(&non_unique);
+		if (non_unique->all_objects->size == 0)
+			break;
+
+		pack_list_insert(min, non_unique);
+
+		for (pl = non_unique->next; pl && pl->all_objects->size > 0;  pl = pl->next)
+			llist_sorted_difference_inplace(pl->all_objects, non_unique->all_objects);
+
+		non_unique = non_unique->next;
+	}
 }
 
 static void load_all_objects(void)
@@ -603,7 +629,7 @@ static void load_all(void)
 int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 {
 	int i;
-	struct pack_list *min, *red, *pl;
+	struct pack_list *min = NULL, *red, *pl;
 	struct llist *ignore;
 	struct object_id *oid;
 	char buf[GIT_MAX_HEXSZ + 2]; /* hex hash + \n + \0 */
@@ -664,6 +690,7 @@ int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 	pl = local_packs;
 	while (pl) {
 		llist_sorted_difference_inplace(pl->unique_objects, ignore);
+		llist_sorted_difference_inplace(pl->all_objects, ignore);
 		pl = pl->next;
 	}
 
Signature · 1 line
2.14.5.agit.2
author
#74 lore raw

PATCH v3 3/3 pack-redundant: remove unused functions

In reply to Jiang Xin

From: Sun Chao <[email protected]>

Remove unused functions to find min packs, such as get_permutations, pll_free, etc.

---
 builtin/pack-redundant.c | 72 ------------------------------------------------
 1 file changed, 72 deletions(-)
builtin/pack-redundant.c +0 -72
index 3655cc7dc6..9630117c90 100644
@@ -285,78 +285,6 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
 	}
 }
 
-static void pll_free(struct pll *l)
-{
-	struct pll *old;
-	struct pack_list *opl;
-
-	while (l) {
-		old = l;
-		while (l->pl) {
-			opl = l->pl;
-			l->pl = opl->next;
-			free(opl);
-		}
-		l = l->next;
-		free(old);
-	}
-}
-
-/* all the permutations have to be free()d at the same time,
- * since they refer to each other
- */
-static struct pll * get_permutations(struct pack_list *list, int n)
-{
-	struct pll *subset, *ret = NULL, *new_pll = NULL;
-
-	if (list == NULL || pack_list_size(list) < n || n == 0)
-		return NULL;
-
-	if (n == 1) {
-		while (list) {
-			new_pll = xmalloc(sizeof(*new_pll));
-			new_pll->pl = NULL;
-			pack_list_insert(&new_pll->pl, list);
-			new_pll->next = ret;
-			ret = new_pll;
-			list = list->next;
-		}
-		return ret;
-	}
-
-	while (list->next) {
-		subset = get_permutations(list->next, n - 1);
-		while (subset) {
-			new_pll = xmalloc(sizeof(*new_pll));
-			new_pll->pl = subset->pl;
-			pack_list_insert(&new_pll->pl, list);
-			new_pll->next = ret;
-			ret = new_pll;
-			subset = subset->next;
-		}
-		list = list->next;
-	}
-	return ret;
-}
-
-static int is_superset(struct pack_list *pl, struct llist *list)
-{
-	struct llist *diff;
-
-	diff = llist_copy(list);
-
-	while (pl) {
-		llist_sorted_difference_inplace(diff, pl->all_objects);
-		if (diff->size == 0) { /* we're done */
-			llist_free(diff);
-			return 1;
-		}
-		pl = pl->next;
-	}
-	llist_free(diff);
-	return 0;
-}
-
 static size_t sizeof_union(struct packed_git *p1, struct packed_git *p2)
 {
 	size_t ret = 0;
Signature · 1 line
2.14.5.agit.2
#75 lore raw

PATCH v4 1/1 pack-redundant: remove unused functions

From: Sun Chao <[email protected]>

Remove unused functions to find min packs, such as get_permutations, pll_free, etc.

---
 builtin/pack-redundant.c | 86 ------------------------------------------------
 1 file changed, 86 deletions(-)
builtin/pack-redundant.c +0 -86
index 3655cc7..eac2350 100644
@@ -35,11 +35,6 @@ static struct pack_list {
 	struct llist *all_objects;
 } *local_packs = NULL, *altodb_packs = NULL;
 
-struct pll {
-	struct pll *next;
-	struct pack_list *pl;
-};
-
 static struct llist_item *free_nodes;
 
 static inline void llist_item_put(struct llist_item *item)
@@ -63,15 +58,6 @@ static inline struct llist_item *llist_item_get(void)
 	return new_item;
 }
 
-static void llist_free(struct llist *list)
-{
-	while ((list->back = list->front)) {
-		list->front = list->front->next;
-		llist_item_put(list->back);
-	}
-	free(list);
-}
-
 static inline void llist_init(struct llist **list)
 {
 	*list = xmalloc(sizeof(struct llist));
@@ -285,78 +271,6 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
 	}
 }
 
-static void pll_free(struct pll *l)
-{
-	struct pll *old;
-	struct pack_list *opl;
-
-	while (l) {
-		old = l;
-		while (l->pl) {
-			opl = l->pl;
-			l->pl = opl->next;
-			free(opl);
-		}
-		l = l->next;
-		free(old);
-	}
-}
-
-/* all the permutations have to be free()d at the same time,
- * since they refer to each other
- */
-static struct pll * get_permutations(struct pack_list *list, int n)
-{
-	struct pll *subset, *ret = NULL, *new_pll = NULL;
-
-	if (list == NULL || pack_list_size(list) < n || n == 0)
-		return NULL;
-
-	if (n == 1) {
-		while (list) {
-			new_pll = xmalloc(sizeof(*new_pll));
-			new_pll->pl = NULL;
-			pack_list_insert(&new_pll->pl, list);
-			new_pll->next = ret;
-			ret = new_pll;
-			list = list->next;
-		}
-		return ret;
-	}
-
-	while (list->next) {
-		subset = get_permutations(list->next, n - 1);
-		while (subset) {
-			new_pll = xmalloc(sizeof(*new_pll));
-			new_pll->pl = subset->pl;
-			pack_list_insert(&new_pll->pl, list);
-			new_pll->next = ret;
-			ret = new_pll;
-			subset = subset->next;
-		}
-		list = list->next;
-	}
-	return ret;
-}
-
-static int is_superset(struct pack_list *pl, struct llist *list)
-{
-	struct llist *diff;
-
-	diff = llist_copy(list);
-
-	while (pl) {
-		llist_sorted_difference_inplace(diff, pl->all_objects);
-		if (diff->size == 0) { /* we're done */
-			llist_free(diff);
-			return 1;
-		}
-		pl = pl->next;
-	}
-	llist_free(diff);
-	return 0;
-}
-
 static size_t sizeof_union(struct packed_git *p1, struct packed_git *p2)
 {
 	size_t ret = 0;
Signature · 1 line
2.8.1
#76 lore raw

v4 cover pack-redundant: remove unused functions

In reply to Jiang Xin

From: Sun Chao <[email protected]>

I'm particularly grateful to Junio and JiangXin for fixing the patches, and I noticed Junio send a new commit to remove more unused codes and suggest to SQUASH it.

So I create this new version of patches to do this work, I also have checked the left codes and remove a unused struct based on Junio's last commit of https://github.com/gitster/git/commits/sc/pack-redundant.

--

Sun Chao (1):
pack-redundant: remove unused functions

 builtin/pack-redundant.c | 86 ------------------------------------------------
 1 file changed, 86 deletions(-)
Signature · 1 line
2.8.1
#79 lore raw

v4 cover pack-redundant: remove unused functions

In reply to Jiang Xin

From: Sun Chao <[email protected]>

I'm particularly grateful to Junio and JiangXin for fixing the patches, and I noticed Junio send a new commit to remove more unused codes and suggest to SQUASH it.

So I create this new version of patches to do this work, I also have checked the left codes and remove a unused struct based on Junio's last commit of https://github.com/gitster/git/commits/sc/pack-redundant.

--

Sun Chao (1):
pack-redundant: remove unused functions

 builtin/pack-redundant.c | 86 ------------------------------------------------
 1 file changed, 86 deletions(-)
Signature · 1 line
2.8.1
#77 lore raw

Re: [PATCH v4 0/1] pack-redundant: remove unused functions

In reply to [email protected]
Quoted 6 lines · [email protected] writes:

From: Sun Chao <[email protected]>

I'm particularly grateful to Junio and JiangXin for fixing the patches, and I noticed Junio send a new commit to remove more unused codes and suggest to SQUASH it.

Quoted 8 lines

So I create this new version of patches to do this work, I also have checked the left codes and remove a unused struct based on Junio's last commit of https://github.com/gitster/git/commits/sc/pack-redundant.

--

Sun Chao (1):
pack-redundant: remove unused functions

Is this meant to replace [v3 3/3]?

#78 lore raw

Re: [PATCH v4 0/1] pack-redundant: remove unused functions

Quoted 12 lines
Quoted 10 lines · [email protected] writes:

From: Sun Chao <[email protected]>

I'm particularly grateful to Junio and JiangXin for fixing the patches, and I noticed Junio send a new commit to remove more unused codes and suggest to SQUASH it.

So I create this new version of patches to do this work, I also have checked the left codes and remove a unused struct based on Junio's last commit of https://github.com/gitster/git/commits/sc/pack-redundant.

Quoted 6 lines
Quoted 4 lines

--

Sun Chao (1):
pack-redundant: remove unused functions

Is this meant to replace [v3 3/3]?

I'm Sun Chao and because my huawei email account can't send email outside from company, so I used 163 email account to send new path at home. I'm sorry for not explaining that.

Yes, this is meant to replace [v3 3/3], and I have noticed the patch is applied, Thanks very much.

#5 lore raw

Re: [PATCH v3 1/3] t5323: test cases for git-pack-redundant

In reply to Jiang Xin
Quoted 17 lines · Jiang Xin

From: Jiang Xin <[email protected]>

Add test cases for git pack-redundant to validate new algorithm for git pack-redundant.

---
 t/t5323-pack-redundant.sh | 84 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)
 create mode 100755 t/t5323-pack-redundant.sh
t/t5323-pack-redundant.sh new file +0 -0
new file mode 100755
index 0000000000..ef6076f065

t/t5323-pack-redundant.sh line 6

Quoted 6 lines
t/t5323-pack-redundant.sh new file +6 -0
@@ -0,0 +1,84 @@
+#!/bin/sh
+#
+# Copyright (c) 2018 Jiang Xin
+#
+
+test_description='git redundant test'

s/redundant/pack-redundant/ ?

Quoted 4 lines
t/t5323-pack-redundant.sh new file +4 -0
+
+. ./test-lib.sh
+
+create_commits()

t/t5323-pack-redundant.sh line 20

Quoted 10 lines
t/t5323-pack-redundant.sh new file +10 -0
+{
+	set -e
+	parent=
+	for name in A B C D E F G H I J K L M
+	do
+		test_tick
+		T=$(git write-tree)
+		if test -z "$parent"
+		then
+			sha1=$(echo $name | git commit-tree $T)

There is a considerable effort going on to switch from SHA-1 to a different hash function, so please don't add any new $sha1 variable; call it $oid or $commit instead.

Quoted 25 lines
t/t5323-pack-redundant.sh new file +25 -0
+		else
+			sha1=$(echo $name | git commit-tree -p $parent $T)
+		fi
+		eval $name=$sha1
+		parent=$sha1
+	done
+	git update-ref refs/heads/master $M
+}
+
+create_redundant_packs()
+{
+	set -e
+	cd .git/objects/pack
+	P1=$(printf "$T\n$A\n" | git pack-objects pack 2>/dev/null)
+	P2=$(printf "$T\n$A\n$B\n$C\n$D\n$E\n" | git pack-objects pack 2>/dev/null)
+	P3=$(printf "$C\n$D\n$F\n$G\n$I\n$J\n" | git pack-objects pack 2>/dev/null)
+	P4=$(printf "$D\n$E\n$G\n$H\n$J\n$K\n" | git pack-objects pack 2>/dev/null)
+	P5=$(printf "$F\n$G\n$H\n" | git pack-objects pack 2>/dev/null)
+	P6=$(printf "$F\n$I\n$L\n" | git pack-objects pack 2>/dev/null)
+	P7=$(printf "$H\n$K\n$M\n" | git pack-objects pack 2>/dev/null)
+	P8=$(printf "$L\n$M\n" | git pack-objects pack 2>/dev/null)
+	cd -
+	eval P$P1=P1:$P1
+	eval P$P2=P2:$P2
+	eval P$P3=P3:$P3

t/t5323-pack-redundant.sh line 55

Quoted 10 lines
t/t5323-pack-redundant.sh new file +10 -0
+	eval P$P4=P4:$P4
+	eval P$P5=P5:$P5
+	eval P$P6=P6:$P6
+	eval P$P7=P7:$P7
+	eval P$P8=P8:$P8
+}
+
+# Create commits and packs
+create_commits
+create_redundant_packs

Please perform all setup tasks in a test_expect_success block, so we get verbose and trace output about what's going on.

Don't use 'set -e', use an &&-chain instead. To fail the test if a command in the for loop were to fail you could do something like this:

for ....
do

    do-this &&
    do-that ||
    return 1

done

t/t5323-pack-redundant.sh line 59

Quoted 4 lines
t/t5323-pack-redundant.sh new file +4 -0
+
+test_expect_success 'clear loose objects' '
+	git prune-packed &&
+	test $(find .git/objects -type f | grep -v pack | wc -l) -eq 0

Use something like

find .git/objects -type f | grep -v pack >out && test_must_be_empty out

instead, so we get an informative error message on failure.

t/t5323-pack-redundant.sh line 70

Quoted 11 lines
t/t5323-pack-redundant.sh new file +11 -0
+'
+
+cat >expected <<EOF
+P1:$P1
+P4:$P4
+P5:$P5
+P6:$P6
+EOF
+
+test_expect_success 'git pack-redundant --all' '
+	git pack-redundant --all | \

Don't run a git command (especially the particular command the test script focuses on) upstream of a pipe, because it hides the command's exit code. Use an intermediate file instead.

t/t5323-pack-redundant.sh line 71

Quoted 1 line
t/t5323-pack-redundant.sh new file +1 -0
+		sed -e "s#^.*/pack-\(.*\)\.\(idx\|pack\)#\1#g" | \

This sed command doesn't seem to work on macOS (on Travis CI), and causes the test to fail with:

  ++git pack-redundant --all
  ++sed -e 's#^.*/pack-\(.*\)\.\(idx\|pack\)#\1#g'
  ++sort -u
  ++read p
  ++sort
  ++eval echo '${P.git/objects/pack/pack-0cf5cb6afaa1bae36b8e61ca398dbe29a15bc74e.idx}'
  ./test-lib.sh: line 697: ${P.git/objects/pack/pack-0cf5cb6afaa1bae36b8e61ca398dbe29a15bc74e.idx}: bad substitution
  ++test_cmp expected actual
  ++diff -u expected actual
  --- expected    2019-01-09 01:53:45.000000000 +0000
  +++ actual      2019-01-09 01:53:45.000000000 +0000
  @@ -1,4 +0,0 @@
  -P1:24ee080366509364d04a138cd4e168dc4ff33354
  -P4:139d8b0cfe7e8970a8f3533835f90278d88de474
  -P5:23e0f02d822fa4bfe5ee63337ba5632cd7be208e
  -P6:deeb289f1749972f1cd57c3b9f359ece2361f60a
  error: last command exited with $?=1
  not ok 2 - git pack-redundant --all

I'm not sure what's wrong with it, though.

Minor nit: 'git pack-redundant' prints one filename per line, so the 'g' at the end of the 's###g' is not necessary.

t/t5323-pack-redundant.sh line 74

Quoted 3 lines
t/t5323-pack-redundant.sh new file +3 -0
+		sort -u | \
+		while read p; do eval echo "\${P$p}"; done | \
+		sort > actual && \

Style nit: no space between redirection operator and filename

Quoted 13 lines
t/t5323-pack-redundant.sh new file +10 -0
+	test_cmp expected actual
+'
+
+test_expect_success 'remove redundant packs' '
+	git pack-redundant --all | xargs rm &&
+	git fsck &&
+	test $(git pack-redundant --all | wc -l) -eq 0
+'
+
+test_done
Signature · 1 line
2.14.5.agit.2
#6 lore raw

Re: [PATCH v3 1/3] t5323: test cases for git-pack-redundant

Quoted 7 lines · SZEDER Gábor

t/t5323-pack-redundant.sh line 67

Quoted 6 lines · Jiang Xin
t/t5323-pack-redundant.sh new file +6 -0
+cat >expected <<EOF
+P1:$P1
+P4:$P4
+P5:$P5
+P6:$P6
+EOF

Creating the expected results could be moved into the test_expect_success block as well.

Quoted 32 lines

t/t5323-pack-redundant.sh line 70

Quoted 3 lines
t/t5323-pack-redundant.sh new file +3 -0
+
+test_expect_success 'git pack-redundant --all' '
+	git pack-redundant --all | \

Don't run a git command (especially the particular command the test script focuses on) upstream of a pipe, because it hides the command's exit code. Use an intermediate file instead.

t/t5323-pack-redundant.sh line 71

Quoted 1 line
t/t5323-pack-redundant.sh new file +1 -0
+		sed -e "s#^.*/pack-\(.*\)\.\(idx\|pack\)#\1#g" | \

This sed command doesn't seem to work on macOS (on Travis CI), and causes the test to fail with:

  ++git pack-redundant --all
  ++sed -e 's#^.*/pack-\(.*\)\.\(idx\|pack\)#\1#g'
  ++sort -u
  ++read p
  ++sort
  ++eval echo '${P.git/objects/pack/pack-0cf5cb6afaa1bae36b8e61ca398dbe29a15bc74e.idx}'
  ./test-lib.sh: line 697: ${P.git/objects/pack/pack-0cf5cb6afaa1bae36b8e61ca398dbe29a15bc74e.idx}: bad substitution
  ++test_cmp expected actual
  ++diff -u expected actual
  --- expected    2019-01-09 01:53:45.000000000 +0000
  +++ actual      2019-01-09 01:53:45.000000000 +0000
  @@ -1,4 +0,0 @@
  -P1:24ee080366509364d04a138cd4e168dc4ff33354
  -P4:139d8b0cfe7e8970a8f3533835f90278d88de474
  -P5:23e0f02d822fa4bfe5ee63337ba5632cd7be208e
  -P6:deeb289f1749972f1cd57c3b9f359ece2361f60a
  error: last command exited with $?=1
  not ok 2 - git pack-redundant --all
Quoted 1 line

I'm not sure what's wrong with it, though.

So, it appears that 'sed' in macOS doesn't understand the '\(idx\|pack\)' part of that regex. Turning that command into

  sed -e "s#^.git/objects/pack/pack-\($OID_REGEX\)\..*#\1#" out | \

makes it work even on macOS, but note that those 40 hexdigits are not actual OIDs but file content checksums, so using $OID_REGEX is not the right thing to do here (though I'm not sure what is supposed to be used instead, as $_x40 hardcodes the number of hexdigits).

Alas, the test as a whole still fails with the following on macOS:

++diff -u expected actual
--- expected 2019-01-09 15:54:49.000000000 +0000
+++ actual 2019-01-09 15:54:49.000000000 +0000
@@ -1,4 +1,4 @@
P1:24ee080366509364d04a138cd4e168dc4ff33354
-P4:139d8b0cfe7e8970a8f3533835f90278d88de474
+P3:0cf5cb6afaa1bae36b8e61ca398dbe29a15bc74e
P5:23e0f02d822fa4bfe5ee63337ba5632cd7be208e
-P6:deeb289f1749972f1cd57c3b9f359ece2361f60a
+P7:4ecc1eb138516a26654cd4e3570b322c0820f170
error: last command exited with $?=1

author
#57 lore raw

Re: [PATCH v3 1/3] t5323: test cases for git-pack-redundant

In reply to SZEDER Gábor
Quoted 6 lines · SZEDER Gábor <[email protected]> 于2019年1月9日周三 下午8:56写道:
Quoted 2 lines · Jiang Xin

From: Jiang Xin <[email protected]>
+test_description='git redundant test'

s/redundant/pack-redundant/ ?

Yes, will correct it.

Quoted 10 lines
Quoted 10 lines
t/t5323-pack-redundant.sh new file +10 -0
+
+. ./test-lib.sh
+
+create_commits()
+{
+     set -e
+     parent=
+     for name in A B C D E F G H I J K L M
+     do
+             test_tick
Quoted 9 lines

t/t5323-pack-redundant.sh line 20

Quoted 4 lines
t/t5323-pack-redundant.sh new file +4 -0
+             T=$(git write-tree)
+             if test -z "$parent"
+             then
+                     sha1=$(echo $name | git commit-tree $T)

There is a considerable effort going on to switch from SHA-1 to a different hash function, so please don't add any new $sha1 variable; call it $oid or $commit instead.

Will do.

Quoted 7 lines

t/t5323-pack-redundant.sh line 55

Quoted 4 lines
t/t5323-pack-redundant.sh new file +4 -0
+
+# Create commits and packs
+create_commits
+create_redundant_packs

Please perform all setup tasks in a test_expect_success block, so we get verbose and trace output about what's going on.

Will do like this:

    test_expect_success 'setup' '
            create_commits  &&
            create_redundant_packs
    '
Quoted 9 lines

Don't use 'set -e', use an &&-chain instead. To fail the test if a command in the for loop were to fail you could do something like this:

for ....
do

    do-this &&
    do-that ||
    return 1

done

Will do.

Quoted 11 lines

t/t5323-pack-redundant.sh line 59

Quoted 4 lines
t/t5323-pack-redundant.sh new file +4 -0
+
+test_expect_success 'clear loose objects' '
+     git prune-packed &&
+     test $(find .git/objects -type f | grep -v pack | wc -l) -eq 0

Use something like

find .git/objects -type f | grep -v pack >out && test_must_be_empty out

instead, so we get an informative error message on failure.

if grep -v pack return empty output, it will return error, so I will use sed -e "/objects\/pack\//d" >out instead.

Quoted 17 lines

t/t5323-pack-redundant.sh line 70

Quoted 12 lines

+'

t/t5323-pack-redundant.sh new file +10 -0
+
+cat >expected <<EOF
+P1:$P1
+P4:$P4
+P5:$P5
+P6:$P6
+EOF
+
+test_expect_success 'git pack-redundant --all' '
+     git pack-redundant --all | \

Don't run a git command (especially the particular command the test script focuses on) upstream of a pipe, because it hides the command's exit code. Use an intermediate file instead.

Quoted 5 lines
Quoted 1 line

+ sed -e "s#^.*/pack-\(.*\)\.\(idx\|pack\)#\1#g" | \

This sed command doesn't seem to work on macOS (on Travis CI), and causes the test to fail with:

It works if rewrite as follows:

    git pack-redundant --all >out &&
    sed -E -e "s#.*/pack-(.*)\.(idx|pack)#\1#" out | \

Without -E, MasOS has to write two seperate sed commands, such as:

    git pack-redundant --all >out &&
    sed -e "s#.*/pack-\(.*\)\.idx#\1#" out | \
    sed -e "s#.*/pack-\(.*\)\.pack#\1#"

Option '-E' is an alias for -r in GNU sed 4.2 (added in 4.2, not documented unti 4.3), released on May 11 2009. I prefer the -E version.

Quoted 9 lines

Minor nit: 'git pack-redundant' prints one filename per line, so the 'g' at the end of the 's###g' is not necessary.

t/t5323-pack-redundant.sh line 74

Quoted 3 lines
t/t5323-pack-redundant.sh new file +3 -0
+             sort -u | \
+             while read p; do eval echo "\${P$p}"; done | \
+             sort > actual && \

Style nit: no space between redirection operator and filename

Will do.

#58 lore raw

Re: [PATCH v3 1/3] t5323: test cases for git-pack-redundant

Am 10.01.19 um 04:28 schrieb Jiang Xin:

Quoted 10 lines
Quoted 6 lines · SZEDER Gábor <[email protected]> 于2019年1月9日周三 下午8:56写道:

Use something like

   find .git/objects -type f | grep -v pack >out &&
   test_must_be_empty out

instead, so we get an informative error message on failure.

if grep -v pack return empty output, it will return error, so I will use sed -e "/objects\/pack\//d" >out instead.

So, you could even write this as

	find .git/objects -type f >out &&
	! grep -v pack out	# must be empty

or
! find .git/objects -type f | grep -v pack

if you want to be terse.

-- Hannes

#59 lore raw

Re: [PATCH v3 1/3] t5323: test cases for git-pack-redundant

In reply to Jiang Xin
Quoted 12 lines · Jiang Xin
Quoted 5 lines · SZEDER Gábor <[email protected]> 于2019年1月9日周三 下午8:56写道:
Quoted 1 line

+ sed -e "s#^.*/pack-\(.*\)\.\(idx\|pack\)#\1#g" | \

This sed command doesn't seem to work on macOS (on Travis CI), and causes the test to fail with:

It works if rewrite as follows:

    git pack-redundant --all >out &&
    sed -E -e "s#.*/pack-(.*)\.(idx|pack)#\1#" out | \
Quoted 8 lines

Without -E, MasOS has to write two seperate sed commands, such as:

    git pack-redundant --all >out &&
    sed -e "s#.*/pack-\(.*\)\.idx#\1#" out | \
    sed -e "s#.*/pack-\(.*\)\.pack#\1#"

Option '-E' is an alias for -r in GNU sed 4.2 (added in 4.2, not documented unti 4.3), released on May 11 2009. I prefer the -E version.

Is 'sed -E' portable enough, e.g. to the various BSDs, Solaris, and whatnot? I don't know, but POSIX doesn't mention it, there is not a single instance of it in our current codebase, and it appears that we've never used it before, either. OTOH, 't/check-non-portable-shell.pl' doesn't catch it as non-portable construct...

author
#7 lore raw

v5 cover pack-redundant: new algorithm to find min packs

In reply to SZEDER Gábor
Quoted 9 lines

Sun Chao (my former colleague at Huawei) found a bug of git-pack-redundant. If there are too many packs and many of them overlap
each other, running git pack-redundant --all will exhaust all memories and the process will be killed by kernel.

There is a script in commit log of commit 2/3, which can be used to create a repository with lots of redundant packs. Running git pack-redundant --all in it can reproduce this issue.

SZEDER reported that t5233 won't pass for MacOS. See solution in patch 4/5.

Changes since reroll v4:

* Rewrite t5323, add more test cases.
* Add two new patches, one for refactor, and another changed sorting method and fixed t5323 for the new algorithm.

Range diff with sc/pack-redundant feature branch:

    1:  702267a888 < -:  ---------- t5323: test cases for git-pack-redundant
    -:  ---------- > 1:  40fea5d67f t5323: test cases for git-pack-redundant
    2:  c4b133d858 = 2:  50cd5a5b47 pack-redundant: new algorithm to find min packs
    -:  ---------- > 3:  6338c6fad4 pack-redundant: rename pack_list.all_objects
    -:  ---------- > 4:  734f4d8a8b pack-redundant: consistent sort method
    3:  2351d7e8b5 ! 5:  b7ccdea1ad pack-redundant: remove unused functions
        @@ -13,7 +13,7 @@
          --- a/builtin/pack-redundant.c
          +++ b/builtin/pack-redundant.c
         @@
        - 	struct llist *all_objects;
        + 	size_t all_objects_size;
          } *local_packs = NULL, *altodb_packs = NULL;
          
         -struct pll {
        @@ -105,7 +105,7 @@
         -	diff = llist_copy(list);
         -
         -	while (pl) {
        --		llist_sorted_difference_inplace(diff, pl->all_objects);
        +-		llist_sorted_difference_inplace(diff, pl->remaining_objects);
         -		if (diff->size == 0) { /* we're done */
         -			llist_free(diff);
         -			return 1;

Jiang Xin (3):
t5323: test cases for git-pack-redundant
pack-redundant: rename pack_list.all_objects
pack-redundant: consistent sort method

Sun Chao (2):
pack-redundant: new algorithm to find min packs
pack-redundant: remove unused functions

 builtin/pack-redundant.c  | 221 +++++++++++++++-----------------------
 t/t5323-pack-redundant.sh | 157 +++++++++++++++++++++++++++
 2 files changed, 242 insertions(+), 136 deletions(-)
 create mode 100755 t/t5323-pack-redundant.sh
Signature · 1 line
2.20.1.101.gc01fadde4e
author
#47 lore raw

PATCH v5 1/5 t5323: test cases for git-pack-redundant

In reply to SZEDER Gábor

From: Jiang Xin <[email protected]>

Add test cases for git pack-redundant to validate new algorithm for git pack-redundant.

---
 t/t5323-pack-redundant.sh | 157 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 157 insertions(+)
 create mode 100755 t/t5323-pack-redundant.sh
t/t5323-pack-redundant.sh new file +157 -0
new file mode 100755
index 0000000000..7410426dee
@@ -0,0 +1,157 @@
+#!/bin/sh
+#
+# Copyright (c) 2018 Jiang Xin
+#
+
+test_description='git pack-redundant test'
+
+. ./test-lib.sh
+
+create_commits()
+{
+	parent=
+	for name in A B C D E F G H I J K L M N O P Q R
+	do
+		test_tick &&
+		T=$(git write-tree) &&
+		if test -z "$parent"
+		then
+			oid=$(echo $name | git commit-tree $T)
+		else
+			oid=$(echo $name | git commit-tree -p $parent $T)
+		fi &&
+		eval $name=$oid &&
+		parent=$oid ||
+		return 1
+	done
+	git update-ref refs/heads/master $M
+}
+
+create_pack_1()
+{
+	P1=$(cd .git/objects/pack; printf "$T\n$A\n$B\n$C\n$D\n$E\n$F\n$R\n" | git pack-objects pack 2>/dev/null) &&
+	eval P$P1=P1:$P1
+}
+
+create_pack_2()
+{
+	P2=$(cd .git/objects/pack; printf "$B\n$C\n$D\n$E\n$G\n$H\n$I\n" | git pack-objects pack 2>/dev/null) &&
+	eval P$P2=P2:$P2
+}
+
+create_pack_3()
+{
+	P3=$(cd .git/objects/pack; printf "$F\n$I\n$J\n$K\n$L\n$M\n" | git pack-objects pack 2>/dev/null) &&
+	eval P$P3=P3:$P3
+}
+
+create_pack_4()
+{
+	P4=$(cd .git/objects/pack; printf "$J\n$K\n$L\n$M\n$P\n" | git pack-objects pack 2>/dev/null) &&
+	eval P$P4=P4:$P4
+}
+
+create_pack_5()
+{
+	P5=$(cd .git/objects/pack; printf "$G\n$H\n$N\n$O\n" | git pack-objects pack 2>/dev/null) &&
+	eval P$P5=P5:$P5
+}
+
+create_pack_6()
+{
+	P6=$(cd .git/objects/pack; printf "$N\n$O\n$Q\n" | git pack-objects pack 2>/dev/null) &&
+	eval P$P6=P6:$P6
+}
+
+create_pack_7()
+{
+	P7=$(cd .git/objects/pack; printf "$P\n$Q\n" | git pack-objects pack 2>/dev/null) &&
+	eval P$P7=P7:$P7
+}
+
+create_pack_8()
+{
+	P8=$(cd .git/objects/pack; printf "$A\n" | git pack-objects pack 2>/dev/null) &&
+	eval P$P8=P8:$P8
+}
+
+test_expect_success 'setup' '
+	create_commits
+'
+
+test_expect_success 'no redundant packs' '
+	create_pack_1 && create_pack_2 && create_pack_3 &&
+	git pack-redundant --all >out &&
+	test_must_be_empty out
+'
+
+test_expect_success 'create pack 4, 5' '
+	create_pack_4 && create_pack_5
+'
+
+cat >expected <<EOF
+P2:$P2
+EOF
+
+test_expect_success 'one of pack-2/pack-3 is redundant' '
+	git pack-redundant --all >out &&
+	sed -E -e "s#.*/pack-(.*)\.(idx|pack)#\1#" out | \
+		sort -u | \
+		while read p; do eval echo "\${P$p}"; done | \
+		sort >actual && \
+	test_cmp expected actual
+'
+
+test_expect_success 'create pack 6, 7' '
+	create_pack_6 && create_pack_7
+'
+
+cat >expected <<EOF
+P2:$P2
+P4:$P4
+P6:$P6
+EOF
+
+test_expect_success 'pack 2, 4, and 6 are redundant' '
+	git pack-redundant --all >out &&
+	sed -E -e "s#.*/pack-(.*)\.(idx|pack)#\1#" out | \
+		sort -u | \
+		while read p; do eval echo "\${P$p}"; done | \
+		sort >actual && \
+	test_cmp expected actual
+'
+
+test_expect_success 'create pack 8' '
+	create_pack_8
+'
+
+cat >expected <<EOF
+P2:$P2
+P4:$P4
+P6:$P6
+P8:$P8
+EOF
+
+test_expect_success 'pack-8, subset of pack-1, is also redundant' '
+	git pack-redundant --all >out &&
+	sed -E -e "s#.*/pack-(.*)\.(idx|pack)#\1#" out | \
+		sort -u | \
+		while read p; do eval echo "\${P$p}"; done | \
+		sort >actual && \
+	test_cmp expected actual
+'
+
+test_expect_success 'clear loose objects' '
+	git prune-packed &&
+	find .git/objects -type f | sed -e "/objects\/pack\//d" >out &&
+	test_must_be_empty out
+'
+
+test_expect_success 'remove redundant packs' '
+	git pack-redundant --all | xargs rm &&
+	git fsck &&
+	git pack-redundant --all >out &&
+	test_must_be_empty out
+'
+
+test_done
Signature · 1 line
2.20.1.101.gc01fadde4e
author
#51 lore raw

PATCH v5 2/5 pack-redundant: new algorithm to find min packs

In reply to SZEDER Gábor

From: Sun Chao <[email protected]>

When calling git pack-redundant --all, if there are too many local packs and too many redundant objects within them, the too deep iteration of get_permutations will exhaust all the resources, and the process of git pack-redundant will be killed.

The following script could create a repository with too many redundant packs, and running git pack-redundant --all in the test.git repo will die soon.

    #!/bin/sh

    repo="$(pwd)/test.git"
    work="$(pwd)/test"
    i=1
    max=199

    if test -d "$repo" || test -d "$work"; then
    	echo >&2 "ERROR: '$repo' or '$work' already exist"
    	exit 1
    fi

    git init -q --bare "$repo"
    git --git-dir="$repo" config gc.auto 0
    git --git-dir="$repo" config transfer.unpackLimit 0
    git clone -q "$repo" "$work" 2>/dev/null

    while :; do
        cd "$work"
        echo "loop $i: $(date +%s)" >$i
        git add $i
        git commit -q -sm "loop $i"
        git push -q origin HEAD:master
        printf "\rCreate pack %4d/%d\t" $i $max
        if test $i -ge $max; then break; fi

        cd "$repo"
        git repack -q
        if test $(($i % 2)) -eq 0; then
            git repack -aq
            pack=$(ls -t $repo/objects/pack/*.pack | head -1)
            touch "${pack%.pack}.keep"
        fi
        i=$((i+1))
    done
    printf "\ndone\n"

To get the min unique pack list, we can replace the iteration in minimize function with a new algorithm, and this could solve this issue:

1. Get the unique and non_uniqe packs, add the unique packs to the min list.

2. Remove the objects of unique packs from non_unique packs, then each object left in the non_unique packs will have at least two copies.

3. Sort the non_unique packs by the objects' size, more objects first, and add the first non_unique pack to min list.

4. Drop the duplicated objects from other packs in the ordered non_unique pack list, and repeat step 3.

Original PR and discussions: https://github.com/jiangxin/git/pull/25

---
 builtin/pack-redundant.c | 109 ++++++++++++++++++++++++---------------
 1 file changed, 68 insertions(+), 41 deletions(-)
builtin/pack-redundant.c +68 -41
index cf9a9aabd4..3655cc7dc6 100644
@@ -421,14 +421,52 @@ static inline off_t pack_set_bytecount(struct pack_list *pl)
 	return ret;
 }
 
+static int cmp_pack_list_reverse(const void *a, const void *b)
+{
+	struct pack_list *pl_a = *((struct pack_list **)a);
+	struct pack_list *pl_b = *((struct pack_list **)b);
+	size_t sz_a = pl_a->all_objects->size;
+	size_t sz_b = pl_b->all_objects->size;
+
+	if (sz_a == sz_b)
+		return 0;
+	else if (sz_a < sz_b)
+		return 1;
+	else
+		return -1;
+}
+
+/* Sort pack_list, greater size of all_objects first */
+static void sort_pack_list(struct pack_list **pl)
+{
+	struct pack_list **ary, *p;
+	int i;
+	size_t n = pack_list_size(*pl);
+
+	if (n < 2)
+		return;
+
+	/* prepare an array of packed_list for easier sorting */
+	ary = xcalloc(n, sizeof(struct pack_list *));
+	for (n = 0, p = *pl; p; p = p->next)
+		ary[n++] = p;
+
+	QSORT(ary, n, cmp_pack_list_reverse);
+
+	/* link them back again */
+	for (i = 0; i < n - 1; i++)
+		ary[i]->next = ary[i + 1];
+	ary[n - 1]->next = NULL;
+	*pl = ary[0];
+
+	free(ary);
+}
+
+
 static void minimize(struct pack_list **min)
 {
-	struct pack_list *pl, *unique = NULL,
-		*non_unique = NULL, *min_perm = NULL;
-	struct pll *perm, *perm_all, *perm_ok = NULL, *new_perm;
-	struct llist *missing;
-	off_t min_perm_size = 0, perm_size;
-	int n;
+	struct pack_list *pl, *unique = NULL, *non_unique = NULL;
+	struct llist *missing, *unique_pack_objects;
 
 	pl = local_packs;
 	while (pl) {
@@ -446,49 +484,37 @@ static void minimize(struct pack_list **min)
 		pl = pl->next;
 	}
 
+	*min = unique;
+
 	/* return if there are no objects missing from the unique set */
 	if (missing->size == 0) {
-		*min = unique;
 		free(missing);
 		return;
 	}
 
-	/* find the permutations which contain all missing objects */
-	for (n = 1; n <= pack_list_size(non_unique) && !perm_ok; n++) {
-		perm_all = perm = get_permutations(non_unique, n);
-		while (perm) {
-			if (is_superset(perm->pl, missing)) {
-				new_perm = xmalloc(sizeof(struct pll));
-				memcpy(new_perm, perm, sizeof(struct pll));
-				new_perm->next = perm_ok;
-				perm_ok = new_perm;
-			}
-			perm = perm->next;
-		}
-		if (perm_ok)
-			break;
-		pll_free(perm_all);
-	}
-	if (perm_ok == NULL)
-		die("Internal error: No complete sets found!");
-
-	/* find the permutation with the smallest size */
-	perm = perm_ok;
-	while (perm) {
-		perm_size = pack_set_bytecount(perm->pl);
-		if (!min_perm_size || min_perm_size > perm_size) {
-			min_perm_size = perm_size;
-			min_perm = perm->pl;
-		}
-		perm = perm->next;
-	}
-	*min = min_perm;
-	/* add the unique packs to the list */
-	pl = unique;
+	unique_pack_objects = llist_copy(all_objects);
+	llist_sorted_difference_inplace(unique_pack_objects, missing);
+
+	/* remove unique pack objects from the non_unique packs */
+	pl = non_unique;
 	while (pl) {
-		pack_list_insert(min, pl);
+		llist_sorted_difference_inplace(pl->all_objects, unique_pack_objects);
 		pl = pl->next;
 	}
+
+	while (non_unique) {
+		/* sort the non_unique packs, greater size of all_objects first */
+		sort_pack_list(&non_unique);
+		if (non_unique->all_objects->size == 0)
+			break;
+
+		pack_list_insert(min, non_unique);
+
+		for (pl = non_unique->next; pl && pl->all_objects->size > 0;  pl = pl->next)
+			llist_sorted_difference_inplace(pl->all_objects, non_unique->all_objects);
+
+		non_unique = non_unique->next;
+	}
 }
 
 static void load_all_objects(void)
@@ -603,7 +629,7 @@ static void load_all(void)
 int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 {
 	int i;
-	struct pack_list *min, *red, *pl;
+	struct pack_list *min = NULL, *red, *pl;
 	struct llist *ignore;
 	struct object_id *oid;
 	char buf[GIT_MAX_HEXSZ + 2]; /* hex hash + \n + \0 */
@@ -664,6 +690,7 @@ int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 	pl = local_packs;
 	while (pl) {
 		llist_sorted_difference_inplace(pl->unique_objects, ignore);
+		llist_sorted_difference_inplace(pl->all_objects, ignore);
 		pl = pl->next;
 	}
 
Signature · 1 line
2.20.1.101.gc01fadde4e
author
#53 lore raw

PATCH v5 3/5 pack-redundant: rename pack_list.all_objects

In reply to SZEDER Gábor

From: Jiang Xin <[email protected]>

New algorithm uses pack_list.all_objects to track remaining objects, so rename it to pack_list.remaining_objects.

---
 builtin/pack-redundant.c | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)
builtin/pack-redundant.c +19 -19
index 3655cc7dc6..56591d283f 100644
@@ -32,7 +32,7 @@ static struct pack_list {
 	struct pack_list *next;
 	struct packed_git *pack;
 	struct llist *unique_objects;
-	struct llist *all_objects;
+	struct llist *remaining_objects;
 } *local_packs = NULL, *altodb_packs = NULL;
 
 struct pll {
@@ -346,7 +346,7 @@ static int is_superset(struct pack_list *pl, struct llist *list)
 	diff = llist_copy(list);
 
 	while (pl) {
-		llist_sorted_difference_inplace(diff, pl->all_objects);
+		llist_sorted_difference_inplace(diff, pl->remaining_objects);
 		if (diff->size == 0) { /* we're done */
 			llist_free(diff);
 			return 1;
@@ -425,8 +425,8 @@ static int cmp_pack_list_reverse(const void *a, const void *b)
 {
 	struct pack_list *pl_a = *((struct pack_list **)a);
 	struct pack_list *pl_b = *((struct pack_list **)b);
-	size_t sz_a = pl_a->all_objects->size;
-	size_t sz_b = pl_b->all_objects->size;
+	size_t sz_a = pl_a->remaining_objects->size;
+	size_t sz_b = pl_b->remaining_objects->size;
 
 	if (sz_a == sz_b)
 		return 0;
@@ -436,7 +436,7 @@ static int cmp_pack_list_reverse(const void *a, const void *b)
 		return -1;
 }
 
-/* Sort pack_list, greater size of all_objects first */
+/* Sort pack_list, greater size of remaining_objects first */
 static void sort_pack_list(struct pack_list **pl)
 {
 	struct pack_list **ary, *p;
@@ -480,7 +480,7 @@ static void minimize(struct pack_list **min)
 	missing = llist_copy(all_objects);
 	pl = unique;
 	while (pl) {
-		llist_sorted_difference_inplace(missing, pl->all_objects);
+		llist_sorted_difference_inplace(missing, pl->remaining_objects);
 		pl = pl->next;
 	}
 
@@ -498,20 +498,20 @@ static void minimize(struct pack_list **min)
 	/* remove unique pack objects from the non_unique packs */
 	pl = non_unique;
 	while (pl) {
-		llist_sorted_difference_inplace(pl->all_objects, unique_pack_objects);
+		llist_sorted_difference_inplace(pl->remaining_objects, unique_pack_objects);
 		pl = pl->next;
 	}
 
 	while (non_unique) {
-		/* sort the non_unique packs, greater size of all_objects first */
+		/* sort the non_unique packs, greater size of remaining_objects first */
 		sort_pack_list(&non_unique);
-		if (non_unique->all_objects->size == 0)
+		if (non_unique->remaining_objects->size == 0)
 			break;
 
 		pack_list_insert(min, non_unique);
 
-		for (pl = non_unique->next; pl && pl->all_objects->size > 0;  pl = pl->next)
-			llist_sorted_difference_inplace(pl->all_objects, non_unique->all_objects);
+		for (pl = non_unique->next; pl && pl->remaining_objects->size > 0;  pl = pl->next)
+			llist_sorted_difference_inplace(pl->remaining_objects, non_unique->remaining_objects);
 
 		non_unique = non_unique->next;
 	}
@@ -526,7 +526,7 @@ static void load_all_objects(void)
 
 	while (pl) {
 		hint = NULL;
-		l = pl->all_objects->front;
+		l = pl->remaining_objects->front;
 		while (l) {
 			hint = llist_insert_sorted_unique(all_objects,
 							  l->oid, hint);
@@ -537,7 +537,7 @@ static void load_all_objects(void)
 	/* remove objects present in remote packs */
 	pl = altodb_packs;
 	while (pl) {
-		llist_sorted_difference_inplace(all_objects, pl->all_objects);
+		llist_sorted_difference_inplace(all_objects, pl->remaining_objects);
 		pl = pl->next;
 	}
 }
@@ -563,10 +563,10 @@ static void scan_alt_odb_packs(void)
 		local = local_packs;
 		while (local) {
 			llist_sorted_difference_inplace(local->unique_objects,
-							alt->all_objects);
+							alt->remaining_objects);
 			local = local->next;
 		}
-		llist_sorted_difference_inplace(all_objects, alt->all_objects);
+		llist_sorted_difference_inplace(all_objects, alt->remaining_objects);
 		alt = alt->next;
 	}
 }
@@ -581,7 +581,7 @@ static struct pack_list * add_pack(struct packed_git *p)
 		return NULL;
 
 	l.pack = p;
-	llist_init(&l.all_objects);
+	llist_init(&l.remaining_objects);
 
 	if (open_pack_index(p))
 		return NULL;
@@ -590,11 +590,11 @@ static struct pack_list * add_pack(struct packed_git *p)
 	base += 256 * 4 + ((p->index_version < 2) ? 4 : 8);
 	step = the_hash_algo->rawsz + ((p->index_version < 2) ? 4 : 0);
 	while (off < p->num_objects * step) {
-		llist_insert_back(l.all_objects, (const struct object_id *)(base + off));
+		llist_insert_back(l.remaining_objects, (const struct object_id *)(base + off));
 		off += step;
 	}
 	/* this list will be pruned in cmp_two_packs later */
-	l.unique_objects = llist_copy(l.all_objects);
+	l.unique_objects = llist_copy(l.remaining_objects);
 	if (p->pack_local)
 		return pack_list_insert(&local_packs, &l);
 	else
@@ -690,7 +690,7 @@ int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 	pl = local_packs;
 	while (pl) {
 		llist_sorted_difference_inplace(pl->unique_objects, ignore);
-		llist_sorted_difference_inplace(pl->all_objects, ignore);
+		llist_sorted_difference_inplace(pl->remaining_objects, ignore);
 		pl = pl->next;
 	}
 
Signature · 1 line
2.20.1.101.gc01fadde4e
author
#54 lore raw

PATCH v5 4/5 pack-redundant: consistent sort method

In reply to SZEDER Gábor

From: Jiang Xin <[email protected]>

SZEDER reported that test case t5323 has different test result on MacOS. This is because cmp_pack_list_reverse cannot give identical result when two pack being sorted has the same size of remaining_objects.

Changes to the sorting function will make consistent test result for t5323.

The new algorithm to find redundant packs is a trade-off to save memory resources, and the result of it may be different with old one, and may be not the best result sometimes. Update t5323 for the new algorithm.

---
 builtin/pack-redundant.c  | 22 +++++++++++++++-------
 t/t5323-pack-redundant.sh |  2 +-
 2 files changed, 16 insertions(+), 8 deletions(-)
builtin/pack-redundant.c +15 -7
index 56591d283f..e9d2586e2e 100644
@@ -33,6 +33,7 @@ static struct pack_list {
 	struct packed_git *pack;
 	struct llist *unique_objects;
 	struct llist *remaining_objects;
+	size_t all_objects_size;
 } *local_packs = NULL, *altodb_packs = NULL;
 
 struct pll {
@@ -421,16 +422,22 @@ static inline off_t pack_set_bytecount(struct pack_list *pl)
 	return ret;
 }
 
-static int cmp_pack_list_reverse(const void *a, const void *b)
+static int cmp_remaining_objects(const void *a, const void *b)
 {
 	struct pack_list *pl_a = *((struct pack_list **)a);
 	struct pack_list *pl_b = *((struct pack_list **)b);
-	size_t sz_a = pl_a->remaining_objects->size;
-	size_t sz_b = pl_b->remaining_objects->size;
 
-	if (sz_a == sz_b)
-		return 0;
-	else if (sz_a < sz_b)
+	/* if have the same remaining_objects, big pack first */
+	if (pl_a->remaining_objects->size == pl_b->remaining_objects->size)
+		if (pl_a->all_objects_size == pl_b->all_objects_size)
+			return 0;
+		else if (pl_a->all_objects_size < pl_b->all_objects_size)
+			return 1;
+		else
+			return -1;
+
+	/* sort according to remaining objects, more remaining objects first */
+	if (pl_a->remaining_objects->size < pl_b->remaining_objects->size)
 		return 1;
 	else
 		return -1;
@@ -451,7 +458,7 @@ static void sort_pack_list(struct pack_list **pl)
 	for (n = 0, p = *pl; p; p = p->next)
 		ary[n++] = p;
 
-	QSORT(ary, n, cmp_pack_list_reverse);
+	QSORT(ary, n, cmp_remaining_objects);
 
 	/* link them back again */
 	for (i = 0; i < n - 1; i++)
@@ -593,6 +600,7 @@ static struct pack_list * add_pack(struct packed_git *p)
 		llist_insert_back(l.remaining_objects, (const struct object_id *)(base + off));
 		off += step;
 	}
+	l.all_objects_size = l.remaining_objects->size;
 	/* this list will be pruned in cmp_two_packs later */
 	l.unique_objects = llist_copy(l.remaining_objects);
 	if (p->pack_local)
t/t5323-pack-redundant.sh +1 -1
index 7410426dee..2a09ff1bfb 100755
@@ -90,7 +90,7 @@ test_expect_success 'create pack 4, 5' '
 '
 
 cat >expected <<EOF
-P2:$P2
+P3:$P3
 EOF
 
 test_expect_success 'one of pack-2/pack-3 is redundant' '
Signature · 1 line
2.20.1.101.gc01fadde4e
author
#56 lore raw

PATCH v5 5/5 pack-redundant: remove unused functions

In reply to SZEDER Gábor

From: Sun Chao <[email protected]>

Remove unused functions to find min packs, such as get_permutations, pll_free, etc.

---
 builtin/pack-redundant.c | 86 ----------------------------------------
 1 file changed, 86 deletions(-)
builtin/pack-redundant.c +0 -86
index e9d2586e2e..dd71fdd435 100644
@@ -36,11 +36,6 @@ static struct pack_list {
 	size_t all_objects_size;
 } *local_packs = NULL, *altodb_packs = NULL;
 
-struct pll {
-	struct pll *next;
-	struct pack_list *pl;
-};
-
 static struct llist_item *free_nodes;
 
 static inline void llist_item_put(struct llist_item *item)
@@ -64,15 +59,6 @@ static inline struct llist_item *llist_item_get(void)
 	return new_item;
 }
 
-static void llist_free(struct llist *list)
-{
-	while ((list->back = list->front)) {
-		list->front = list->front->next;
-		llist_item_put(list->back);
-	}
-	free(list);
-}
-
 static inline void llist_init(struct llist **list)
 {
 	*list = xmalloc(sizeof(struct llist));
@@ -286,78 +272,6 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
 	}
 }
 
-static void pll_free(struct pll *l)
-{
-	struct pll *old;
-	struct pack_list *opl;
-
-	while (l) {
-		old = l;
-		while (l->pl) {
-			opl = l->pl;
-			l->pl = opl->next;
-			free(opl);
-		}
-		l = l->next;
-		free(old);
-	}
-}
-
-/* all the permutations have to be free()d at the same time,
- * since they refer to each other
- */
-static struct pll * get_permutations(struct pack_list *list, int n)
-{
-	struct pll *subset, *ret = NULL, *new_pll = NULL;
-
-	if (list == NULL || pack_list_size(list) < n || n == 0)
-		return NULL;
-
-	if (n == 1) {
-		while (list) {
-			new_pll = xmalloc(sizeof(*new_pll));
-			new_pll->pl = NULL;
-			pack_list_insert(&new_pll->pl, list);
-			new_pll->next = ret;
-			ret = new_pll;
-			list = list->next;
-		}
-		return ret;
-	}
-
-	while (list->next) {
-		subset = get_permutations(list->next, n - 1);
-		while (subset) {
-			new_pll = xmalloc(sizeof(*new_pll));
-			new_pll->pl = subset->pl;
-			pack_list_insert(&new_pll->pl, list);
-			new_pll->next = ret;
-			ret = new_pll;
-			subset = subset->next;
-		}
-		list = list->next;
-	}
-	return ret;
-}
-
-static int is_superset(struct pack_list *pl, struct llist *list)
-{
-	struct llist *diff;
-
-	diff = llist_copy(list);
-
-	while (pl) {
-		llist_sorted_difference_inplace(diff, pl->remaining_objects);
-		if (diff->size == 0) { /* we're done */
-			llist_free(diff);
-			return 1;
-		}
-		pl = pl->next;
-	}
-	llist_free(diff);
-	return 0;
-}
-
 static size_t sizeof_union(struct packed_git *p1, struct packed_git *p2)
 {
 	size_t ret = 0;
Signature · 1 line
2.20.1.101.gc01fadde4e
#60 lore raw

Re: [PATCH v3 1/3] t5323: test cases for git-pack-redundant

In reply to SZEDER Gábor
Quoted 19 lines · SZEDER Gábor
Quoted 18 lines · Jiang Xin
Quoted 5 lines · SZEDER Gábor <[email protected]> 于2019年1月9日周三 下午8:56写道:
Quoted 1 line

+ sed -e "s#^.*/pack-\(.*\)\.\(idx\|pack\)#\1#g" | \

This sed command doesn't seem to work on macOS (on Travis CI), and causes the test to fail with:

It works if rewrite as follows:

    git pack-redundant --all >out &&
    sed -E -e "s#.*/pack-(.*)\.(idx|pack)#\1#" out | \

Without -E, MasOS has to write two seperate sed commands, such as:

    git pack-redundant --all >out &&
    sed -e "s#.*/pack-\(.*\)\.idx#\1#" out | \
    sed -e "s#.*/pack-\(.*\)\.pack#\1#"
Quoted 7 lines
Quoted 2 lines

Option '-E' is an alias for -r in GNU sed 4.2 (added in 4.2, not documented unti 4.3), released on May 11 2009. I prefer the -E version.

Is 'sed -E' portable enough, e.g. to the various BSDs, Solaris, and whatnot? I don't know, but POSIX doesn't mention it, there is not a single instance of it in our current codebase, and it appears that we've never used it before, either. OTOH,

If we can use "two seperate sed commands" i would (really) prefer to so, to avoid "sed -E".
My conclusion is that it is not portable enough.

Quoted 2 lines

't/check-non-portable-shell.pl' doesn't catch it as non-portable construct...

Good point.
Actually that script only checks "known non-portable" options. Every time somebody finds a non-portable option, we update it. A growing blacklist, so to say.
May be we should have a white list instead.

#61 lore raw

Re: [PATCH v3 1/3] t5323: test cases for git-pack-redundant

In reply to SZEDER Gábor
Quoted 5 lines · SZEDER Gábor
Quoted 5 lines

Without -E, MasOS has to write two seperate sed commands, such as:

    git pack-redundant --all >out &&
    sed -e "s#.*/pack-\(.*\)\.idx#\1#" out | \
    sed -e "s#.*/pack-\(.*\)\.pack#\1#"

Two commands, perhaps, but does it have to be two separate sed processes piped together? Why won't something like this work?

	sed -e 's|.*/pack-\([0-9a-f]*\)\.idx$|\1' \
	-e 's|.*/pack-\([0-9a-f]*\)\.pack$|\1'
#55 lore raw

Re: [PATCH v5 4/5] pack-redundant: consistent sort method

In reply to Jiang Xin
Quoted 18 lines · Jiang Xin
builtin/pack-redundant.c +1 -4
index 56591d283f..e9d2586e2e 100644
@@ -421,16 +422,22 @@ static inline off_t pack_set_bytecount(struct pack_list *pl)
 	return ret;
 }
 
-static int cmp_pack_list_reverse(const void *a, const void *b)
+static int cmp_remaining_objects(const void *a, const void *b)
 {
 	struct pack_list *pl_a = *((struct pack_list **)a);
 	struct pack_list *pl_b = *((struct pack_list **)b);
-	size_t sz_a = pl_a->remaining_objects->size;
-	size_t sz_b = pl_b->remaining_objects->size;
 
-	if (sz_a == sz_b)

builtin/pack-redundant.c static inline off_t pack_set_bytecount(struct pack_list *pl) · line 437

Quoted 10 lines
builtin/pack-redundant.c +8 -2
-		return 0;
-	else if (sz_a < sz_b)
+	/* if have the same remaining_objects, big pack first */
+	if (pl_a->remaining_objects->size == pl_b->remaining_objects->size)
+		if (pl_a->all_objects_size == pl_b->all_objects_size)
+			return 0;
+		else if (pl_a->all_objects_size < pl_b->all_objects_size)
+			return 1;
+		else
+			return -1;

My compiler complains about the above nested if statements:

builtin/pack-redundant.c: In function ‘cmp_remaining_objects’:
builtin/pack-redundant.c:345:5: error: suggest explicit braces to avoid ambiguous ‘else’ [-Werror=parentheses]

    if (pl_a->remaining_objects->size == pl_b->remaining_objects->size)
       ^

cc1: all warnings being treated as errors Makefile:2302: recipe for target 'builtin/pack-redundant.o' failed

After adding a pair of {} to the outer if statement 't5323-pack-redundant.sh' passed successfully even on macOS (on Travis CI).

#48 lore raw

Re: [PATCH v5 1/5] t5323: test cases for git-pack-redundant

In reply to Jiang Xin
Quoted 19 lines · Jiang Xin

From: Jiang Xin <[email protected]>

Add test cases for git pack-redundant to validate new algorithm for git pack-redundant.

---
 t/t5323-pack-redundant.sh | 157 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 157 insertions(+)
 create mode 100755 t/t5323-pack-redundant.sh
t/t5323-pack-redundant.sh new file +1 -0
new file mode 100755
index 0000000000..7410426dee
@@ -0,0 +1,157 @@
+#!/bin/sh

t/t5323-pack-redundant.sh line 11

Quoted 10 lines
t/t5323-pack-redundant.sh new file +10 -0
+#
+# Copyright (c) 2018 Jiang Xin
+#
+
+test_description='git pack-redundant test'
+
+. ./test-lib.sh
+
+create_commits()
+{

Style (see Documentation/CodingGuidelines).

Quoted 11 lines
t/t5323-pack-redundant.sh new file +11 -0
+	parent=
+	for name in A B C D E F G H I J K L M N O P Q R
+	do
+		test_tick &&
+		T=$(git write-tree) &&
+		if test -z "$parent"
+		then
+			oid=$(echo $name | git commit-tree $T)
+		else
+			oid=$(echo $name | git commit-tree -p $parent $T)
+		fi &&

t/t5323-pack-redundant.sh line 32

Quoted 10 lines
t/t5323-pack-redundant.sh new file +10 -0
+		eval $name=$oid &&
+		parent=$oid ||
+		return 1
+	done
+	git update-ref refs/heads/master $M
+}
+
+create_pack_1()
+{
+	P1=$(cd .git/objects/pack; printf "$T\n$A\n$B\n$C\n$D\n$E\n$F\n$R\n" | git pack-objects pack 2>/dev/null) &&

Yikes. Can't "git pack-objects" get the input directly without overlong printf, something along the lines of...

	P1=$(git -C .git/objects/pack pack-objects pack <<-EOF
		$A
		$B
		$C
		...
		$R
		EOF
	)
Quoted 10 lines
t/t5323-pack-redundant.sh new file +9 -0
+	eval P$P1=P1:$P1
+}
...
+test_expect_success 'setup' '
+	create_commits
+'
+
+test_expect_success 'no redundant packs' '
+	create_pack_1 && create_pack_2 && create_pack_3 &&
+	git pack-redundant --all >out &&

t/t5323-pack-redundant.sh line 94

Quoted 10 lines
t/t5323-pack-redundant.sh new file +10 -0
+	test_must_be_empty out
+'
+
+test_expect_success 'create pack 4, 5' '
+	create_pack_4 && create_pack_5
+'
+
+cat >expected <<EOF
+P2:$P2
+EOF

Move this to the next "expect success" block?

t/t5323-pack-redundant.sh line 98

Quoted 3 lines
t/t5323-pack-redundant.sh new file +3 -0
+test_expect_success 'one of pack-2/pack-3 is redundant' '
+	git pack-redundant --all >out &&
+	sed -E -e "s#.*/pack-(.*)\.(idx|pack)#\1#" out | \

How portable is "sed -E" (it is not even in POSIX.1)? Wouldn't it be easier to work with to have two "-e" fed to a single sed invocation instead?

#52 lore raw
In reply to Jiang Xin
Quoted 24 lines · Jiang Xin
builtin/pack-redundant.c +2 -7
index cf9a9aabd4..3655cc7dc6 100644
@@ -446,49 +484,37 @@ static void minimize(struct pack_list **min)
 		pl = pl->next;
 	}
 
+	*min = unique;
+
 	/* return if there are no objects missing from the unique set */
 	if (missing->size == 0) {
-		*min = unique;
 		free(missing);
 		return;
 	}
 
-	/* find the permutations which contain all missing objects */
-	for (n = 1; n <= pack_list_size(non_unique) && !perm_ok; n++) {
-		perm_all = perm = get_permutations(non_unique, n);
-		while (perm) {
-			if (is_superset(perm->pl, missing)) {
-				new_perm = xmalloc(sizeof(struct pll));

builtin/pack-redundant.c static void minimize(struct pack_list **min) · line 471

Quoted 10 lines
builtin/pack-redundant.c +0 -10
-				memcpy(new_perm, perm, sizeof(struct pll));
-				new_perm->next = perm_ok;
-				perm_ok = new_perm;
-			}
-			perm = perm->next;
-		}
-		if (perm_ok)
-			break;
-		pll_free(perm_all);
-	}

Please make sure that all commits in the patch series can be build cleanly without any warnings (with '-Werror' or preferably with 'make DEVELOPER=1') and pass the test suite. This is important, because unbuildable commits will cause trouble later on, when e.g. 'git bisect' happens to pick such a commit.

In this case, the removal of the above loop removes all callsites of the static functions get_permutations(), is_superset(), and pll_free(), resulting the following compiler error:

builtin/pack-redundant.c: At top level:
builtin/pack-redundant.c:289:13: error: ‘pll_free’ defined but not used [-Werror=unused-function]
static void pll_free(struct pll *l)

               ^
  builtin/pack-redundant.c:309:21: error: ‘get_permutations’ defined but not used [-Werror=unused-function]
   static struct pll * get_permutations(struct pack_list *list, int n)
                       ^
  builtin/pack-redundant.c:343:12: error: ‘is_superset’ defined but not used [-Werror=unused-function]
   static int is_superset(struct pack_list *pl, struct llist *list)
              ^

I see that the last patch in this series removes those three unused functions, but that patch should be squashed into this one to keep Git buildable with '-Werror' or DEVELOPER=1.

Furthermore, after building this patch (without '-Werror'), several tests in 't5323-pack-redundant.sh' fail. To avoid the test failure I think the fourth patch ensuring a consistent sort order should be squashed in as well.

author
#49 lore raw

Re: [PATCH v5 1/5] t5323: test cases for git-pack-redundant

In reply to Junio C Hamano
Quoted 8 lines · Junio C Hamano <[email protected]> 于2019年1月11日周五 上午5:11写道:

Review

Quoted 3 lines · Jiang Xin

From: Jiang Xin <[email protected]>

(quoted diff) +2 -0
+create_commits()
+{

Style (see Documentation/CodingGuidelines).

OK, parenthese after function name.

Quoted 8 lines

t/t5323-pack-redundant.sh line 32

Quoted 4 lines
t/t5323-pack-redundant.sh new file +3 -0
+create_pack_1()
+{
+     P1=$(cd .git/objects/pack; printf "$T\n$A\n$B\n$C\n$D\n$E\n$F\n$R\n" | git pack-objects pack 2>/dev/null) &&

Yikes. Can't "git pack-objects" get the input directly without overlong printf, something along the lines of...

Quoted 8 lines
        P1=$(git -C .git/objects/pack pack-objects pack <<-EOF
                $A
                $B
                $C
                ...
                $R
                EOF
        )

Find that no space before <OID>, because git-pack-objects not allow that, and mached parentheses should in the same line.
So Will write like this:

    create_pack_1() {
            P1=$(git -C .git/objects/pack pack-objects pack <<-EOF) &&
    $T
    $A
    $B
    $R
    EOF
            eval P$P1=P1:$P1
    }
Quoted 7 lines
Quoted 7 lines
t/t5323-pack-redundant.sh new file +7 -0
+test_expect_success 'no redundant packs' '
+     create_pack_1 && create_pack_2 && create_pack_3 &&
+     git pack-redundant --all >out &&
+     test_must_be_empty out
+'
+
+test_expect_success 'create pack 4, 5' '
Quoted 8 lines

t/t5323-pack-redundant.sh line 94

Quoted 6 lines
t/t5323-pack-redundant.sh new file +6 -0
+     create_pack_4 && create_pack_5
+'
+
+cat >expected <<EOF
+P2:$P2
+EOF

Move this to the next "expect success" block?

$P4 and $P5 are defined after calling create_pack_4 and create_pack_5, so create pack functions should be called before write expected file, if puts $P4 and/or $P5 in the expected file.

For this case, $P4 and $P5 not in expected file, we can move create_pack_4 and 5 to the following test_expect_success block, but the new algorithm may change the expected file.

Quoted 8 lines

t/t5323-pack-redundant.sh line 98

Quoted 4 lines
t/t5323-pack-redundant.sh new file +3 -0
+test_expect_success 'one of pack-2/pack-3 is redundant' '
+     git pack-redundant --all >out &&
+     sed -E -e "s#.*/pack-(.*)\.(idx|pack)#\1#" out | \

How portable is "sed -E" (it is not even in POSIX.1)? Wouldn't it be easier to work with to have two "-e" fed to a single sed invocation instead?

will fix using two '-e' commands.

#50 lore raw

Re: [PATCH v5 1/5] t5323: test cases for git-pack-redundant

Quoted 28 lines · Jiang Xin
Quoted 8 lines · Junio C Hamano <[email protected]> 于2019年1月11日周五 上午5:11写道:

Review

Quoted 3 lines · Jiang Xin

From: Jiang Xin <[email protected]>

(quoted diff) +2 -0
+create_commits()
+{

Style (see Documentation/CodingGuidelines).

OK, parenthese after function name.

Quoted 8 lines

t/t5323-pack-redundant.sh line 32

Quoted 4 lines
t/t5323-pack-redundant.sh new file +3 -0
+create_pack_1()
+{
+     P1=$(cd .git/objects/pack; printf "$T\n$A\n$B\n$C\n$D\n$E\n$F\n$R\n" | git pack-objects pack 2>/dev/null) &&

Yikes. Can't "git pack-objects" get the input directly without overlong printf, something along the lines of...

Quoted 8 lines
        P1=$(git -C .git/objects/pack pack-objects pack <<-EOF
                $A
                $B
                $C
                ...
                $R
                EOF
        )
Quoted 7 lines

Find that no space before <OID>, because git-pack-objects not allow that, and mached parentheses should in the same line.
So Will write like this:

    create_pack_1() {
            P1=$(git -C .git/objects/pack pack-objects pack <<-EOF) &&
    $T

Isn't the whole point of <<-EOF (notice the leading dash) to allow us to indent the here-doc with horizontal tab?

author
#8 lore raw

v6 cover pack-redundant: new algorithm to find min packs

In reply to Jiang Xin
Quoted 8 lines

Sun Chao (my former colleague at Huawei) found a bug of git-pack-redundant. If there are too many packs and many of them overlap each other, running git pack-redundant --all will exhaust all memories and the process will be killed by kernel.

There is a script in commit log of commit 2/5, which can be used to create a repository with lots of redundant packs. Running git pack-redundant --all in it can reproduce this issue.

Quoted 16 lines · Junio C Hamano <[email protected]> 于2019年1月12日周六 上午2:00写道:
Quoted 16 lines
Quoted 11 lines

Yikes. Can't "git pack-objects" get the input directly without overlong printf, something along the lines of...

        P1=$(git -C .git/objects/pack pack-objects pack <<-EOF
                $A
                $B
                $C
                ...
                $R
                EOF
        )

Find that no space before <OID>, because git-pack-objects not allow that, and mached parentheses should in the same line.
So Will write like this:

Quoted 6 lines
Quoted 3 lines
    create_pack_1() {
            P1=$(git -C .git/objects/pack pack-objects pack <<-EOF) &&
    $T

Isn't the whole point of <<-EOF (notice the leading dash) to allow us to indent the here-doc with horizontal tab?

The reason that indents are not stripped even with <<-EOF is I mixed tabs and spaces to make a better align.

If put the heredoc outside the parentheses, it will failed on MacOS, so use the syntax Junio previously suggested.

Quoted 8 lines · SZEDER Gábor <[email protected]> 于2019年1月11日周五 上午9:19写道:

I see that the last patch in this series removes those three unused functions, but that patch should be squashed into this one to keep Git buildable with '-Werror' or DEVELOPER=1.

Furthermore, after building this patch (without '-Werror'), several tests in 't5323-pack-redundant.sh' fail. To avoid the test failure I think the fourth patch ensuring a consistent sort order should be squashed in as well.

Patch 3/5 to 5/5 can be squashed to patch 2/5.

## Changes since reroll v5

1: 40fea5d67f ! 1: 7e4e703083 t5323: test cases for git-pack-redundant

    @@ -22,8 +22,7 @@
     +
     +. ./test-lib.sh
     +
    -+create_commits()
    -+{
    ++create_commits() {
     +	parent=
     +	for name in A B C D E F G H I J K L M N O P Q R
     +	do
    @@ -39,54 +38,98 @@
     +		parent=$oid ||
     +		return 1
     +	done
    -+	git update-ref refs/heads/master $M
    ++	git update-ref refs/heads/master $R
     +}
     +
    -+create_pack_1()
    -+{
    -+	P1=$(cd .git/objects/pack; printf "$T\n$A\n$B\n$C\n$D\n$E\n$F\n$R\n" | git pack-objects pack 2>/dev/null) &&
    ++create_pack_1() {
    ++	P1=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
    ++		$T
    ++		$A
    ++		$B
    ++		$C
    ++		$D
    ++		$E
    ++		$F
    ++		$R
    ++		EOF
    ++	) &&
     +	eval P$P1=P1:$P1
     +}
     +
    -+create_pack_2()
    -+{
    -+	P2=$(cd .git/objects/pack; printf "$B\n$C\n$D\n$E\n$G\n$H\n$I\n" | git pack-objects pack 2>/dev/null) &&
    ++create_pack_2() {
    ++	P2=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
    ++		$B
    ++		$C
    ++		$D
    ++		$E
    ++		$G
    ++		$H
    ++		$I
    ++		EOF
    ++	) &&
     +	eval P$P2=P2:$P2
     +}
     +
    -+create_pack_3()
    -+{
    -+	P3=$(cd .git/objects/pack; printf "$F\n$I\n$J\n$K\n$L\n$M\n" | git pack-objects pack 2>/dev/null) &&
    ++create_pack_3() {
    ++	P3=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
    ++		$F
    ++		$I
    ++		$J
    ++		$K
    ++		$L
    ++		$M
    ++		EOF
    ++	) &&
     +	eval P$P3=P3:$P3
     +}
     +
    -+create_pack_4()
    -+{
    -+	P4=$(cd .git/objects/pack; printf "$J\n$K\n$L\n$M\n$P\n" | git pack-objects pack 2>/dev/null) &&
    ++create_pack_4() {
    ++	P4=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
    ++		$J
    ++		$K
    ++		$L
    ++		$M
    ++		$P
    ++		EOF
    ++	) &&
     +	eval P$P4=P4:$P4
     +}
     +
    -+create_pack_5()
    -+{
    -+	P5=$(cd .git/objects/pack; printf "$G\n$H\n$N\n$O\n" | git pack-objects pack 2>/dev/null) &&
    ++create_pack_5() {
    ++	P5=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
    ++		$G
    ++		$H
    ++		$N
    ++		$O
    ++		EOF
    ++	) &&
     +	eval P$P5=P5:$P5
     +}
     +
    -+create_pack_6()
    -+{
    -+	P6=$(cd .git/objects/pack; printf "$N\n$O\n$Q\n" | git pack-objects pack 2>/dev/null) &&
    ++create_pack_6() {
    ++	P6=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
    ++		$N
    ++		$O
    ++		$Q
    ++		EOF
    ++	) &&
     +	eval P$P6=P6:$P6
     +}
     +
    -+create_pack_7()
    -+{
    -+	P7=$(cd .git/objects/pack; printf "$P\n$Q\n" | git pack-objects pack 2>/dev/null) &&
    ++create_pack_7() {
    ++	P7=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
    ++		$P
    ++		$Q
    ++		EOF
    ++	) &&
     +	eval P$P7=P7:$P7
     +}
     +
    -+create_pack_8()
    -+{
    -+	P8=$(cd .git/objects/pack; printf "$A\n" | git pack-objects pack 2>/dev/null) &&
    ++create_pack_8() {
    ++	P8=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
    ++		$A
    ++		EOF
    ++	) &&
     +	eval P$P8=P8:$P8
     +}
     +
    @@ -110,10 +153,12 @@
     +
     +test_expect_success 'one of pack-2/pack-3 is redundant' '
     +	git pack-redundant --all >out &&
    -+	sed -E -e "s#.*/pack-(.*)\.(idx|pack)#\1#" out | \
    -+		sort -u | \
    -+		while read p; do eval echo "\${P$p}"; done | \
    -+		sort >actual && \
    ++	sed \
    ++		-e "s#.*/pack-\(.*\)\.idx#\1#" \
    ++		-e "s#.*/pack-\(.*\)\.pack#\1#" out |
    ++		sort -u |
    ++		while read p; do eval echo "\${P$p}"; done |
    ++		sort >actual &&
     +	test_cmp expected actual
     +'
     +
    @@ -121,6 +166,7 @@
     +	create_pack_6 && create_pack_7
     +'
     +
    ++# Only after calling create_pack_6, we can use $P6 variable.
     +cat >expected <<EOF
     +P2:$P2
     +P4:$P4
    @@ -129,10 +175,12 @@
     +
     +test_expect_success 'pack 2, 4, and 6 are redundant' '
     +	git pack-redundant --all >out &&
    -+	sed -E -e "s#.*/pack-(.*)\.(idx|pack)#\1#" out | \
    -+		sort -u | \
    -+		while read p; do eval echo "\${P$p}"; done | \
    -+		sort >actual && \
    ++	sed \
    ++		-e "s#.*/pack-\(.*\)\.idx#\1#" \
    ++		-e "s#.*/pack-\(.*\)\.pack#\1#" out |
    ++		sort -u |
    ++		while read p; do eval echo "\${P$p}"; done |
    ++		sort >actual &&
     +	test_cmp expected actual
     +'
     +
    @@ -147,24 +195,26 @@
     +P8:$P8
     +EOF
     +
    -+test_expect_success 'pack-8, subset of pack-1, is also redundant' '
    ++test_expect_success 'pack-8 (subset of pack-1) is also redundant' '
     +	git pack-redundant --all >out &&
    -+	sed -E -e "s#.*/pack-(.*)\.(idx|pack)#\1#" out | \
    -+		sort -u | \
    -+		while read p; do eval echo "\${P$p}"; done | \
    -+		sort >actual && \
    ++	sed \
    ++		-e "s#.*/pack-\(.*\)\.idx#\1#" \
    ++		-e "s#.*/pack-\(.*\)\.pack#\1#" out |
    ++		sort -u |
    ++		while read p; do eval echo "\${P$p}"; done |
    ++		sort >actual &&
     +	test_cmp expected actual
     +'
     +
    -+test_expect_success 'clear loose objects' '
    ++test_expect_success 'clean loose objects' '
     +	git prune-packed &&
     +	find .git/objects -type f | sed -e "/objects\/pack\//d" >out &&
     +	test_must_be_empty out
     +'
     +
    -+test_expect_success 'remove redundant packs' '
    ++test_expect_success 'remove redundant packs and pass fsck' '
     +	git pack-redundant --all | xargs rm &&
    -+	git fsck &&
    ++	git fsck --no-progress &&
     +	git pack-redundant --all >out &&
     +	test_must_be_empty out
     +'

2: 50cd5a5b47 ! 2: 51a9c2d8a5 pack-redundant: new algorithm to find min packs
@@ -67,7 +67,7 @@
Original PR and discussions: https://github.com/jiangxin/git/pull/25

- Signed-off-by: Jiang Xin <[email protected]>
+ Signed-off-by: Jiang Xin <[email protected]>

diff --git a/builtin/pack-redundant.c b/builtin/pack-redundant.c
5: b7ccdea1ad ! 3: c5eb21c23c pack-redundant: remove unused functions

    @@ -6,14 +6,14 @@
         `pll_free`, etc.
     
         Signed-off-by: Sun Chao <[email protected]>
    -    Signed-off-by: Jiang Xin <[email protected]>
    +    Signed-off-by: Jiang Xin <[email protected]>
         Signed-off-by: Junio C Hamano <[email protected]>
     
      diff --git a/builtin/pack-redundant.c b/builtin/pack-redundant.c
      --- a/builtin/pack-redundant.c
      +++ b/builtin/pack-redundant.c
     @@
    - 	size_t all_objects_size;
    + 	struct llist *all_objects;
      } *local_packs = NULL, *altodb_packs = NULL;
      
     -struct pll {
    @@ -105,7 +105,7 @@
     -	diff = llist_copy(list);
     -
     -	while (pl) {
    --		llist_sorted_difference_inplace(diff, pl->remaining_objects);
    +-		llist_sorted_difference_inplace(diff, pl->all_objects);
     -		if (diff->size == 0) { /* we're done */
     -			llist_free(diff);
     -			return 1;
3:  6338c6fad4 ! 4:  1acdd0af1e pack-redundant: rename pack_list.all_objects
    @@ -18,16 +18,7 @@
     +	struct llist *remaining_objects;
      } *local_packs = NULL, *altodb_packs = NULL;
      
    - struct pll {
    -@@
    - 	diff = llist_copy(list);
    - 
    - 	while (pl) {
    --		llist_sorted_difference_inplace(diff, pl->all_objects);
    -+		llist_sorted_difference_inplace(diff, pl->remaining_objects);
    - 		if (diff->size == 0) { /* we're done */
    - 			llist_free(diff);
    - 			return 1;
    + static struct llist_item *free_nodes;
     @@
      {
      	struct pack_list *pl_a = *((struct pack_list **)a);
4:  734f4d8a8b ! 5:  306d515cda pack-redundant: consistent sort method
    @@ -26,7 +26,7 @@
     +	size_t all_objects_size;
      } *local_packs = NULL, *altodb_packs = NULL;
      
    - struct pll {
    + static struct llist_item *free_nodes;
     @@
      	return ret;
      }
    @@ -42,20 +42,24 @@
     -	if (sz_a == sz_b)
     -		return 0;
     -	else if (sz_a < sz_b)
    -+	/* if have the same remaining_objects, big pack first */
    -+	if (pl_a->remaining_objects->size == pl_b->remaining_objects->size)
    ++	if (pl_a->remaining_objects->size == pl_b->remaining_objects->size) {
    ++		/* have the same remaining_objects, big pack first */
     +		if (pl_a->all_objects_size == pl_b->all_objects_size)
     +			return 0;
     +		else if (pl_a->all_objects_size < pl_b->all_objects_size)
     +			return 1;
     +		else
     +			return -1;
    -+
    -+	/* sort according to remaining objects, more remaining objects first */
    -+	if (pl_a->remaining_objects->size < pl_b->remaining_objects->size)
    ++	} else if (pl_a->remaining_objects->size < pl_b->remaining_objects->size) {
    ++		/* sort by remaining objects, more objects first */
      		return 1;
    - 	else
    +-	else
    ++	} else {
      		return -1;
    ++	}
    + }
    + 
    + /* Sort pack_list, greater size of remaining_objects first */
     @@
      	for (n = 0, p = *pl; p; p = p->next)
      		ary[n++] = p;

## This reroll has the following commits:

Jiang Xin (3):
t5323: test cases for git-pack-redundant
pack-redundant: rename pack_list.all_objects
pack-redundant: consistent sort method

Sun Chao (2):
pack-redundant: new algorithm to find min packs
pack-redundant: remove unused functions

 builtin/pack-redundant.c  | 221 +++++++++++++++-----------------------
 t/t5323-pack-redundant.sh | 207 +++++++++++++++++++++++++++++++++++
 2 files changed, 292 insertions(+), 136 deletions(-)
 create mode 100755 t/t5323-pack-redundant.sh
Signature · 1 line
2.20.0.3.gc45e608566
author
#42 lore raw

PATCH v6 1/5 t5323: test cases for git-pack-redundant

In reply to Jiang Xin

From: Jiang Xin <[email protected]>

Add test cases for git pack-redundant to validate new algorithm for git pack-redundant.

---
 t/t5323-pack-redundant.sh | 207 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 207 insertions(+)
 create mode 100755 t/t5323-pack-redundant.sh
t/t5323-pack-redundant.sh new file +207 -0
new file mode 100755
index 0000000000..407838f0e8
@@ -0,0 +1,207 @@
+#!/bin/sh
+#
+# Copyright (c) 2018 Jiang Xin
+#
+
+test_description='git pack-redundant test'
+
+. ./test-lib.sh
+
+create_commits() {
+	parent=
+	for name in A B C D E F G H I J K L M N O P Q R
+	do
+		test_tick &&
+		T=$(git write-tree) &&
+		if test -z "$parent"
+		then
+			oid=$(echo $name | git commit-tree $T)
+		else
+			oid=$(echo $name | git commit-tree -p $parent $T)
+		fi &&
+		eval $name=$oid &&
+		parent=$oid ||
+		return 1
+	done
+	git update-ref refs/heads/master $R
+}
+
+create_pack_1() {
+	P1=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
+		$T
+		$A
+		$B
+		$C
+		$D
+		$E
+		$F
+		$R
+		EOF
+	) &&
+	eval P$P1=P1:$P1
+}
+
+create_pack_2() {
+	P2=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
+		$B
+		$C
+		$D
+		$E
+		$G
+		$H
+		$I
+		EOF
+	) &&
+	eval P$P2=P2:$P2
+}
+
+create_pack_3() {
+	P3=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
+		$F
+		$I
+		$J
+		$K
+		$L
+		$M
+		EOF
+	) &&
+	eval P$P3=P3:$P3
+}
+
+create_pack_4() {
+	P4=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
+		$J
+		$K
+		$L
+		$M
+		$P
+		EOF
+	) &&
+	eval P$P4=P4:$P4
+}
+
+create_pack_5() {
+	P5=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
+		$G
+		$H
+		$N
+		$O
+		EOF
+	) &&
+	eval P$P5=P5:$P5
+}
+
+create_pack_6() {
+	P6=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
+		$N
+		$O
+		$Q
+		EOF
+	) &&
+	eval P$P6=P6:$P6
+}
+
+create_pack_7() {
+	P7=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
+		$P
+		$Q
+		EOF
+	) &&
+	eval P$P7=P7:$P7
+}
+
+create_pack_8() {
+	P8=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
+		$A
+		EOF
+	) &&
+	eval P$P8=P8:$P8
+}
+
+test_expect_success 'setup' '
+	create_commits
+'
+
+test_expect_success 'no redundant packs' '
+	create_pack_1 && create_pack_2 && create_pack_3 &&
+	git pack-redundant --all >out &&
+	test_must_be_empty out
+'
+
+test_expect_success 'create pack 4, 5' '
+	create_pack_4 && create_pack_5
+'
+
+cat >expected <<EOF
+P2:$P2
+EOF
+
+test_expect_success 'one of pack-2/pack-3 is redundant' '
+	git pack-redundant --all >out &&
+	sed \
+		-e "s#.*/pack-\(.*\)\.idx#\1#" \
+		-e "s#.*/pack-\(.*\)\.pack#\1#" out |
+		sort -u |
+		while read p; do eval echo "\${P$p}"; done |
+		sort >actual &&
+	test_cmp expected actual
+'
+
+test_expect_success 'create pack 6, 7' '
+	create_pack_6 && create_pack_7
+'
+
+# Only after calling create_pack_6, we can use $P6 variable.
+cat >expected <<EOF
+P2:$P2
+P4:$P4
+P6:$P6
+EOF
+
+test_expect_success 'pack 2, 4, and 6 are redundant' '
+	git pack-redundant --all >out &&
+	sed \
+		-e "s#.*/pack-\(.*\)\.idx#\1#" \
+		-e "s#.*/pack-\(.*\)\.pack#\1#" out |
+		sort -u |
+		while read p; do eval echo "\${P$p}"; done |
+		sort >actual &&
+	test_cmp expected actual
+'
+
+test_expect_success 'create pack 8' '
+	create_pack_8
+'
+
+cat >expected <<EOF
+P2:$P2
+P4:$P4
+P6:$P6
+P8:$P8
+EOF
+
+test_expect_success 'pack-8 (subset of pack-1) is also redundant' '
+	git pack-redundant --all >out &&
+	sed \
+		-e "s#.*/pack-\(.*\)\.idx#\1#" \
+		-e "s#.*/pack-\(.*\)\.pack#\1#" out |
+		sort -u |
+		while read p; do eval echo "\${P$p}"; done |
+		sort >actual &&
+	test_cmp expected actual
+'
+
+test_expect_success 'clean loose objects' '
+	git prune-packed &&
+	find .git/objects -type f | sed -e "/objects\/pack\//d" >out &&
+	test_must_be_empty out
+'
+
+test_expect_success 'remove redundant packs and pass fsck' '
+	git pack-redundant --all | xargs rm &&
+	git fsck --no-progress &&
+	git pack-redundant --all >out &&
+	test_must_be_empty out
+'
+
+test_done
Signature · 1 line
2.20.0.3.gc45e608566
author
#43 lore raw

PATCH v6 2/5 pack-redundant: new algorithm to find min packs

In reply to Jiang Xin

From: Sun Chao <[email protected]>

When calling git pack-redundant --all, if there are too many local packs and too many redundant objects within them, the too deep iteration of get_permutations will exhaust all the resources, and the process of git pack-redundant will be killed.

The following script could create a repository with too many redundant packs, and running git pack-redundant --all in the test.git repo will die soon.

    #!/bin/sh

    repo="$(pwd)/test.git"
    work="$(pwd)/test"
    i=1
    max=199

    if test -d "$repo" || test -d "$work"; then
    	echo >&2 "ERROR: '$repo' or '$work' already exist"
    	exit 1
    fi

    git init -q --bare "$repo"
    git --git-dir="$repo" config gc.auto 0
    git --git-dir="$repo" config transfer.unpackLimit 0
    git clone -q "$repo" "$work" 2>/dev/null

    while :; do
        cd "$work"
        echo "loop $i: $(date +%s)" >$i
        git add $i
        git commit -q -sm "loop $i"
        git push -q origin HEAD:master
        printf "\rCreate pack %4d/%d\t" $i $max
        if test $i -ge $max; then break; fi

        cd "$repo"
        git repack -q
        if test $(($i % 2)) -eq 0; then
            git repack -aq
            pack=$(ls -t $repo/objects/pack/*.pack | head -1)
            touch "${pack%.pack}.keep"
        fi
        i=$((i+1))
    done
    printf "\ndone\n"

To get the min unique pack list, we can replace the iteration in minimize function with a new algorithm, and this could solve this issue:

1. Get the unique and non_uniqe packs, add the unique packs to the min list.

2. Remove the objects of unique packs from non_unique packs, then each object left in the non_unique packs will have at least two copies.

3. Sort the non_unique packs by the objects' size, more objects first, and add the first non_unique pack to min list.

4. Drop the duplicated objects from other packs in the ordered non_unique pack list, and repeat step 3.

Original PR and discussions: https://github.com/jiangxin/git/pull/25

---
 builtin/pack-redundant.c | 109 ++++++++++++++++++++++++---------------
 1 file changed, 68 insertions(+), 41 deletions(-)
builtin/pack-redundant.c +68 -41
index cf9a9aabd4..3655cc7dc6 100644
@@ -421,14 +421,52 @@ static inline off_t pack_set_bytecount(struct pack_list *pl)
 	return ret;
 }
 
+static int cmp_pack_list_reverse(const void *a, const void *b)
+{
+	struct pack_list *pl_a = *((struct pack_list **)a);
+	struct pack_list *pl_b = *((struct pack_list **)b);
+	size_t sz_a = pl_a->all_objects->size;
+	size_t sz_b = pl_b->all_objects->size;
+
+	if (sz_a == sz_b)
+		return 0;
+	else if (sz_a < sz_b)
+		return 1;
+	else
+		return -1;
+}
+
+/* Sort pack_list, greater size of all_objects first */
+static void sort_pack_list(struct pack_list **pl)
+{
+	struct pack_list **ary, *p;
+	int i;
+	size_t n = pack_list_size(*pl);
+
+	if (n < 2)
+		return;
+
+	/* prepare an array of packed_list for easier sorting */
+	ary = xcalloc(n, sizeof(struct pack_list *));
+	for (n = 0, p = *pl; p; p = p->next)
+		ary[n++] = p;
+
+	QSORT(ary, n, cmp_pack_list_reverse);
+
+	/* link them back again */
+	for (i = 0; i < n - 1; i++)
+		ary[i]->next = ary[i + 1];
+	ary[n - 1]->next = NULL;
+	*pl = ary[0];
+
+	free(ary);
+}
+
+
 static void minimize(struct pack_list **min)
 {
-	struct pack_list *pl, *unique = NULL,
-		*non_unique = NULL, *min_perm = NULL;
-	struct pll *perm, *perm_all, *perm_ok = NULL, *new_perm;
-	struct llist *missing;
-	off_t min_perm_size = 0, perm_size;
-	int n;
+	struct pack_list *pl, *unique = NULL, *non_unique = NULL;
+	struct llist *missing, *unique_pack_objects;
 
 	pl = local_packs;
 	while (pl) {
@@ -446,49 +484,37 @@ static void minimize(struct pack_list **min)
 		pl = pl->next;
 	}
 
+	*min = unique;
+
 	/* return if there are no objects missing from the unique set */
 	if (missing->size == 0) {
-		*min = unique;
 		free(missing);
 		return;
 	}
 
-	/* find the permutations which contain all missing objects */
-	for (n = 1; n <= pack_list_size(non_unique) && !perm_ok; n++) {
-		perm_all = perm = get_permutations(non_unique, n);
-		while (perm) {
-			if (is_superset(perm->pl, missing)) {
-				new_perm = xmalloc(sizeof(struct pll));
-				memcpy(new_perm, perm, sizeof(struct pll));
-				new_perm->next = perm_ok;
-				perm_ok = new_perm;
-			}
-			perm = perm->next;
-		}
-		if (perm_ok)
-			break;
-		pll_free(perm_all);
-	}
-	if (perm_ok == NULL)
-		die("Internal error: No complete sets found!");
-
-	/* find the permutation with the smallest size */
-	perm = perm_ok;
-	while (perm) {
-		perm_size = pack_set_bytecount(perm->pl);
-		if (!min_perm_size || min_perm_size > perm_size) {
-			min_perm_size = perm_size;
-			min_perm = perm->pl;
-		}
-		perm = perm->next;
-	}
-	*min = min_perm;
-	/* add the unique packs to the list */
-	pl = unique;
+	unique_pack_objects = llist_copy(all_objects);
+	llist_sorted_difference_inplace(unique_pack_objects, missing);
+
+	/* remove unique pack objects from the non_unique packs */
+	pl = non_unique;
 	while (pl) {
-		pack_list_insert(min, pl);
+		llist_sorted_difference_inplace(pl->all_objects, unique_pack_objects);
 		pl = pl->next;
 	}
+
+	while (non_unique) {
+		/* sort the non_unique packs, greater size of all_objects first */
+		sort_pack_list(&non_unique);
+		if (non_unique->all_objects->size == 0)
+			break;
+
+		pack_list_insert(min, non_unique);
+
+		for (pl = non_unique->next; pl && pl->all_objects->size > 0;  pl = pl->next)
+			llist_sorted_difference_inplace(pl->all_objects, non_unique->all_objects);
+
+		non_unique = non_unique->next;
+	}
 }
 
 static void load_all_objects(void)
@@ -603,7 +629,7 @@ static void load_all(void)
 int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 {
 	int i;
-	struct pack_list *min, *red, *pl;
+	struct pack_list *min = NULL, *red, *pl;
 	struct llist *ignore;
 	struct object_id *oid;
 	char buf[GIT_MAX_HEXSZ + 2]; /* hex hash + \n + \0 */
@@ -664,6 +690,7 @@ int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 	pl = local_packs;
 	while (pl) {
 		llist_sorted_difference_inplace(pl->unique_objects, ignore);
+		llist_sorted_difference_inplace(pl->all_objects, ignore);
 		pl = pl->next;
 	}
 
Signature · 1 line
2.20.0.3.gc45e608566
author
#44 lore raw

PATCH v6 3/5 pack-redundant: remove unused functions

In reply to Jiang Xin

From: Sun Chao <[email protected]>

Remove unused functions to find min packs, such as get_permutations, pll_free, etc.

---
 builtin/pack-redundant.c | 86 ----------------------------------------
 1 file changed, 86 deletions(-)
builtin/pack-redundant.c +0 -86
index 3655cc7dc6..eac23500ee 100644
@@ -35,11 +35,6 @@ static struct pack_list {
 	struct llist *all_objects;
 } *local_packs = NULL, *altodb_packs = NULL;
 
-struct pll {
-	struct pll *next;
-	struct pack_list *pl;
-};
-
 static struct llist_item *free_nodes;
 
 static inline void llist_item_put(struct llist_item *item)
@@ -63,15 +58,6 @@ static inline struct llist_item *llist_item_get(void)
 	return new_item;
 }
 
-static void llist_free(struct llist *list)
-{
-	while ((list->back = list->front)) {
-		list->front = list->front->next;
-		llist_item_put(list->back);
-	}
-	free(list);
-}
-
 static inline void llist_init(struct llist **list)
 {
 	*list = xmalloc(sizeof(struct llist));
@@ -285,78 +271,6 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
 	}
 }
 
-static void pll_free(struct pll *l)
-{
-	struct pll *old;
-	struct pack_list *opl;
-
-	while (l) {
-		old = l;
-		while (l->pl) {
-			opl = l->pl;
-			l->pl = opl->next;
-			free(opl);
-		}
-		l = l->next;
-		free(old);
-	}
-}
-
-/* all the permutations have to be free()d at the same time,
- * since they refer to each other
- */
-static struct pll * get_permutations(struct pack_list *list, int n)
-{
-	struct pll *subset, *ret = NULL, *new_pll = NULL;
-
-	if (list == NULL || pack_list_size(list) < n || n == 0)
-		return NULL;
-
-	if (n == 1) {
-		while (list) {
-			new_pll = xmalloc(sizeof(*new_pll));
-			new_pll->pl = NULL;
-			pack_list_insert(&new_pll->pl, list);
-			new_pll->next = ret;
-			ret = new_pll;
-			list = list->next;
-		}
-		return ret;
-	}
-
-	while (list->next) {
-		subset = get_permutations(list->next, n - 1);
-		while (subset) {
-			new_pll = xmalloc(sizeof(*new_pll));
-			new_pll->pl = subset->pl;
-			pack_list_insert(&new_pll->pl, list);
-			new_pll->next = ret;
-			ret = new_pll;
-			subset = subset->next;
-		}
-		list = list->next;
-	}
-	return ret;
-}
-
-static int is_superset(struct pack_list *pl, struct llist *list)
-{
-	struct llist *diff;
-
-	diff = llist_copy(list);
-
-	while (pl) {
-		llist_sorted_difference_inplace(diff, pl->all_objects);
-		if (diff->size == 0) { /* we're done */
-			llist_free(diff);
-			return 1;
-		}
-		pl = pl->next;
-	}
-	llist_free(diff);
-	return 0;
-}
-
 static size_t sizeof_union(struct packed_git *p1, struct packed_git *p2)
 {
 	size_t ret = 0;
Signature · 1 line
2.20.0.3.gc45e608566
author
#45 lore raw

PATCH v6 4/5 pack-redundant: rename pack_list.all_objects

In reply to Jiang Xin

From: Jiang Xin <[email protected]>

New algorithm uses pack_list.all_objects to track remaining objects, so rename it to pack_list.remaining_objects.

---
 builtin/pack-redundant.c | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)
builtin/pack-redundant.c +18 -18
index eac23500ee..64eec3e297 100644
@@ -32,7 +32,7 @@ static struct pack_list {
 	struct pack_list *next;
 	struct packed_git *pack;
 	struct llist *unique_objects;
-	struct llist *all_objects;
+	struct llist *remaining_objects;
 } *local_packs = NULL, *altodb_packs = NULL;
 
 static struct llist_item *free_nodes;
@@ -339,8 +339,8 @@ static int cmp_pack_list_reverse(const void *a, const void *b)
 {
 	struct pack_list *pl_a = *((struct pack_list **)a);
 	struct pack_list *pl_b = *((struct pack_list **)b);
-	size_t sz_a = pl_a->all_objects->size;
-	size_t sz_b = pl_b->all_objects->size;
+	size_t sz_a = pl_a->remaining_objects->size;
+	size_t sz_b = pl_b->remaining_objects->size;
 
 	if (sz_a == sz_b)
 		return 0;
@@ -350,7 +350,7 @@ static int cmp_pack_list_reverse(const void *a, const void *b)
 		return -1;
 }
 
-/* Sort pack_list, greater size of all_objects first */
+/* Sort pack_list, greater size of remaining_objects first */
 static void sort_pack_list(struct pack_list **pl)
 {
 	struct pack_list **ary, *p;
@@ -394,7 +394,7 @@ static void minimize(struct pack_list **min)
 	missing = llist_copy(all_objects);
 	pl = unique;
 	while (pl) {
-		llist_sorted_difference_inplace(missing, pl->all_objects);
+		llist_sorted_difference_inplace(missing, pl->remaining_objects);
 		pl = pl->next;
 	}
 
@@ -412,20 +412,20 @@ static void minimize(struct pack_list **min)
 	/* remove unique pack objects from the non_unique packs */
 	pl = non_unique;
 	while (pl) {
-		llist_sorted_difference_inplace(pl->all_objects, unique_pack_objects);
+		llist_sorted_difference_inplace(pl->remaining_objects, unique_pack_objects);
 		pl = pl->next;
 	}
 
 	while (non_unique) {
-		/* sort the non_unique packs, greater size of all_objects first */
+		/* sort the non_unique packs, greater size of remaining_objects first */
 		sort_pack_list(&non_unique);
-		if (non_unique->all_objects->size == 0)
+		if (non_unique->remaining_objects->size == 0)
 			break;
 
 		pack_list_insert(min, non_unique);
 
-		for (pl = non_unique->next; pl && pl->all_objects->size > 0;  pl = pl->next)
-			llist_sorted_difference_inplace(pl->all_objects, non_unique->all_objects);
+		for (pl = non_unique->next; pl && pl->remaining_objects->size > 0;  pl = pl->next)
+			llist_sorted_difference_inplace(pl->remaining_objects, non_unique->remaining_objects);
 
 		non_unique = non_unique->next;
 	}
@@ -440,7 +440,7 @@ static void load_all_objects(void)
 
 	while (pl) {
 		hint = NULL;
-		l = pl->all_objects->front;
+		l = pl->remaining_objects->front;
 		while (l) {
 			hint = llist_insert_sorted_unique(all_objects,
 							  l->oid, hint);
@@ -451,7 +451,7 @@ static void load_all_objects(void)
 	/* remove objects present in remote packs */
 	pl = altodb_packs;
 	while (pl) {
-		llist_sorted_difference_inplace(all_objects, pl->all_objects);
+		llist_sorted_difference_inplace(all_objects, pl->remaining_objects);
 		pl = pl->next;
 	}
 }
@@ -477,10 +477,10 @@ static void scan_alt_odb_packs(void)
 		local = local_packs;
 		while (local) {
 			llist_sorted_difference_inplace(local->unique_objects,
-							alt->all_objects);
+							alt->remaining_objects);
 			local = local->next;
 		}
-		llist_sorted_difference_inplace(all_objects, alt->all_objects);
+		llist_sorted_difference_inplace(all_objects, alt->remaining_objects);
 		alt = alt->next;
 	}
 }
@@ -495,7 +495,7 @@ static struct pack_list * add_pack(struct packed_git *p)
 		return NULL;
 
 	l.pack = p;
-	llist_init(&l.all_objects);
+	llist_init(&l.remaining_objects);
 
 	if (open_pack_index(p))
 		return NULL;
@@ -504,11 +504,11 @@ static struct pack_list * add_pack(struct packed_git *p)
 	base += 256 * 4 + ((p->index_version < 2) ? 4 : 8);
 	step = the_hash_algo->rawsz + ((p->index_version < 2) ? 4 : 0);
 	while (off < p->num_objects * step) {
-		llist_insert_back(l.all_objects, (const struct object_id *)(base + off));
+		llist_insert_back(l.remaining_objects, (const struct object_id *)(base + off));
 		off += step;
 	}
 	/* this list will be pruned in cmp_two_packs later */
-	l.unique_objects = llist_copy(l.all_objects);
+	l.unique_objects = llist_copy(l.remaining_objects);
 	if (p->pack_local)
 		return pack_list_insert(&local_packs, &l);
 	else
@@ -604,7 +604,7 @@ int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 	pl = local_packs;
 	while (pl) {
 		llist_sorted_difference_inplace(pl->unique_objects, ignore);
-		llist_sorted_difference_inplace(pl->all_objects, ignore);
+		llist_sorted_difference_inplace(pl->remaining_objects, ignore);
 		pl = pl->next;
 	}
 
Signature · 1 line
2.20.0.3.gc45e608566
author
#46 lore raw

PATCH v6 5/5 pack-redundant: consistent sort method

In reply to Jiang Xin

From: Jiang Xin <[email protected]>

SZEDER reported that test case t5323 has different test result on MacOS. This is because cmp_pack_list_reverse cannot give identical result when two pack being sorted has the same size of remaining_objects.

Changes to the sorting function will make consistent test result for t5323.

The new algorithm to find redundant packs is a trade-off to save memory resources, and the result of it may be different with old one, and may be not the best result sometimes. Update t5323 for the new algorithm.

---
 builtin/pack-redundant.c  | 24 ++++++++++++++++--------
 t/t5323-pack-redundant.sh |  2 +-
 2 files changed, 17 insertions(+), 9 deletions(-)
builtin/pack-redundant.c +16 -8
index 64eec3e297..4448e58a10 100644
@@ -33,6 +33,7 @@ static struct pack_list {
 	struct packed_git *pack;
 	struct llist *unique_objects;
 	struct llist *remaining_objects;
+	size_t all_objects_size;
 } *local_packs = NULL, *altodb_packs = NULL;
 
 static struct llist_item *free_nodes;
@@ -335,19 +336,25 @@ static inline off_t pack_set_bytecount(struct pack_list *pl)
 	return ret;
 }
 
-static int cmp_pack_list_reverse(const void *a, const void *b)
+static int cmp_remaining_objects(const void *a, const void *b)
 {
 	struct pack_list *pl_a = *((struct pack_list **)a);
 	struct pack_list *pl_b = *((struct pack_list **)b);
-	size_t sz_a = pl_a->remaining_objects->size;
-	size_t sz_b = pl_b->remaining_objects->size;
 
-	if (sz_a == sz_b)
-		return 0;
-	else if (sz_a < sz_b)
+	if (pl_a->remaining_objects->size == pl_b->remaining_objects->size) {
+		/* have the same remaining_objects, big pack first */
+		if (pl_a->all_objects_size == pl_b->all_objects_size)
+			return 0;
+		else if (pl_a->all_objects_size < pl_b->all_objects_size)
+			return 1;
+		else
+			return -1;
+	} else if (pl_a->remaining_objects->size < pl_b->remaining_objects->size) {
+		/* sort by remaining objects, more objects first */
 		return 1;
-	else
+	} else {
 		return -1;
+	}
 }
 
 /* Sort pack_list, greater size of remaining_objects first */
@@ -365,7 +372,7 @@ static void sort_pack_list(struct pack_list **pl)
 	for (n = 0, p = *pl; p; p = p->next)
 		ary[n++] = p;
 
-	QSORT(ary, n, cmp_pack_list_reverse);
+	QSORT(ary, n, cmp_remaining_objects);
 
 	/* link them back again */
 	for (i = 0; i < n - 1; i++)
@@ -507,6 +514,7 @@ static struct pack_list * add_pack(struct packed_git *p)
 		llist_insert_back(l.remaining_objects, (const struct object_id *)(base + off));
 		off += step;
 	}
+	l.all_objects_size = l.remaining_objects->size;
 	/* this list will be pruned in cmp_two_packs later */
 	l.unique_objects = llist_copy(l.remaining_objects);
 	if (p->pack_local)
t/t5323-pack-redundant.sh +1 -1
index 407838f0e8..663328ab30 100755
@@ -133,7 +133,7 @@ test_expect_success 'create pack 4, 5' '
 '
 
 cat >expected <<EOF
-P2:$P2
+P3:$P3
 EOF
 
 test_expect_success 'one of pack-2/pack-3 is redundant' '
Signature · 1 line
2.20.0.3.gc45e608566
#62 lore raw

PATCH 1/1 test-lint: sed -E (or -a, -l) are not portable

In reply to SZEDER Gábor

From: Torsten Bögershausen <[email protected]>

From man sed (on a Mac OS X box):
The -E, -a and -i options are non-standard FreeBSD extensions and may not be available on other operating systems.

From man sed on a Linux box:
REGULAR EXPRESSIONS
POSIX.2 BREs should be supported, but they aren't completely because of performance problems. The \n sequence in a regular expression matches the newline character, and similarly for \a, \t, and other sequences. The -E option switches to using extended regular expressions instead; the -E option has been supported for years by GNU sed, and is now included in POSIX.

Well, there are still a lot of systems out there, which don't support it.

Beside that, see IEEE Std 1003.1TM-2017
http://pubs.opengroup.org/onlinepubs/9699919799/
does not mention -E either.

To be on the safe side, don't allow it.

---

I am somewhat unsure if we should disable all options except -e -f -n instead ?
/\bsed\s+-[^efn]/ and err 'Not portable option with sed. Only -n -e -f are portable';

That would cause a false positive in t9001 here:
"--cc-cmd=./cccmd-sed --suppress-cc=self"

which could either be fixed by an anchor:
/^\s*sed\s+-[^efn]/

Or by allowing '--' like this:
/\bsed\s+-[^-efn]/

Any thoughts, please ?

t/check-non-portable-shell.pl | 2 +-

 1 file changed, 1 insertion(+), 1 deletion(-)
t/check-non-portable-shell.pl +1 -1
index b45bdac688..96b6afdeb8 100755
@@ -35,7 +35,7 @@ sub err {
 		chomp;
 	}
 
-	/\bsed\s+-i/ and err 'sed -i is not portable';
+	/\bsed\s+-[Eail]/ and err 'Not portable option with sed. Only -e -f -n are portable';
 	/\becho\s+-[neE]/ and err 'echo with option is not portable (use printf)';
 	/^\s*declare\s+/ and err 'arrays/declare not portable';
 	/^\s*[^#]\s*which\s/ and err 'which is not portable (use type)';

--
2.20.1.2.gb21ebb671

#63 lore raw

Re: [PATCH/RFC v1 1/1] test-lint: sed -E (or -a, -l) are not portable

Quoted 10 lines · On Tue, Jan 15, 2019 at 3:31 PM <[email protected]> wrote:

From man sed (on a Mac OS X box):
The -E, -a and -i options are non-standard FreeBSD extensions and may not be available on other operating systems.
[...]
To be on the safe side, don't allow it.

---

t/check-non-portable-shell.pl sub err { · line 38

Quoted 5 lines
t/check-non-portable-shell.pl +1 -1
@@ -35,7 +35,7 @@ sub err {
-       /\bsed\s+-i/ and err 'sed -i is not portable';
+       /\bsed\s+-[Eail]/ and err 'Not portable option with sed. Only -e -f -n are portable';
        /\becho\s+-[neE]/ and err 'echo with option is not portable (use printf)';
        /^\s*declare\s+/ and err 'arrays/declare not portable';
        /^\s*[^#]\s*which\s/ and err 'which is not portable (use type)';

Please update the new message to be more consistent with existing surrounding error messages. For instance:

    err 'sed -i/-a/-l/-E not portable (use only -e/-f/-n)'

or something. Thanks.

#64 lore raw

Re: [PATCH/RFC v1 1/1] test-lint: sed -E (or -a, -l) are not portable

In reply to [email protected]
Quoted 50 lines · On Tue, Jan 15 2019, [email protected] wrote:

From: Torsten Bögershausen <[email protected]>

From man sed (on a Mac OS X box):
The -E, -a and -i options are non-standard FreeBSD extensions and may not be available on other operating systems.

From man sed on a Linux box:
REGULAR EXPRESSIONS
POSIX.2 BREs should be supported, but they aren't completely because of performance problems. The \n sequence in a regular expression matches the newline character, and similarly for \a, \t, and other sequences. The -E option switches to using extended regular expressions instead; the -E option has been supported for years by GNU sed, and is now included in POSIX.

Well, there are still a lot of systems out there, which don't support it.

Beside that, see IEEE Std 1003.1TM-2017
http://pubs.opengroup.org/onlinepubs/9699919799/
does not mention -E either.

To be on the safe side, don't allow it.

---

I am somewhat unsure if we should disable all options except -e -f -n instead ?
/\bsed\s+-[^efn]/ and err 'Not portable option with sed. Only -n -e -f are portable';

That would cause a false positive in t9001 here:
"--cc-cmd=./cccmd-sed --suppress-cc=self"

which could either be fixed by an anchor:
/^\s*sed\s+-[^efn]/

Or by allowing '--' like this:
/\bsed\s+-[^-efn]/

Any thoughts, please ?

t/check-non-portable-shell.pl | 2 +-

 1 file changed, 1 insertion(+), 1 deletion(-)
t/check-non-portable-shell.pl +0 -0
index b45bdac688..96b6afdeb8 100755

t/check-non-portable-shell.pl sub err { · line 41

Quoted 8 lines
t/check-non-portable-shell.pl +1 -1
@@ -35,7 +35,7 @@ sub err {
 		chomp;
 	}
 
-	/\bsed\s+-i/ and err 'sed -i is not portable';
+	/\bsed\s+-[Eail]/ and err 'Not portable option with sed. Only -e -f -n are portable';
 	/\becho\s+-[neE]/ and err 'echo with option is not portable (use printf)';
 	/^\s*declare\s+/ and err 'arrays/declare not portable';
 	/^\s*[^#]\s*which\s/ and err 'which is not portable (use type)';

I'd just go for your /\bsed\s+-[^-efn]/ suggestion. Just a note if we do go for the whitelist: According to GNU sed's manpage -E is also known as -r, so /\bsed\s+-[Erail]/ would be better.

#65 lore raw

PATCH v2 1/1 test-lint: Only use only sed [-n] [-e command] [-f command_file]

In reply to SZEDER Gábor

From: Torsten Bögershausen <[email protected]>

From man sed (on a Mac OS X box):
The -E, -a and -i options are non-standard FreeBSD extensions and may not be available on other operating systems.

From man sed on a Linux box:
REGULAR EXPRESSIONS
POSIX.2 BREs should be supported, but they aren't completely because of performance problems. The \n sequence in a regular expression matches the newline character, and similarly for \a, \t, and other sequences.
The -E option switches to using extended regular expressions instead; the -E option has been supported for years by GNU sed, and is now included in POSIX.

Well, there are still a lot of systems out there, which don't support it. Beside that, IEEE Std 1003.1TM-2017, see
http://pubs.opengroup.org/onlinepubs/9699919799/
does not mention -E either.

To be on the safe side, don't allow -E (or -r, which is GNU). Change check-non-portable-shell.pl to only accept the portable options:
sed [-n] [-e command] [-f command_file]

---
 t/check-non-portable-shell.pl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
t/check-non-portable-shell.pl +1 -1
index b45bdac688..6c798608a9 100755
@@ -35,7 +35,7 @@ sub err {
 		chomp;
 	}
 
-	/\bsed\s+-i/ and err 'sed -i is not portable';
+	/\bsed\s+-[^efn]\s+/ and err 'Not portable option with sed (use only [-n] [-e command] [-f command_file])';
 	/\becho\s+-[neE]/ and err 'echo with option is not portable (use printf)';
 	/^\s*declare\s+/ and err 'arrays/declare not portable';
 	/^\s*[^#]\s*which\s/ and err 'which is not portable (use type)';

--
2.20.1.2.gb21ebb671

#66 lore raw

Re: [PATCH/RFC v2 1/1] test-lint: Only use only sed [-n] [-e command] [-f command_file]

t/check-non-portable-shell.pl sub err { · line 38

Quoted 8 lines · [email protected] writes:

From: Torsten Bögershausen <[email protected]>

From man sed (on a Mac OS X box):
The -E, -a and -i options are non-standard FreeBSD extensions and may not be available on other operating systems.

t/check-non-portable-shell.pl +1 -1
-	/\bsed\s+-i/ and err 'sed -i is not portable';
+	/\bsed\s+-[^efn]\s+/ and err 'Not portable option with sed (use only [-n] [-e command] [-f command_file])';

"sed -n -i -E -e 's/foo/bar/p'" won't be caught with this as an error, but that's OK ;-).

Is this still an RFC patch?

#67 lore raw

Re: [PATCH/RFC v2 1/1] test-lint: Only use only sed [-n] [-e command] [-f command_file]

Quoted 8 lines · Junio C Hamano
Quoted 6 lines · [email protected] writes:

From: Torsten Bögershausen <[email protected]>

From man sed (on a Mac OS X box):
The -E, -a and -i options are non-standard FreeBSD extensions and may not be available on other operating systems.

Quoted 8 lines

t/check-non-portable-shell.pl sub err { · line 38

Quoted 2 lines
t/check-non-portable-shell.pl +1 -1
-	/\bsed\s+-i/ and err 'sed -i is not portable';
+	/\bsed\s+-[^efn]\s+/ and err 'Not portable option with sed (use only [-n] [-e command] [-f command_file])';

"sed -n -i -E -e 's/foo/bar/p'" won't be caught with this as an error, but that's OK ;-).

Is this still an RFC patch?

It seems as if everybody is happy with it,
so it may be ready for for pu.

#68 lore raw

Re: [PATCH/RFC v2 1/1] test-lint: Only use only sed [-n] [-e command] [-f command_file]

Quoted 11 lines · Torsten Bögershausen
Quoted 7 lines · Junio C Hamano

t/check-non-portable-shell.pl sub err { · line 38

Quoted 2 lines · [email protected] writes:
t/check-non-portable-shell.pl +1 -1
-    /\bsed\s+-i/ and err 'sed -i is not portable';
+    /\bsed\s+-[^efn]\s+/ and err 'Not portable option with sed (use only [-n] [-e command] [-f command_file])';

"sed -n -i -E -e 's/foo/bar/p'" won't be caught with this as an error, but that's OK ;-).
Is this still an RFC patch?

It seems as if everybody is happy with it,
so it may be ready for for pu.

I'd still prefer to see a more terse[1] (and not capitalized) message to be consistent with existing error messages and to keep the reported errors more compact overall to make them easier to digest[2,3]:

err 'sed option not portable (use only -n, -e, -f)'

But that's just a very minor nit.

[1]: http://public-inbox.org/git/CAPig+cSeDNYFGYC2WznjW3zYMJCWZbZFY1KM5H5ir2L=Jxwy7w@mail.gmail.com/
[2]: http://public-inbox.org/git/[email protected]/
[3]: http://public-inbox.org/git/[email protected]/

#69 lore raw

Re: [PATCH/RFC v2 1/1] test-lint: Only use only sed [-n] [-e command] [-f command_file]

Quoted 10 lines · Eric Sunshine
Quoted 9 lines · Torsten Bögershausen
Quoted 7 lines · Junio C Hamano

t/check-non-portable-shell.pl sub err { · line 38

Quoted 2 lines · [email protected] writes:
t/check-non-portable-shell.pl +1 -1
-    /\bsed\s+-i/ and err 'sed -i is not portable';
+    /\bsed\s+-[^efn]\s+/ and err 'Not portable option with sed (use only [-n] [-e command] [-f command_file])';

"sed -n -i -E -e 's/foo/bar/p'" won't be caught with this as an error, but that's OK ;-).
Is this still an RFC patch?

Quoted 9 lines
Quoted 2 lines

It seems as if everybody is happy with it,
so it may be ready for for pu.

I'd still prefer to see a more terse[1] (and not capitalized) message to be consistent with existing error messages and to keep the reported errors more compact overall to make them easier to digest[2,3]:

err 'sed option not portable (use only -n, -e, -f)'

That's OK for me - lets see if there are more comments.

Quoted 6 lines

But that's just a very minor nit.

[1]: http://public-inbox.org/git/CAPig+cSeDNYFGYC2WznjW3zYMJCWZbZFY1KM5H5ir2L=Jxwy7w@mail.gmail.com/
[2]: http://public-inbox.org/git/[email protected]/
[3]: http://public-inbox.org/git/[email protected]/

#70 lore raw

Re: [PATCH/RFC v2 1/1] test-lint: Only use only sed [-n] [-e command] [-f command_file]

Quoted 7 lines · Torsten Bögershausen
Quoted 5 lines

I'd still prefer to see a more terse[1] (and not capitalized) message to be consistent with existing error messages and to keep the reported errors more compact overall to make them easier to digest[2,3]:

err 'sed option not portable (use only -n, -e, -f)'

That's OK for me - lets see if there are more comments.

Thanks, both. Eric's proposed message looks good to me too.

#71 lore raw

Re: [PATCH/RFC v2 1/1] test-lint: Only use only sed [-n] [-e command] [-f command_file]

Quoted 11 lines · Junio C Hamano
Quoted 7 lines · Torsten Bögershausen
Quoted 5 lines

I'd still prefer to see a more terse[1] (and not capitalized) message to be consistent with existing error messages and to keep the reported errors more compact overall to make them easier to digest[2,3]:

err 'sed option not portable (use only -n, -e, -f)'

That's OK for me - lets see if there are more comments.

Thanks, both. Eric's proposed message looks good to me too.

Do you want to ammend the patch locally ?
Or should I send a new version ?

#72 lore raw

Re: [PATCH/RFC v2 1/1] test-lint: Only use only sed [-n] [-e command] [-f command_file]

Quoted 3 lines · Torsten Bögershausen
Quoted 1 line

Thanks, both. Eric's proposed message looks good to me too.

Do you want to ammend the patch locally ?

I just amended it; thanks.

author
#9 lore raw

v7 cover pack-redundant: new algorithm to find min packs

In reply to Jiang Xin

Sun Chao (my former colleague at Huawei) found a bug of git-pack-redundant. If there are too many packs and many of them overlap each other, running git pack-redundant --all will exhaust all memories and the process will be killed by kernel.

There is a script in commit log of commit 3/6, which can be used to create a repository with lots of redundant packs. Running git pack-redundant --all in it can reproduce this issue.

Quoted 8 lines · Derrick Stolee <[email protected]> 于2019年1月20日周日 上午9:08写道:

Here is today's test coverage report.

builtin/pack-redundant.c
a338d10395 builtin/pack-redundant.c 339) static int cmp_remaining_objects(const void *a, const void *b) e4e2c2884e builtin/pack-redundant.c 341) struct pack_list *pl_a = *((struct pack_list **)a); e4e2c2884e builtin/pack-redundant.c 342) struct pack_list *pl_b = *((struct pack_list **)b);
...

Add new test cases in t5323 for better test coverage.

## Changes since reroll v6

* Add new test cases in t5323.

* Add new patch 2/6 (pack-redundant: delay creation of unique_objects), which will fix a bug which fail to find redundant packs if turn on alt-odb searching with --alt-odb option. This is because alt-odb objects are only remove in unique_objects fields but not in all_objects fields of pack_list.

## Range diff

1: be6555ae60 ! 1: 799e804d5e t5323: test cases for git-pack-redundant

    @@ -43,7 +43,7 @@
     +}
     +
     +create_pack_1 () {
    -+	P1=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
    ++	P1=$(git -C objects/pack pack-objects -q pack <<-EOF
     +		$T
     +		$A
     +		$B
    @@ -58,7 +58,7 @@
     +}
     +
     +create_pack_2 () {
    -+	P2=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
    ++	P2=$(git -C objects/pack pack-objects -q pack <<-EOF
     +		$B
     +		$C
     +		$D
    @@ -72,7 +72,7 @@
     +}
     +
     +create_pack_3 () {
    -+	P3=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
    ++	P3=$(git -C objects/pack pack-objects -q pack <<-EOF
     +		$F
     +		$I
     +		$J
    @@ -85,7 +85,7 @@
     +}
     +
     +create_pack_4 () {
    -+	P4=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
    ++	P4=$(git -C objects/pack pack-objects -q pack <<-EOF
     +		$J
     +		$K
     +		$L
    @@ -97,7 +97,7 @@
     +}
     +
     +create_pack_5 () {
    -+	P5=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
    ++	P5=$(git -C objects/pack pack-objects -q pack <<-EOF
     +		$G
     +		$H
     +		$N
    @@ -108,7 +108,7 @@
     +}
     +
     +create_pack_6 () {
    -+	P6=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
    ++	P6=$(git -C objects/pack pack-objects -q pack <<-EOF
     +		$N
     +		$O
     +		$Q
    @@ -118,7 +118,7 @@
     +}
     +
     +create_pack_7 () {
    -+	P7=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
    ++	P7=$(git -C objects/pack pack-objects -q pack <<-EOF
     +		$P
     +		$Q
     +		EOF
    @@ -127,18 +127,37 @@
     +}
     +
     +create_pack_8 () {
    -+	P8=$(git -C .git/objects/pack pack-objects -q pack <<-EOF
    ++	P8=$(git -C objects/pack pack-objects -q pack <<-EOF
     +		$A
     +		EOF
     +	) &&
     +	eval P$P8=P8:$P8
     +}
     +
    -+test_expect_success 'setup' '
    ++format_packfiles () {
    ++	sed \
    ++		-e "s#.*/pack-\(.*\)\.idx#\1#" \
    ++		-e "s#.*/pack-\(.*\)\.pack#\1#" |
    ++	sort -u |
    ++	while read p
    ++	do
    ++		if test -z "$(eval echo \${P$p})"
    ++		then
    ++			echo $p
    ++		else
    ++			eval echo "\${P$p}"
    ++		fi
    ++	done |
    ++	sort
    ++}
    ++
    ++test_expect_success 'setup master.git' '
    ++	git init --bare master.git &&
    ++	cd master.git &&
     +	create_commits
     +'
     +
    -+test_expect_success 'no redundant packs' '
    ++test_expect_success 'no redundant for pack 1, 2, 3' '
     +	create_pack_1 && create_pack_2 && create_pack_3 &&
     +	git pack-redundant --all >out &&
     +	test_must_be_empty out
    @@ -154,12 +173,7 @@
     +
     +test_expect_success 'one of pack-2/pack-3 is redundant' '
     +	git pack-redundant --all >out &&
    -+	sed \
    -+		-e "s#.*/pack-\(.*\)\.idx#\1#" \
    -+		-e "s#.*/pack-\(.*\)\.pack#\1#" out |
    -+		sort -u |
    -+		while read p; do eval echo "\${P$p}"; done |
    -+		sort >actual &&
    ++	format_packfiles <out >actual &&
     +	test_cmp expected actual
     +'
     +
    @@ -176,12 +190,7 @@
     +
     +test_expect_success 'pack 2, 4, and 6 are redundant' '
     +	git pack-redundant --all >out &&
    -+	sed \
    -+		-e "s#.*/pack-\(.*\)\.idx#\1#" \
    -+		-e "s#.*/pack-\(.*\)\.pack#\1#" out |
    -+		sort -u |
    -+		while read p; do eval echo "\${P$p}"; done |
    -+		sort >actual &&
    ++	format_packfiles <out >actual &&
     +	test_cmp expected actual
     +'
     +
    @@ -198,18 +207,13 @@
     +
     +test_expect_success 'pack-8 (subset of pack-1) is also redundant' '
     +	git pack-redundant --all >out &&
    -+	sed \
    -+		-e "s#.*/pack-\(.*\)\.idx#\1#" \
    -+		-e "s#.*/pack-\(.*\)\.pack#\1#" out |
    -+		sort -u |
    -+		while read p; do eval echo "\${P$p}"; done |
    -+		sort >actual &&
    ++	format_packfiles <out >actual &&
     +	test_cmp expected actual
     +'
     +
     +test_expect_success 'clean loose objects' '
     +	git prune-packed &&
    -+	find .git/objects -type f | sed -e "/objects\/pack\//d" >out &&
    ++	find objects -type f | sed -e "/objects\/pack\//d" >out &&
     +	test_must_be_empty out
     +'
     +
    @@ -220,4 +224,115 @@
     +	test_must_be_empty out
     +'
     +
    ++test_expect_success 'setup shared.git' '
    ++	cd "$TRASH_DIRECTORY" &&
    ++	git clone -q --mirror master.git shared.git &&
    ++	cd shared.git &&
    ++	printf "../../master.git/objects" >objects/info/alternates
    ++'
    ++
    ++test_expect_success 'no redundant packs without --alt-odb' '
    ++	git pack-redundant --all >out &&
    ++	test_must_be_empty out
    ++'
    ++
    ++cat >expected <<EOF
    ++P1:$P1
    ++P3:$P3
    ++P5:$P5
    ++P7:$P7
    ++EOF
    ++
    ++test_expect_success 'pack-redundant --verbose: show duplicate packs in stderr' '
    ++	git pack-redundant --all --verbose >out 2>out.err &&
    ++	test_must_be_empty out &&
    ++	grep "pack$" out.err | format_packfiles >actual &&
    ++	test_cmp expected actual
    ++'
    ++
    ++cat >expected <<EOF
    ++fatal: Zero packs found!
    ++EOF
    ++
    ++test_expect_success 'remove redundant packs by alt-odb, no packs left' '
    ++	git pack-redundant --all --alt-odb | xargs rm &&
    ++	git fsck --no-progress &&
    ++	test_must_fail git pack-redundant --all --alt-odb >actual 2>&1 &&
    ++	test_cmp expected actual
    ++'
    ++
    ++create_commits_others () {
    ++	parent=$(git rev-parse HEAD)
    ++	for name in X Y Z
    ++	do
    ++		test_tick &&
    ++		T=$(git write-tree) &&
    ++		if test -z "$parent"
    ++		then
    ++			oid=$(echo $name | git commit-tree $T)
    ++		else
    ++			oid=$(echo $name | git commit-tree -p $parent $T)
    ++		fi &&
    ++		eval $name=$oid &&
    ++		parent=$oid ||
    ++		return 1
    ++	done
    ++	git update-ref refs/heads/master $Z
    ++}
    ++
    ++create_pack_x1 () {
    ++	Px1=$(git -C objects/pack pack-objects -q pack <<-EOF
    ++		$X
    ++		$Y
    ++		$Z
    ++		$A
    ++		$B
    ++		$C
    ++		EOF
    ++	) &&
    ++	eval P${Px1}=Px1:${Px1}
    ++}
    ++
    ++create_pack_x2 () {
    ++	Px2=$(git -C objects/pack pack-objects -q pack <<-EOF
    ++		$X
    ++		$Y
    ++		$Z
    ++		$D
    ++		$E
    ++		$F
    ++		EOF
    ++	) &&
    ++	eval P${Px2}=Px2:${Px2}
    ++}
    ++
    ++test_expect_success 'new objects and packs in shared.git' '
    ++	create_commits_others &&
    ++	create_pack_x1 &&
    ++	create_pack_x2 &&
    ++	git pack-redundant --all >out &&
    ++	test_must_be_empty out
    ++'
    ++
    ++test_expect_success 'one pack is redundant' '
    ++	git pack-redundant --all --alt-odb >out &&
    ++	format_packfiles <out >actual &&
    ++	test_line_count = 1 actual
    ++'
    ++
    ++cat >expected <<EOF
    ++Px1:$Px1
    ++Px2:$Px2
    ++EOF
    ++
    ++test_expect_success 'set ignore objects and all two packs are redundant' '
    ++	git pack-redundant --all --alt-odb >out <<-EOF &&
    ++		$X
    ++		$Y
    ++		$Z
    ++		EOF
    ++	format_packfiles <out >actual &&
    ++	test_cmp expected actual
    ++'
    ++
     +test_done
-:  ---------- > 2:  520f6277fb pack-redundant: delay creation of unique_objects
2:  e4e2c2884e ! 3:  ab1c2c4950 pack-redundant: new algorithm to find min packs

@@ -64,6 +64,9 @@
4. Drop the duplicated objects from other packs in the ordered non_unique pack list, and repeat step 3.

+ Some test cases will fail on Mac OS X. Mark them and will resolve in
+ later commit.
+
Original PR and discussions: https://github.com/jiangxin/git/pull/25

@@ -213,11 +216,61 @@
struct llist *ignore;
struct object_id *oid;
char buf[GIT_MAX_HEXSZ + 2]; /* hex hash + \n + \0 */
+
+ diff --git a/t/t5323-pack-redundant.sh b/t/t5323-pack-redundant.sh
+ --- a/t/t5323-pack-redundant.sh
+ +++ b/t/t5323-pack-redundant.sh
@@
- pl = local_packs;
- while (pl) {
- llist_sorted_difference_inplace(pl->unique_objects, ignore);
-+ llist_sorted_difference_inplace(pl->all_objects, ignore);
- pl = pl->next;
- }
+ P2:$P2
+ EOF
+
+-test_expect_success 'one of pack-2/pack-3 is redundant' ' ++test_expect_failure 'one of pack-2/pack-3 is redundant' '
+ git pack-redundant --all >out &&
+ format_packfiles <out >actual &&
+ test_cmp expected actual
+@@
+ P6:$P6
+ EOF
+
+-test_expect_success 'pack 2, 4, and 6 are redundant' ' ++test_expect_failure 'pack 2, 4, and 6 are redundant' '
+ git pack-redundant --all >out &&
+ format_packfiles <out >actual &&
+ test_cmp expected actual
+@@
+ P8:$P8
+ EOF
+
+-test_expect_success 'pack-8 (subset of pack-1) is also redundant' ' ++test_expect_failure 'pack-8 (subset of pack-1) is also redundant' '
+ git pack-redundant --all >out &&
+ format_packfiles <out >actual &&
+ test_cmp expected actual
+@@
+ test_must_be_empty out
+ '
+
+-test_expect_success 'remove redundant packs and pass fsck' ' ++test_expect_failure 'remove redundant packs and pass fsck' '
+ git pack-redundant --all | xargs rm &&
+ git fsck --no-progress &&
+ git pack-redundant --all >out &&
+@@
+ printf "../../master.git/objects" >objects/info/alternates
+ '
+
+-test_expect_success 'no redundant packs without --alt-odb' ' ++test_expect_failure 'no redundant packs without --alt-odb' '
+ git pack-redundant --all >out &&
+ test_must_be_empty out
+ '
+@@
+ P7:$P7
+ EOF

+-test_expect_success 'pack-redundant --verbose: show duplicate packs in stderr' ' ++test_expect_failure 'pack-redundant --verbose: show duplicate packs in stderr' '
+ git pack-redundant --all --verbose >out 2>out.err &&
+ test_must_be_empty out &&
+ grep "pack$" out.err | format_packfiles >actual &&

3:  e60b134e66 = 4:  3c3a7ea40f pack-redundant: remove unused functions
4:  cb7e0336fc ! 5:  bc4b681f40 pack-redundant: rename pack_list.all_objects
    @@ -20,6 +20,18 @@
      } *local_packs = NULL, *altodb_packs = NULL;
      
      static struct llist_item *free_nodes;
    +@@
    + 	const unsigned int hashsz = the_hash_algo->rawsz;
    + 
    + 	if (!p1->unique_objects)
    +-		p1->unique_objects = llist_copy(p1->all_objects);
    ++		p1->unique_objects = llist_copy(p1->remaining_objects);
    + 	if (!p2->unique_objects)
    +-		p2->unique_objects = llist_copy(p2->all_objects);
    ++		p2->unique_objects = llist_copy(p2->remaining_objects);
    + 
    + 	p1_base = p1->pack->index_data;
    + 	p2_base = p2->pack->index_data;
     @@
      {
      	struct pack_list *pl_a = *((struct pack_list **)a);
    @@ -94,10 +106,12 @@
      	}
      }
     @@
    + 	while (alt) {
      		local = local_packs;
      		while (local) {
    - 			llist_sorted_difference_inplace(local->unique_objects,
    +-			llist_sorted_difference_inplace(local->all_objects,
     -							alt->all_objects);
    ++			llist_sorted_difference_inplace(local->remaining_objects,
     +							alt->remaining_objects);
      			local = local->next;
      		}
    @@ -123,16 +137,11 @@
     +		llist_insert_back(l.remaining_objects, (const struct object_id *)(base + off));
      		off += step;
      	}
    - 	/* this list will be pruned in cmp_two_packs later */
    --	l.unique_objects = llist_copy(l.all_objects);
    -+	l.unique_objects = llist_copy(l.remaining_objects);
    - 	if (p->pack_local)
    - 		return pack_list_insert(&local_packs, &l);
    - 	else
    + 	l.unique_objects = NULL;
     @@
    + 	llist_sorted_difference_inplace(all_objects, ignore);
      	pl = local_packs;
      	while (pl) {
    - 		llist_sorted_difference_inplace(pl->unique_objects, ignore);
     -		llist_sorted_difference_inplace(pl->all_objects, ignore);
     +		llist_sorted_difference_inplace(pl->remaining_objects, ignore);
      		pl = pl->next;

5: a338d10395 ! 6: 6cfba5b4b2 pack-redundant: consistent sort method
@@ -75,9 +75,9 @@
off += step;
}
+ l.all_objects_size = l.remaining_objects->size;
- /* this list will be pruned in cmp_two_packs later */
- l.unique_objects = llist_copy(l.remaining_objects);
+ l.unique_objects = NULL;
if (p->pack_local)
+ return pack_list_insert(&local_packs, &l);

diff --git a/t/t5323-pack-redundant.sh b/t/t5323-pack-redundant.sh --- a/t/t5323-pack-redundant.sh
@@ -90,4 +90,53 @@
+P3:$P3
EOF

- test_expect_success 'one of pack-2/pack-3 is redundant' ' +-test_expect_failure 'one of pack-2/pack-3 is redundant' ' ++test_expect_success 'one of pack-2/pack-3 is redundant' '
+ git pack-redundant --all >out &&
+ format_packfiles <out >actual &&
+ test_cmp expected actual
+@@
+ P6:$P6
+ EOF
+
+-test_expect_failure 'pack 2, 4, and 6 are redundant' ' ++test_expect_success 'pack 2, 4, and 6 are redundant' '
+ git pack-redundant --all >out &&
+ format_packfiles <out >actual &&
+ test_cmp expected actual
+@@
+ P8:$P8
+ EOF
+
+-test_expect_failure 'pack-8 (subset of pack-1) is also redundant' ' ++test_expect_success 'pack-8 (subset of pack-1) is also redundant' '
+ git pack-redundant --all >out &&
+ format_packfiles <out >actual &&
+ test_cmp expected actual
+@@
+ test_must_be_empty out
+ '
+
+-test_expect_failure 'remove redundant packs and pass fsck' ' ++test_expect_success 'remove redundant packs and pass fsck' '
+ git pack-redundant --all | xargs rm &&
+ git fsck --no-progress &&
+ git pack-redundant --all >out &&
+@@
+ printf "../../master.git/objects" >objects/info/alternates
+ '
+
+-test_expect_failure 'no redundant packs without --alt-odb' ' ++test_expect_success 'no redundant packs without --alt-odb' '
+ git pack-redundant --all >out &&
+ test_must_be_empty out
+ '
+@@
+ P7:$P7
+ EOF
+
+-test_expect_failure 'pack-redundant --verbose: show duplicate packs in stderr' ' ++test_expect_success 'pack-redundant --verbose: show duplicate packs in stderr' '
+ git pack-redundant --all --verbose >out 2>out.err &&
+ test_must_be_empty out &&
+ grep "pack$" out.err | format_packfiles >actual &&

--
Jiang Xin (4):
t5323: test cases for git-pack-redundant
pack-redundant: delay creation of unique_objects
pack-redundant: rename pack_list.all_objects
pack-redundant: consistent sort method

Sun Chao (2):
pack-redundant: new algorithm to find min packs
pack-redundant: remove unused functions

 builtin/pack-redundant.c  | 233 +++++++++++----------------
 t/t5323-pack-redundant.sh | 322 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 415 insertions(+), 140 deletions(-)
 create mode 100755 t/t5323-pack-redundant.sh
Signature · 1 line
2.20.1.103.ged0fc2ca7b
author
#27 lore raw

PATCH v7 1/6 t5323: test cases for git-pack-redundant

In reply to Jiang Xin

From: Jiang Xin <[email protected]>

Add test cases for git pack-redundant to validate new algorithm for git pack-redundant.

---
 t/t5323-pack-redundant.sh | 322 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 322 insertions(+)
 create mode 100755 t/t5323-pack-redundant.sh
t/t5323-pack-redundant.sh new file +322 -0
new file mode 100755
index 0000000000..710fe9884c
@@ -0,0 +1,322 @@
+#!/bin/sh
+#
+# Copyright (c) 2018 Jiang Xin
+#
+
+test_description='git pack-redundant test'
+
+. ./test-lib.sh
+
+create_commits () {
+	parent=
+	for name in A B C D E F G H I J K L M N O P Q R
+	do
+		test_tick &&
+		T=$(git write-tree) &&
+		if test -z "$parent"
+		then
+			oid=$(echo $name | git commit-tree $T)
+		else
+			oid=$(echo $name | git commit-tree -p $parent $T)
+		fi &&
+		eval $name=$oid &&
+		parent=$oid ||
+		return 1
+	done
+	git update-ref refs/heads/master $R
+}
+
+create_pack_1 () {
+	P1=$(git -C objects/pack pack-objects -q pack <<-EOF
+		$T
+		$A
+		$B
+		$C
+		$D
+		$E
+		$F
+		$R
+		EOF
+	) &&
+	eval P$P1=P1:$P1
+}
+
+create_pack_2 () {
+	P2=$(git -C objects/pack pack-objects -q pack <<-EOF
+		$B
+		$C
+		$D
+		$E
+		$G
+		$H
+		$I
+		EOF
+	) &&
+	eval P$P2=P2:$P2
+}
+
+create_pack_3 () {
+	P3=$(git -C objects/pack pack-objects -q pack <<-EOF
+		$F
+		$I
+		$J
+		$K
+		$L
+		$M
+		EOF
+	) &&
+	eval P$P3=P3:$P3
+}
+
+create_pack_4 () {
+	P4=$(git -C objects/pack pack-objects -q pack <<-EOF
+		$J
+		$K
+		$L
+		$M
+		$P
+		EOF
+	) &&
+	eval P$P4=P4:$P4
+}
+
+create_pack_5 () {
+	P5=$(git -C objects/pack pack-objects -q pack <<-EOF
+		$G
+		$H
+		$N
+		$O
+		EOF
+	) &&
+	eval P$P5=P5:$P5
+}
+
+create_pack_6 () {
+	P6=$(git -C objects/pack pack-objects -q pack <<-EOF
+		$N
+		$O
+		$Q
+		EOF
+	) &&
+	eval P$P6=P6:$P6
+}
+
+create_pack_7 () {
+	P7=$(git -C objects/pack pack-objects -q pack <<-EOF
+		$P
+		$Q
+		EOF
+	) &&
+	eval P$P7=P7:$P7
+}
+
+create_pack_8 () {
+	P8=$(git -C objects/pack pack-objects -q pack <<-EOF
+		$A
+		EOF
+	) &&
+	eval P$P8=P8:$P8
+}
+
+format_packfiles () {
+	sed \
+		-e "s#.*/pack-\(.*\)\.idx#\1#" \
+		-e "s#.*/pack-\(.*\)\.pack#\1#" |
+	sort -u |
+	while read p
+	do
+		if test -z "$(eval echo \${P$p})"
+		then
+			echo $p
+		else
+			eval echo "\${P$p}"
+		fi
+	done |
+	sort
+}
+
+test_expect_success 'setup master.git' '
+	git init --bare master.git &&
+	cd master.git &&
+	create_commits
+'
+
+test_expect_success 'no redundant for pack 1, 2, 3' '
+	create_pack_1 && create_pack_2 && create_pack_3 &&
+	git pack-redundant --all >out &&
+	test_must_be_empty out
+'
+
+test_expect_success 'create pack 4, 5' '
+	create_pack_4 && create_pack_5
+'
+
+cat >expected <<EOF
+P2:$P2
+EOF
+
+test_expect_success 'one of pack-2/pack-3 is redundant' '
+	git pack-redundant --all >out &&
+	format_packfiles <out >actual &&
+	test_cmp expected actual
+'
+
+test_expect_success 'create pack 6, 7' '
+	create_pack_6 && create_pack_7
+'
+
+# Only after calling create_pack_6, we can use $P6 variable.
+cat >expected <<EOF
+P2:$P2
+P4:$P4
+P6:$P6
+EOF
+
+test_expect_success 'pack 2, 4, and 6 are redundant' '
+	git pack-redundant --all >out &&
+	format_packfiles <out >actual &&
+	test_cmp expected actual
+'
+
+test_expect_success 'create pack 8' '
+	create_pack_8
+'
+
+cat >expected <<EOF
+P2:$P2
+P4:$P4
+P6:$P6
+P8:$P8
+EOF
+
+test_expect_success 'pack-8 (subset of pack-1) is also redundant' '
+	git pack-redundant --all >out &&
+	format_packfiles <out >actual &&
+	test_cmp expected actual
+'
+
+test_expect_success 'clean loose objects' '
+	git prune-packed &&
+	find objects -type f | sed -e "/objects\/pack\//d" >out &&
+	test_must_be_empty out
+'
+
+test_expect_success 'remove redundant packs and pass fsck' '
+	git pack-redundant --all | xargs rm &&
+	git fsck --no-progress &&
+	git pack-redundant --all >out &&
+	test_must_be_empty out
+'
+
+test_expect_success 'setup shared.git' '
+	cd "$TRASH_DIRECTORY" &&
+	git clone -q --mirror master.git shared.git &&
+	cd shared.git &&
+	printf "../../master.git/objects" >objects/info/alternates
+'
+
+test_expect_success 'no redundant packs without --alt-odb' '
+	git pack-redundant --all >out &&
+	test_must_be_empty out
+'
+
+cat >expected <<EOF
+P1:$P1
+P3:$P3
+P5:$P5
+P7:$P7
+EOF
+
+test_expect_success 'pack-redundant --verbose: show duplicate packs in stderr' '
+	git pack-redundant --all --verbose >out 2>out.err &&
+	test_must_be_empty out &&
+	grep "pack$" out.err | format_packfiles >actual &&
+	test_cmp expected actual
+'
+
+cat >expected <<EOF
+fatal: Zero packs found!
+EOF
+
+test_expect_success 'remove redundant packs by alt-odb, no packs left' '
+	git pack-redundant --all --alt-odb | xargs rm &&
+	git fsck --no-progress &&
+	test_must_fail git pack-redundant --all --alt-odb >actual 2>&1 &&
+	test_cmp expected actual
+'
+
+create_commits_others () {
+	parent=$(git rev-parse HEAD)
+	for name in X Y Z
+	do
+		test_tick &&
+		T=$(git write-tree) &&
+		if test -z "$parent"
+		then
+			oid=$(echo $name | git commit-tree $T)
+		else
+			oid=$(echo $name | git commit-tree -p $parent $T)
+		fi &&
+		eval $name=$oid &&
+		parent=$oid ||
+		return 1
+	done
+	git update-ref refs/heads/master $Z
+}
+
+create_pack_x1 () {
+	Px1=$(git -C objects/pack pack-objects -q pack <<-EOF
+		$X
+		$Y
+		$Z
+		$A
+		$B
+		$C
+		EOF
+	) &&
+	eval P${Px1}=Px1:${Px1}
+}
+
+create_pack_x2 () {
+	Px2=$(git -C objects/pack pack-objects -q pack <<-EOF
+		$X
+		$Y
+		$Z
+		$D
+		$E
+		$F
+		EOF
+	) &&
+	eval P${Px2}=Px2:${Px2}
+}
+
+test_expect_success 'new objects and packs in shared.git' '
+	create_commits_others &&
+	create_pack_x1 &&
+	create_pack_x2 &&
+	git pack-redundant --all >out &&
+	test_must_be_empty out
+'
+
+test_expect_success 'one pack is redundant' '
+	git pack-redundant --all --alt-odb >out &&
+	format_packfiles <out >actual &&
+	test_line_count = 1 actual
+'
+
+cat >expected <<EOF
+Px1:$Px1
+Px2:$Px2
+EOF
+
+test_expect_success 'set ignore objects and all two packs are redundant' '
+	git pack-redundant --all --alt-odb >out <<-EOF &&
+		$X
+		$Y
+		$Z
+		EOF
+	format_packfiles <out >actual &&
+	test_cmp expected actual
+'
+
+test_done
Signature · 1 line
2.20.1.103.ged0fc2ca7b
author
#34 lore raw

PATCH v7 2/6 pack-redundant: delay creation of unique_objects

In reply to Jiang Xin

From: Jiang Xin <[email protected]>

Instead of initializing unique_objects in add_pack(), copy from all_objects in cmp_two_packs(), when unwanted objects are removed from all_objects.

This will save memory (no allocate memory for alt-odb packs), and run llist_sorted_difference_inplace() only once when removing ignored objects and removing objects in alt-odb in scan_alt_odb_packs().

---
 builtin/pack-redundant.c | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)
builtin/pack-redundant.c +10 -5
index cf9a9aabd4..f7dab0ec60 100644
@@ -254,6 +254,11 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
 	struct llist_item *p1_hint = NULL, *p2_hint = NULL;
 	const unsigned int hashsz = the_hash_algo->rawsz;
 
+	if (!p1->unique_objects)
+		p1->unique_objects = llist_copy(p1->all_objects);
+	if (!p2->unique_objects)
+		p2->unique_objects = llist_copy(p2->all_objects);
+
 	p1_base = p1->pack->index_data;
 	p2_base = p2->pack->index_data;
 	p1_base += 256 * 4 + ((p1->pack->index_version < 2) ? 4 : 8);
@@ -536,7 +541,7 @@ static void scan_alt_odb_packs(void)
 	while (alt) {
 		local = local_packs;
 		while (local) {
-			llist_sorted_difference_inplace(local->unique_objects,
+			llist_sorted_difference_inplace(local->all_objects,
 							alt->all_objects);
 			local = local->next;
 		}
@@ -567,8 +572,7 @@ static struct pack_list * add_pack(struct packed_git *p)
 		llist_insert_back(l.all_objects, (const struct object_id *)(base + off));
 		off += step;
 	}
-	/* this list will be pruned in cmp_two_packs later */
-	l.unique_objects = llist_copy(l.all_objects);
+	l.unique_objects = NULL;
 	if (p->pack_local)
 		return pack_list_insert(&local_packs, &l);
 	else
@@ -646,7 +650,6 @@ int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 
 	load_all_objects();
 
-	cmp_local_packs();
 	if (alt_odb)
 		scan_alt_odb_packs();
 
@@ -663,10 +666,12 @@ int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 	llist_sorted_difference_inplace(all_objects, ignore);
 	pl = local_packs;
 	while (pl) {
-		llist_sorted_difference_inplace(pl->unique_objects, ignore);
+		llist_sorted_difference_inplace(pl->all_objects, ignore);
 		pl = pl->next;
 	}
 
+	cmp_local_packs();
+
 	minimize(&min);
 
 	if (verbose) {
Signature · 1 line
2.20.1.103.ged0fc2ca7b
author
#35 lore raw

PATCH v7 3/6 pack-redundant: new algorithm to find min packs

In reply to Jiang Xin

From: Sun Chao <[email protected]>

When calling git pack-redundant --all, if there are too many local packs and too many redundant objects within them, the too deep iteration of get_permutations will exhaust all the resources, and the process of git pack-redundant will be killed.

The following script could create a repository with too many redundant packs, and running git pack-redundant --all in the test.git repo will die soon.

    #!/bin/sh

    repo="$(pwd)/test.git"
    work="$(pwd)/test"
    i=1
    max=199

    if test -d "$repo" || test -d "$work"; then
    	echo >&2 "ERROR: '$repo' or '$work' already exist"
    	exit 1
    fi

    git init -q --bare "$repo"
    git --git-dir="$repo" config gc.auto 0
    git --git-dir="$repo" config transfer.unpackLimit 0
    git clone -q "$repo" "$work" 2>/dev/null

    while :; do
        cd "$work"
        echo "loop $i: $(date +%s)" >$i
        git add $i
        git commit -q -sm "loop $i"
        git push -q origin HEAD:master
        printf "\rCreate pack %4d/%d\t" $i $max
        if test $i -ge $max; then break; fi

        cd "$repo"
        git repack -q
        if test $(($i % 2)) -eq 0; then
            git repack -aq
            pack=$(ls -t $repo/objects/pack/*.pack | head -1)
            touch "${pack%.pack}.keep"
        fi
        i=$((i+1))
    done
    printf "\ndone\n"

To get the min unique pack list, we can replace the iteration in minimize function with a new algorithm, and this could solve this issue:

1. Get the unique and non_uniqe packs, add the unique packs to the min list.

2. Remove the objects of unique packs from non_unique packs, then each object left in the non_unique packs will have at least two copies.

3. Sort the non_unique packs by the objects' size, more objects first, and add the first non_unique pack to min list.

4. Drop the duplicated objects from other packs in the ordered non_unique pack list, and repeat step 3.

Some test cases will fail on Mac OS X. Mark them and will resolve in later commit.

Original PR and discussions: https://github.com/jiangxin/git/pull/25

---
 builtin/pack-redundant.c  | 108 +++++++++++++++++++++++---------------
 t/t5323-pack-redundant.sh |  12 ++---
 2 files changed, 73 insertions(+), 47 deletions(-)
builtin/pack-redundant.c +67 -41
index f7dab0ec60..b3f3bb5e09 100644
@@ -426,14 +426,52 @@ static inline off_t pack_set_bytecount(struct pack_list *pl)
 	return ret;
 }
 
+static int cmp_pack_list_reverse(const void *a, const void *b)
+{
+	struct pack_list *pl_a = *((struct pack_list **)a);
+	struct pack_list *pl_b = *((struct pack_list **)b);
+	size_t sz_a = pl_a->all_objects->size;
+	size_t sz_b = pl_b->all_objects->size;
+
+	if (sz_a == sz_b)
+		return 0;
+	else if (sz_a < sz_b)
+		return 1;
+	else
+		return -1;
+}
+
+/* Sort pack_list, greater size of all_objects first */
+static void sort_pack_list(struct pack_list **pl)
+{
+	struct pack_list **ary, *p;
+	int i;
+	size_t n = pack_list_size(*pl);
+
+	if (n < 2)
+		return;
+
+	/* prepare an array of packed_list for easier sorting */
+	ary = xcalloc(n, sizeof(struct pack_list *));
+	for (n = 0, p = *pl; p; p = p->next)
+		ary[n++] = p;
+
+	QSORT(ary, n, cmp_pack_list_reverse);
+
+	/* link them back again */
+	for (i = 0; i < n - 1; i++)
+		ary[i]->next = ary[i + 1];
+	ary[n - 1]->next = NULL;
+	*pl = ary[0];
+
+	free(ary);
+}
+
+
 static void minimize(struct pack_list **min)
 {
-	struct pack_list *pl, *unique = NULL,
-		*non_unique = NULL, *min_perm = NULL;
-	struct pll *perm, *perm_all, *perm_ok = NULL, *new_perm;
-	struct llist *missing;
-	off_t min_perm_size = 0, perm_size;
-	int n;
+	struct pack_list *pl, *unique = NULL, *non_unique = NULL;
+	struct llist *missing, *unique_pack_objects;
 
 	pl = local_packs;
 	while (pl) {
@@ -451,49 +489,37 @@ static void minimize(struct pack_list **min)
 		pl = pl->next;
 	}
 
+	*min = unique;
+
 	/* return if there are no objects missing from the unique set */
 	if (missing->size == 0) {
-		*min = unique;
 		free(missing);
 		return;
 	}
 
-	/* find the permutations which contain all missing objects */
-	for (n = 1; n <= pack_list_size(non_unique) && !perm_ok; n++) {
-		perm_all = perm = get_permutations(non_unique, n);
-		while (perm) {
-			if (is_superset(perm->pl, missing)) {
-				new_perm = xmalloc(sizeof(struct pll));
-				memcpy(new_perm, perm, sizeof(struct pll));
-				new_perm->next = perm_ok;
-				perm_ok = new_perm;
-			}
-			perm = perm->next;
-		}
-		if (perm_ok)
-			break;
-		pll_free(perm_all);
-	}
-	if (perm_ok == NULL)
-		die("Internal error: No complete sets found!");
-
-	/* find the permutation with the smallest size */
-	perm = perm_ok;
-	while (perm) {
-		perm_size = pack_set_bytecount(perm->pl);
-		if (!min_perm_size || min_perm_size > perm_size) {
-			min_perm_size = perm_size;
-			min_perm = perm->pl;
-		}
-		perm = perm->next;
-	}
-	*min = min_perm;
-	/* add the unique packs to the list */
-	pl = unique;
+	unique_pack_objects = llist_copy(all_objects);
+	llist_sorted_difference_inplace(unique_pack_objects, missing);
+
+	/* remove unique pack objects from the non_unique packs */
+	pl = non_unique;
 	while (pl) {
-		pack_list_insert(min, pl);
+		llist_sorted_difference_inplace(pl->all_objects, unique_pack_objects);
 		pl = pl->next;
 	}
+
+	while (non_unique) {
+		/* sort the non_unique packs, greater size of all_objects first */
+		sort_pack_list(&non_unique);
+		if (non_unique->all_objects->size == 0)
+			break;
+
+		pack_list_insert(min, non_unique);
+
+		for (pl = non_unique->next; pl && pl->all_objects->size > 0;  pl = pl->next)
+			llist_sorted_difference_inplace(pl->all_objects, non_unique->all_objects);
+
+		non_unique = non_unique->next;
+	}
 }
 
 static void load_all_objects(void)
@@ -607,7 +633,7 @@ static void load_all(void)
 int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 {
 	int i;
-	struct pack_list *min, *red, *pl;
+	struct pack_list *min = NULL, *red, *pl;
 	struct llist *ignore;
 	struct object_id *oid;
 	char buf[GIT_MAX_HEXSZ + 2]; /* hex hash + \n + \0 */
t/t5323-pack-redundant.sh +6 -6
index 710fe9884c..d176a6eb91 100755
@@ -155,7 +155,7 @@ cat >expected <<EOF
 P2:$P2
 EOF
 
-test_expect_success 'one of pack-2/pack-3 is redundant' '
+test_expect_failure 'one of pack-2/pack-3 is redundant' '
 	git pack-redundant --all >out &&
 	format_packfiles <out >actual &&
 	test_cmp expected actual
@@ -172,7 +172,7 @@ P4:$P4
 P6:$P6
 EOF
 
-test_expect_success 'pack 2, 4, and 6 are redundant' '
+test_expect_failure 'pack 2, 4, and 6 are redundant' '
 	git pack-redundant --all >out &&
 	format_packfiles <out >actual &&
 	test_cmp expected actual
@@ -189,7 +189,7 @@ P6:$P6
 P8:$P8
 EOF
 
-test_expect_success 'pack-8 (subset of pack-1) is also redundant' '
+test_expect_failure 'pack-8 (subset of pack-1) is also redundant' '
 	git pack-redundant --all >out &&
 	format_packfiles <out >actual &&
 	test_cmp expected actual
@@ -201,7 +201,7 @@ test_expect_success 'clean loose objects' '
 	test_must_be_empty out
 '
 
-test_expect_success 'remove redundant packs and pass fsck' '
+test_expect_failure 'remove redundant packs and pass fsck' '
 	git pack-redundant --all | xargs rm &&
 	git fsck --no-progress &&
 	git pack-redundant --all >out &&
@@ -215,7 +215,7 @@ test_expect_success 'setup shared.git' '
 	printf "../../master.git/objects" >objects/info/alternates
 '
 
-test_expect_success 'no redundant packs without --alt-odb' '
+test_expect_failure 'no redundant packs without --alt-odb' '
 	git pack-redundant --all >out &&
 	test_must_be_empty out
 '
@@ -227,7 +227,7 @@ P5:$P5
 P7:$P7
 EOF
 
-test_expect_success 'pack-redundant --verbose: show duplicate packs in stderr' '
+test_expect_failure 'pack-redundant --verbose: show duplicate packs in stderr' '
 	git pack-redundant --all --verbose >out 2>out.err &&
 	test_must_be_empty out &&
 	grep "pack$" out.err | format_packfiles >actual &&
Signature · 1 line
2.20.1.103.ged0fc2ca7b
author
#38 lore raw

PATCH v7 4/6 pack-redundant: remove unused functions

In reply to Jiang Xin

From: Sun Chao <[email protected]>

Remove unused functions to find min packs, such as get_permutations, pll_free, etc.

---
 builtin/pack-redundant.c | 86 ----------------------------------------
 1 file changed, 86 deletions(-)
builtin/pack-redundant.c +0 -86
index b3f3bb5e09..b83d55031a 100644
@@ -35,11 +35,6 @@ static struct pack_list {
 	struct llist *all_objects;
 } *local_packs = NULL, *altodb_packs = NULL;
 
-struct pll {
-	struct pll *next;
-	struct pack_list *pl;
-};
-
 static struct llist_item *free_nodes;
 
 static inline void llist_item_put(struct llist_item *item)
@@ -63,15 +58,6 @@ static inline struct llist_item *llist_item_get(void)
 	return new_item;
 }
 
-static void llist_free(struct llist *list)
-{
-	while ((list->back = list->front)) {
-		list->front = list->front->next;
-		llist_item_put(list->back);
-	}
-	free(list);
-}
-
 static inline void llist_init(struct llist **list)
 {
 	*list = xmalloc(sizeof(struct llist));
@@ -290,78 +276,6 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
 	}
 }
 
-static void pll_free(struct pll *l)
-{
-	struct pll *old;
-	struct pack_list *opl;
-
-	while (l) {
-		old = l;
-		while (l->pl) {
-			opl = l->pl;
-			l->pl = opl->next;
-			free(opl);
-		}
-		l = l->next;
-		free(old);
-	}
-}
-
-/* all the permutations have to be free()d at the same time,
- * since they refer to each other
- */
-static struct pll * get_permutations(struct pack_list *list, int n)
-{
-	struct pll *subset, *ret = NULL, *new_pll = NULL;
-
-	if (list == NULL || pack_list_size(list) < n || n == 0)
-		return NULL;
-
-	if (n == 1) {
-		while (list) {
-			new_pll = xmalloc(sizeof(*new_pll));
-			new_pll->pl = NULL;
-			pack_list_insert(&new_pll->pl, list);
-			new_pll->next = ret;
-			ret = new_pll;
-			list = list->next;
-		}
-		return ret;
-	}
-
-	while (list->next) {
-		subset = get_permutations(list->next, n - 1);
-		while (subset) {
-			new_pll = xmalloc(sizeof(*new_pll));
-			new_pll->pl = subset->pl;
-			pack_list_insert(&new_pll->pl, list);
-			new_pll->next = ret;
-			ret = new_pll;
-			subset = subset->next;
-		}
-		list = list->next;
-	}
-	return ret;
-}
-
-static int is_superset(struct pack_list *pl, struct llist *list)
-{
-	struct llist *diff;
-
-	diff = llist_copy(list);
-
-	while (pl) {
-		llist_sorted_difference_inplace(diff, pl->all_objects);
-		if (diff->size == 0) { /* we're done */
-			llist_free(diff);
-			return 1;
-		}
-		pl = pl->next;
-	}
-	llist_free(diff);
-	return 0;
-}
-
 static size_t sizeof_union(struct packed_git *p1, struct packed_git *p2)
 {
 	size_t ret = 0;
Signature · 1 line
2.20.1.103.ged0fc2ca7b
author
#40 lore raw

PATCH v7 5/6 pack-redundant: rename pack_list.all_objects

In reply to Jiang Xin

From: Jiang Xin <[email protected]>

New algorithm uses pack_list.all_objects to track remaining objects, so rename it to pack_list.remaining_objects.

---
 builtin/pack-redundant.c | 40 ++++++++++++++++++++--------------------
 1 file changed, 20 insertions(+), 20 deletions(-)
builtin/pack-redundant.c +20 -20
index b83d55031a..c145852b8b 100644
@@ -32,7 +32,7 @@ static struct pack_list {
 	struct pack_list *next;
 	struct packed_git *pack;
 	struct llist *unique_objects;
-	struct llist *all_objects;
+	struct llist *remaining_objects;
 } *local_packs = NULL, *altodb_packs = NULL;
 
 static struct llist_item *free_nodes;
@@ -241,9 +241,9 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
 	const unsigned int hashsz = the_hash_algo->rawsz;
 
 	if (!p1->unique_objects)
-		p1->unique_objects = llist_copy(p1->all_objects);
+		p1->unique_objects = llist_copy(p1->remaining_objects);
 	if (!p2->unique_objects)
-		p2->unique_objects = llist_copy(p2->all_objects);
+		p2->unique_objects = llist_copy(p2->remaining_objects);
 
 	p1_base = p1->pack->index_data;
 	p2_base = p2->pack->index_data;
@@ -344,8 +344,8 @@ static int cmp_pack_list_reverse(const void *a, const void *b)
 {
 	struct pack_list *pl_a = *((struct pack_list **)a);
 	struct pack_list *pl_b = *((struct pack_list **)b);
-	size_t sz_a = pl_a->all_objects->size;
-	size_t sz_b = pl_b->all_objects->size;
+	size_t sz_a = pl_a->remaining_objects->size;
+	size_t sz_b = pl_b->remaining_objects->size;
 
 	if (sz_a == sz_b)
 		return 0;
@@ -355,7 +355,7 @@ static int cmp_pack_list_reverse(const void *a, const void *b)
 		return -1;
 }
 
-/* Sort pack_list, greater size of all_objects first */
+/* Sort pack_list, greater size of remaining_objects first */
 static void sort_pack_list(struct pack_list **pl)
 {
 	struct pack_list **ary, *p;
@@ -399,7 +399,7 @@ static void minimize(struct pack_list **min)
 	missing = llist_copy(all_objects);
 	pl = unique;
 	while (pl) {
-		llist_sorted_difference_inplace(missing, pl->all_objects);
+		llist_sorted_difference_inplace(missing, pl->remaining_objects);
 		pl = pl->next;
 	}
 
@@ -417,20 +417,20 @@ static void minimize(struct pack_list **min)
 	/* remove unique pack objects from the non_unique packs */
 	pl = non_unique;
 	while (pl) {
-		llist_sorted_difference_inplace(pl->all_objects, unique_pack_objects);
+		llist_sorted_difference_inplace(pl->remaining_objects, unique_pack_objects);
 		pl = pl->next;
 	}
 
 	while (non_unique) {
-		/* sort the non_unique packs, greater size of all_objects first */
+		/* sort the non_unique packs, greater size of remaining_objects first */
 		sort_pack_list(&non_unique);
-		if (non_unique->all_objects->size == 0)
+		if (non_unique->remaining_objects->size == 0)
 			break;
 
 		pack_list_insert(min, non_unique);
 
-		for (pl = non_unique->next; pl && pl->all_objects->size > 0;  pl = pl->next)
-			llist_sorted_difference_inplace(pl->all_objects, non_unique->all_objects);
+		for (pl = non_unique->next; pl && pl->remaining_objects->size > 0;  pl = pl->next)
+			llist_sorted_difference_inplace(pl->remaining_objects, non_unique->remaining_objects);
 
 		non_unique = non_unique->next;
 	}
@@ -445,7 +445,7 @@ static void load_all_objects(void)
 
 	while (pl) {
 		hint = NULL;
-		l = pl->all_objects->front;
+		l = pl->remaining_objects->front;
 		while (l) {
 			hint = llist_insert_sorted_unique(all_objects,
 							  l->oid, hint);
@@ -456,7 +456,7 @@ static void load_all_objects(void)
 	/* remove objects present in remote packs */
 	pl = altodb_packs;
 	while (pl) {
-		llist_sorted_difference_inplace(all_objects, pl->all_objects);
+		llist_sorted_difference_inplace(all_objects, pl->remaining_objects);
 		pl = pl->next;
 	}
 }
@@ -481,11 +481,11 @@ static void scan_alt_odb_packs(void)
 	while (alt) {
 		local = local_packs;
 		while (local) {
-			llist_sorted_difference_inplace(local->all_objects,
-							alt->all_objects);
+			llist_sorted_difference_inplace(local->remaining_objects,
+							alt->remaining_objects);
 			local = local->next;
 		}
-		llist_sorted_difference_inplace(all_objects, alt->all_objects);
+		llist_sorted_difference_inplace(all_objects, alt->remaining_objects);
 		alt = alt->next;
 	}
 }
@@ -500,7 +500,7 @@ static struct pack_list * add_pack(struct packed_git *p)
 		return NULL;
 
 	l.pack = p;
-	llist_init(&l.all_objects);
+	llist_init(&l.remaining_objects);
 
 	if (open_pack_index(p))
 		return NULL;
@@ -509,7 +509,7 @@ static struct pack_list * add_pack(struct packed_git *p)
 	base += 256 * 4 + ((p->index_version < 2) ? 4 : 8);
 	step = the_hash_algo->rawsz + ((p->index_version < 2) ? 4 : 0);
 	while (off < p->num_objects * step) {
-		llist_insert_back(l.all_objects, (const struct object_id *)(base + off));
+		llist_insert_back(l.remaining_objects, (const struct object_id *)(base + off));
 		off += step;
 	}
 	l.unique_objects = NULL;
@@ -606,7 +606,7 @@ int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 	llist_sorted_difference_inplace(all_objects, ignore);
 	pl = local_packs;
 	while (pl) {
-		llist_sorted_difference_inplace(pl->all_objects, ignore);
+		llist_sorted_difference_inplace(pl->remaining_objects, ignore);
 		pl = pl->next;
 	}
 
Signature · 1 line
2.20.1.103.ged0fc2ca7b
author
#41 lore raw

PATCH v7 6/6 pack-redundant: consistent sort method

In reply to Jiang Xin

From: Jiang Xin <[email protected]>

SZEDER reported that test case t5323 has different test result on MacOS. This is because cmp_pack_list_reverse cannot give identical result when two pack being sorted has the same size of remaining_objects.

Changes to the sorting function will make consistent test result for t5323.

The new algorithm to find redundant packs is a trade-off to save memory resources, and the result of it may be different with old one, and may be not the best result sometimes. Update t5323 for the new algorithm.

---
 builtin/pack-redundant.c  | 24 ++++++++++++++++--------
 t/t5323-pack-redundant.sh | 14 +++++++-------
 2 files changed, 23 insertions(+), 15 deletions(-)
builtin/pack-redundant.c +16 -8
index c145852b8b..0316a400ad 100644
@@ -33,6 +33,7 @@ static struct pack_list {
 	struct packed_git *pack;
 	struct llist *unique_objects;
 	struct llist *remaining_objects;
+	size_t all_objects_size;
 } *local_packs = NULL, *altodb_packs = NULL;
 
 static struct llist_item *free_nodes;
@@ -340,19 +341,25 @@ static inline off_t pack_set_bytecount(struct pack_list *pl)
 	return ret;
 }
 
-static int cmp_pack_list_reverse(const void *a, const void *b)
+static int cmp_remaining_objects(const void *a, const void *b)
 {
 	struct pack_list *pl_a = *((struct pack_list **)a);
 	struct pack_list *pl_b = *((struct pack_list **)b);
-	size_t sz_a = pl_a->remaining_objects->size;
-	size_t sz_b = pl_b->remaining_objects->size;
 
-	if (sz_a == sz_b)
-		return 0;
-	else if (sz_a < sz_b)
+	if (pl_a->remaining_objects->size == pl_b->remaining_objects->size) {
+		/* have the same remaining_objects, big pack first */
+		if (pl_a->all_objects_size == pl_b->all_objects_size)
+			return 0;
+		else if (pl_a->all_objects_size < pl_b->all_objects_size)
+			return 1;
+		else
+			return -1;
+	} else if (pl_a->remaining_objects->size < pl_b->remaining_objects->size) {
+		/* sort by remaining objects, more objects first */
 		return 1;
-	else
+	} else {
 		return -1;
+	}
 }
 
 /* Sort pack_list, greater size of remaining_objects first */
@@ -370,7 +377,7 @@ static void sort_pack_list(struct pack_list **pl)
 	for (n = 0, p = *pl; p; p = p->next)
 		ary[n++] = p;
 
-	QSORT(ary, n, cmp_pack_list_reverse);
+	QSORT(ary, n, cmp_remaining_objects);
 
 	/* link them back again */
 	for (i = 0; i < n - 1; i++)
@@ -512,6 +519,7 @@ static struct pack_list * add_pack(struct packed_git *p)
 		llist_insert_back(l.remaining_objects, (const struct object_id *)(base + off));
 		off += step;
 	}
+	l.all_objects_size = l.remaining_objects->size;
 	l.unique_objects = NULL;
 	if (p->pack_local)
 		return pack_list_insert(&local_packs, &l);
t/t5323-pack-redundant.sh +7 -7
index d176a6eb91..4469128203 100755
@@ -152,10 +152,10 @@ test_expect_success 'create pack 4, 5' '
 '
 
 cat >expected <<EOF
-P2:$P2
+P3:$P3
 EOF
 
-test_expect_failure 'one of pack-2/pack-3 is redundant' '
+test_expect_success 'one of pack-2/pack-3 is redundant' '
 	git pack-redundant --all >out &&
 	format_packfiles <out >actual &&
 	test_cmp expected actual
@@ -172,7 +172,7 @@ P4:$P4
 P6:$P6
 EOF
 
-test_expect_failure 'pack 2, 4, and 6 are redundant' '
+test_expect_success 'pack 2, 4, and 6 are redundant' '
 	git pack-redundant --all >out &&
 	format_packfiles <out >actual &&
 	test_cmp expected actual
@@ -189,7 +189,7 @@ P6:$P6
 P8:$P8
 EOF
 
-test_expect_failure 'pack-8 (subset of pack-1) is also redundant' '
+test_expect_success 'pack-8 (subset of pack-1) is also redundant' '
 	git pack-redundant --all >out &&
 	format_packfiles <out >actual &&
 	test_cmp expected actual
@@ -201,7 +201,7 @@ test_expect_success 'clean loose objects' '
 	test_must_be_empty out
 '
 
-test_expect_failure 'remove redundant packs and pass fsck' '
+test_expect_success 'remove redundant packs and pass fsck' '
 	git pack-redundant --all | xargs rm &&
 	git fsck --no-progress &&
 	git pack-redundant --all >out &&
@@ -215,7 +215,7 @@ test_expect_success 'setup shared.git' '
 	printf "../../master.git/objects" >objects/info/alternates
 '
 
-test_expect_failure 'no redundant packs without --alt-odb' '
+test_expect_success 'no redundant packs without --alt-odb' '
 	git pack-redundant --all >out &&
 	test_must_be_empty out
 '
@@ -227,7 +227,7 @@ P5:$P5
 P7:$P7
 EOF
 
-test_expect_failure 'pack-redundant --verbose: show duplicate packs in stderr' '
+test_expect_success 'pack-redundant --verbose: show duplicate packs in stderr' '
 	git pack-redundant --all --verbose >out 2>out.err &&
 	test_must_be_empty out &&
 	grep "pack$" out.err | format_packfiles >actual &&
Signature · 1 line
2.20.1.103.ged0fc2ca7b
#39 lore raw

PATCH v8 1/1 pack-redundant: delete redundant code

In reply to Jiang Xin

From: Sun Chao <[email protected]>

The objects in alt-odb are removed from all_objects twice in load_all_objects and scan_alt_odb_packs, remove it from the later function.

---
 builtin/pack-redundant.c | 1 -
 1 file changed, 1 deletion(-)
builtin/pack-redundant.c +0 -1
index 0316a400ad..29ff5e99cb 100644
@@ -492,7 +492,6 @@ static void scan_alt_odb_packs(void)
 							alt->remaining_objects);
 			local = local->next;
 		}
-		llist_sorted_difference_inplace(all_objects, alt->remaining_objects);
 		alt = alt->next;
 	}
 }
Signature · 1 line
2.20.1
#36 lore raw
In reply to Jiang Xin
Quoted 6 lines · Jiang Xin

From: Sun Chao <[email protected]>

When calling git pack-redundant --all, if there are too many local packs and too many redundant objects within them, the too deep iteration of get_permutations will exhaust all the resources, and the process of git pack-redundant will be killed.

Build each step of the series with "make DEVELOPER=YesPlease" and your compiler hopefully would notice...

CC builtin/pack-redundant.o
builtin/pack-redundant.c:347:12: error: 'is_superset' defined but not used [-Werror=unused-function]
static int is_superset(struct pack_list *pl, struct llist *list)

            ^~~~~~~~~~~
builtin/pack-redundant.c:313:21: error: 'get_permutations' defined but not used [-Werror=unused-function]
 static struct pll * get_permutations(struct pack_list *list, int n)
                     ^~~~~~~~~~~~~~~~
builtin/pack-redundant.c:293:13: error: 'pll_free' defined but not used [-Werror=unused-function]
 static void pll_free(struct pll *l)
             ^~~~~~~~

cc1: all warnings being treated as errors Makefile:2302: recipe for target 'builtin/pack-redundant.o' failed
make: *** [builtin/pack-redundant.o] Error 1

#28 lore raw

Re: [PATCH v7 1/6] t5323: test cases for git-pack-redundant

In reply to Jiang Xin
Quoted 11 lines · Jiang Xin
t/t5323-pack-redundant.sh new file +5 -0
new file mode 100755
index 0000000000..710fe9884c
@@ -0,0 +1,322 @@
+#!/bin/sh
+#
+# Copyright (c) 2018 Jiang Xin
+#
+

t/t5323-pack-redundant.sh line 15

Quoted 10 lines
t/t5323-pack-redundant.sh new file +10 -0
+test_description='git pack-redundant test'
+
+. ./test-lib.sh
+
+create_commits () {
+	parent=
+	for name in A B C D E F G H I J K L M N O P Q R
+	do
+		test_tick &&
+		T=$(git write-tree) &&

Move this outside loop, not for efficiency but for clarity. This helper function creates a single empty tree and bunch of commits that hold the same empty tree, arranged as a single strand of pearls.

By the way, I had to draw a table like this to figure out ...

T A B C D E F G H I J K L M N O P Q R
1 x x x x x x x x
2 x x x x x x x
3 x x x x x x
4 x x x x x
5 x x x x
6 x x x
7 x x
8 x

... what is going on. Perhaps something like this would help other readers near the top of the file (or in test_description)?

t/t5323-pack-redundant.sh line 130

Quoted 10 lines
t/t5323-pack-redundant.sh new file +10 -0
+format_packfiles () {
+	sed \
+		-e "s#.*/pack-\(.*\)\.idx#\1#" \
+		-e "s#.*/pack-\(.*\)\.pack#\1#" |
+	sort -u |
+	while read p
+	do
+		if test -z "$(eval echo \${P$p})"
+		then
+			echo $p

All the "expected output" below will expect P$n:${P$n} prepared by various create_pack_$n helpers we saw earlier, so an unknown packfile would be detected as a line that this emits. Is that the idea?

t/t5323-pack-redundant.sh line 142

Quoted 12 lines
t/t5323-pack-redundant.sh new file +12 -0
+		else
+			eval echo "\${P$p}"
+		fi
+	done |
+	sort
+}
+
+test_expect_success 'setup master.git' '
+	git init --bare master.git &&
+	cd master.git &&
+	create_commits
+'

Everything below will be done inside master.git? Avoid cd'ing around in random places in the test script, as a failure in any of the steps that does cd would start later tests in an unexpected place, if you can.

Quoted 10 lines
t/t5323-pack-redundant.sh new file +10 -0
+test_expect_success 'no redundant for pack 1, 2, 3' '
+	create_pack_1 && create_pack_2 && create_pack_3 &&
+	git pack-redundant --all >out &&
+	test_must_be_empty out
+'
+
+test_expect_success 'create pack 4, 5' '
+	create_pack_4 && create_pack_5
+'
+

t/t5323-pack-redundant.sh line 162

Quoted 10 lines
t/t5323-pack-redundant.sh new file +9 -0
 
+cat >expected <<EOF
+P2:$P2
+EOF
+
+test_expect_success 'one of pack-2/pack-3 is redundant' '
+	git pack-redundant --all >out &&
+	format_packfiles <out >actual &&
+	test_cmp expected actual
+'

Do the preparation of file "expect" (most of the tests compare 'expect' vs 'actual', not 'expected') _inside_ the next test that uses it. i.e.

	test_expect_success 'with 1 4 and 5, either 2 or 3 can be omitted' '
		cat >expect <<-EOF &&
		P2:$P2
		EOF
		git pack-redundant --all >out &&
		format ... >actual &&
		test_cmp expect actual
	'

Again, I needed to draw this to see if the "one of ... is redundant" in the title is a valid claim. Something like it would help future readers.

T A B C D E F G H I J K L M N O P Q R
1245 x x x x x x x x x x x x x x x x x
3 x x x x x x

T A B C D E F G H I J K L M N O P Q R
1345 x x x x x x x x x x x x x x x x x
2 x x x x x x x

I won't repeat the same for tests that appear later in this file, but they share the same issue.

t/t5323-pack-redundant.sh line 213

Quoted 3 lines
t/t5323-pack-redundant.sh new file +3 -0
+test_expect_success 'setup shared.git' '
+	cd "$TRASH_DIRECTORY" &&
+	git clone -q --mirror master.git shared.git &&

Why "-q"?

t/t5323-pack-redundant.sh line 216

Quoted 3 lines
t/t5323-pack-redundant.sh new file +3 -0
+	cd shared.git &&
+	printf "../../master.git/objects" >objects/info/alternates
+'

Why not echo? I recall designing the alternates file to be a plain text file. Is it necessary to leave the line incomplete?

t/t5323-pack-redundant.sh line 243

Quoted 3 lines
t/t5323-pack-redundant.sh new file +3 -0
+test_expect_success 'remove redundant packs by alt-odb, no packs left' '
+	git pack-redundant --all --alt-odb | xargs rm &&
+	git fsck --no-progress &&

Why "--no-progress"?

t/t5323-pack-redundant.sh line 249

Quoted 6 lines
t/t5323-pack-redundant.sh new file +6 -0
+	test_must_fail git pack-redundant --all --alt-odb >actual 2>&1 &&
+	test_cmp expected actual
+'
+
+create_commits_others () {
+	parent=$(git rev-parse HEAD)

If this fails, you'd still go ahead and enter the loop, which is not what you want.

t/t5323-pack-redundant.sh line 253

Quoted 4 lines
t/t5323-pack-redundant.sh new file +4 -0
+	for name in X Y Z
+	do
+		test_tick &&
+		T=$(git write-tree) &&

Lift this outside the loop.

Quoted 12 lines
t/t5323-pack-redundant.sh new file +12 -0
+		if test -z "$parent"
+		then
+			oid=$(echo $name | git commit-tree $T)
+		else
+			oid=$(echo $name | git commit-tree -p $parent $T)
+		fi &&
+		eval $name=$oid &&
+		parent=$oid ||
+		return 1
+	done
+	git update-ref refs/heads/master $Z
+}
author
#29 lore raw

Re: [PATCH v7 1/6] t5323: test cases for git-pack-redundant

Quoted 11 lines · Junio C Hamano <[email protected]> 于2019年2月1日周五 上午5:44写道:

t/t5323-pack-redundant.sh line 15

Quoted 6 lines
t/t5323-pack-redundant.sh new file +6 -0
+create_commits () {
+     parent=
+     for name in A B C D E F G H I J K L M N O P Q R
+     do
+             test_tick &&
+             T=$(git write-tree) &&

Move this outside loop, not for efficiency but for clarity. This helper function creates a single empty tree and bunch of commits that hold the same empty tree, arranged as a single strand of pearls.

Will rewrite as:

    create_commits () {
            parent=
            T=$(git write-tree) &&
            for name in A B C D E F G H I J K L M N O P Q R
Quoted 13 lines

By the way, I had to draw a table like this to figure out ...

T A B C D E F G H I J K L M N O P Q R
1 x x x x x x x x
2 x x x x x x x
3 x x x x x x
4 x x x x x
5 x x x x
6 x x x
7 x x
8 x

Quoted 2 lines

... what is going on. Perhaps something like this would help other readers near the top of the file (or in test_description)?

Nice chart, will edit test_description as follows:

test_description='git pack-redundant test

In order to test git-pack-redundant, we will create a number of
redundant
packs in the repository master.git. The relationship between
packs (P1-P8)
and objects (T,A-R) is show in the following chart:

| T A B C D E F G H I J K L M N O P Q R
---+--------------------------------------
P1 | x x x x x x x x
P2 | x x x x x x x
P3 | x x x x x x
P4 | x x x x x
P5 | x x x x
P6 | x x x
P7 | x x
P8 | x

Another repoisitory shared.git has unique objects (X-Z), while
share others
objects through alt-odb (of master.git). The relationship
between packs
and objects is as follows:

| T A B C D E F G H I J K L M N O P Q R X Y Z
---+----------------------------------------------
Px1| x x x x x x
Px2| x x x x x x
'

Quoted 13 lines

t/t5323-pack-redundant.sh line 130

Quoted 12 lines
t/t5323-pack-redundant.sh new file +10 -0
+format_packfiles () {
+     sed \
+             -e "s#.*/pack-\(.*\)\.idx#\1#" \
+             -e "s#.*/pack-\(.*\)\.pack#\1#" |
+     sort -u |
+     while read p
+     do
+             if test -z "$(eval echo \${P$p})"
+             then
+                     echo $p
Quoted 4 lines

All the "expected output" below will expect P$n:${P$n} prepared by various create_pack_$n helpers we saw earlier, so an unknown packfile would be detected as a line that this emits. Is that the idea?

Right. During the reroll, a typo makes an empty output, so I decide to make this change.

Quoted 9 lines
Quoted 9 lines
t/t5323-pack-redundant.sh new file +9 -0
+             else
+                     eval echo "\${P$p}"
+             fi
+     done |
+     sort
+}
+
+test_expect_success 'setup master.git' '
+     git init --bare master.git &&
Quoted 8 lines

t/t5323-pack-redundant.sh line 142

Quoted 3 lines
t/t5323-pack-redundant.sh new file +3 -0
+     cd master.git &&
+     create_commits
+'

Everything below will be done inside master.git? Avoid cd'ing around in random places in the test script, as a failure in any of the steps that does cd would start later tests in an unexpected place, if you can.

The first 10 test cases will run inside master.git, and others will run inside shared.git. Only run cd inside the two setup test cases.

Quoted 14 lines

t/t5323-pack-redundant.sh line 162

Quoted 9 lines
t/t5323-pack-redundant.sh new file +9 -0
+cat >expected <<EOF
+P2:$P2
+EOF
+
+test_expect_success 'one of pack-2/pack-3 is redundant' '
+     git pack-redundant --all >out &&
+     format_packfiles <out >actual &&
+     test_cmp expected actual
+'

Do the preparation of file "expect" (most of the tests compare 'expect' vs 'actual', not 'expected') _inside_ the next test that uses it. i.e.

Quoted 8 lines
        test_expect_success 'with 1 4 and 5, either 2 or 3 can be omitted' '
                cat >expect <<-EOF &&
                P2:$P2
                EOF
                git pack-redundant --all >out &&
                format ... >actual &&
                test_cmp expect actual
        '

Will do.

Quoted 5 lines

t/t5323-pack-redundant.sh line 213

Quoted 3 lines
t/t5323-pack-redundant.sh new file +3 -0
+test_expect_success 'setup shared.git' '
+     cd "$TRASH_DIRECTORY" &&
+     git clone -q --mirror master.git shared.git &&

Why "-q"?

To make verbose output cleaner.

Quoted 6 lines

t/t5323-pack-redundant.sh line 216

Quoted 3 lines
t/t5323-pack-redundant.sh new file +3 -0
+     cd shared.git &&
+     printf "../../master.git/objects" >objects/info/alternates
+'

Why not echo? I recall designing the alternates file to be a plain text file. Is it necessary to leave the line incomplete?

Forgot "\n", will append.

Quoted 6 lines

t/t5323-pack-redundant.sh line 243

Quoted 4 lines
t/t5323-pack-redundant.sh new file +3 -0
+test_expect_success 'remove redundant packs by alt-odb, no packs left' '
+     git pack-redundant --all --alt-odb | xargs rm &&
+     git fsck --no-progress &&

Why "--no-progress"?

To make verbose output cleaner.

Quoted 7 lines

t/t5323-pack-redundant.sh line 249

Quoted 7 lines
t/t5323-pack-redundant.sh new file +6 -0
+     test_must_fail git pack-redundant --all --alt-odb >actual 2>&1 &&
+     test_cmp expected actual
+'
+
+create_commits_others () {
+     parent=$(git rev-parse HEAD)

Will append "&&".

#30 lore raw

Re: [PATCH v7 1/6] t5323: test cases for git-pack-redundant

Quoted 6 lines · Jiang Xin
Quoted 5 lines

Junio C Hamano <[email protected]> 于2019年2月1日周五 上午5:44写道: Move this outside loop, not for efficiency but for clarity. This helper function creates a single empty tree and bunch of commits that hold the same empty tree, arranged as a single strand of pearls.

Quoted 6 lines

Will rewrite as:

    create_commits () {
            parent=
            T=$(git write-tree) &&
            for name in A B C D E F G H I J K L M N O P Q R

Don't forget the && at the end of the 'parent=' line to protect against someone later adding code above that line. So:

    create_commits () {
        parent= &&
        T=$(git write-tree) &&
        ...
Quoted 10 lines

Nice chart, will edit test_description as follows:

test_description='git pack-redundant test

In order to test git-pack-redundant, we will create a number of
redundant
packs in the repository master.git. The relationship between
packs (P1-P8)
and objects (T,A-R) is show in the following chart:

Quoted 10 lines

| T A B C D E F G H I J K L M N O P Q R
---+--------------------------------------
P1 | x x x x x x x x
P2 | x x x x x x x
P3 | x x x x x x
P4 | x x x x x
P5 | x x x x
P6 | x x x
P7 | x x
P8 | x

test_description should be a meaningful one-liner; it should not contain this other information, but this information should appear as comments in the test script.

Quoted 2 lines

Another repoisitory shared.git has unique objects (X-Z), while
share others

s/repoisitory/repository/

Quoted 6 lines
Quoted 6 lines
Quoted 5 lines
t/t5323-pack-redundant.sh new file +5 -0
+test_expect_success 'setup master.git' '
+     git init --bare master.git &&
+     cd master.git &&
+     create_commits
+'
Quoted 7 lines
Quoted 4 lines

Everything below will be done inside master.git? Avoid cd'ing around in random places in the test script, as a failure in any of the steps that does cd would start later tests in an unexpected place, if you can.

The first 10 test cases will run inside master.git, and others will run inside shared.git. Only run cd inside the two setup test cases.

That's not what Junio meant. It's okay for tests to 'cd', but each test which does so _must_ ensure that the 'cd' is undone at the end of the test, even if the test fails. The correct way to do this within each test is by using 'cd' in a subhsell, like this:

    test_expect_success 'setup master.git' '
        git init --bare master.git &&
        (
            cd master.git &&
            create_commits
        )
    '

Then, each test which needs to use "master.git" would 'cd' itself, like this:

    test_expect_success 'some test' '
        (
            cd master.git &&
            ...
        )
    '
Quoted 7 lines
Quoted 5 lines

t/t5323-pack-redundant.sh line 213

Quoted 3 lines
t/t5323-pack-redundant.sh new file +3 -0
+test_expect_success 'setup shared.git' '
+     cd "$TRASH_DIRECTORY" &&
+     git clone -q --mirror master.git shared.git &&

Why "-q"?

To make verbose output cleaner.

What Junio really meant by asking that question was that you should not do this. When something goes wrong with a test, we want as much output as possible to help diagnose the problem, so suppressing output is undesirable. To summarize, don't use -q, --no-progress, or any other such option and don't redirect to /dev/null.

author
#31 lore raw

Re: [PATCH v7 1/6] t5323: test cases for git-pack-redundant

Quoted 15 lines · Eric Sunshine <[email protected]> 于2019年2月1日周五 下午2:11写道:
Quoted 6 lines · Jiang Xin
Quoted 5 lines

Junio C Hamano <[email protected]> 于2019年2月1日周五 上午5:44写道: Move this outside loop, not for efficiency but for clarity. This helper function creates a single empty tree and bunch of commits that hold the same empty tree, arranged as a single strand of pearls.

Quoted 6 lines

Will rewrite as:

    create_commits () {
            parent=
            T=$(git write-tree) &&
            for name in A B C D E F G H I J K L M N O P Q R
Quoted 7 lines

Don't forget the && at the end of the 'parent=' line to protect against someone later adding code above that line. So:

    create_commits () {
        parent= &&
        T=$(git write-tree) &&
        ...

Will do.

Quoted 21 lines
Quoted 10 lines

Nice chart, will edit test_description as follows:

test_description='git pack-redundant test

In order to test git-pack-redundant, we will create a number of
redundant
packs in the repository master.git. The relationship between
packs (P1-P8)
and objects (T,A-R) is show in the following chart:

Quoted 10 lines

| T A B C D E F G H I J K L M N O P Q R
---+--------------------------------------
P1 | x x x x x x x x
P2 | x x x x x x x
P3 | x x x x x x
P4 | x x x x x
P5 | x x x x
P6 | x x x
P7 | x x
P8 | x

Quoted 3 lines

test_description should be a meaningful one-liner; it should not contain this other information, but this information should appear as comments in the test script.

In 't/t0000-basic.sh', there is also a very long test_description. After read 't/test-lib.sh', the only usage of test_description is showing it as help, when runing:

    sh ./t0000-basic.sh

So write a long test_description is ok, I think.

Quoted 4 lines
Quoted 2 lines

Another repoisitory shared.git has unique objects (X-Z), while
share others

s/repoisitory/repository/

Thanks, will fix.

Quoted 27 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines
t/t5323-pack-redundant.sh new file +5 -0
+test_expect_success 'setup master.git' '
+     git init --bare master.git &&
+     cd master.git &&
+     create_commits
+'
Quoted 7 lines
Quoted 4 lines

Everything below will be done inside master.git? Avoid cd'ing around in random places in the test script, as a failure in any of the steps that does cd would start later tests in an unexpected place, if you can.

The first 10 test cases will run inside master.git, and others will run inside shared.git. Only run cd inside the two setup test cases.

That's not what Junio meant. It's okay for tests to 'cd', but each test which does so _must_ ensure that the 'cd' is undone at the end of the test, even if the test fails. The correct way to do this within each test is by using 'cd' in a subhsell, like this:

    test_expect_success 'setup master.git' '
        git init --bare master.git &&
        (
            cd master.git &&
            create_commits
        )
    '
Quoted 8 lines

Then, each test which needs to use "master.git" would 'cd' itself, like this:

    test_expect_success 'some test' '
        (
            cd master.git &&
            ...
        )
    '

Nice explaination, will do.

Quoted 6 lines
Quoted 6 lines
Quoted 5 lines

t/t5323-pack-redundant.sh line 213

Quoted 3 lines
t/t5323-pack-redundant.sh new file +3 -0
+test_expect_success 'setup shared.git' '
+     cd "$TRASH_DIRECTORY" &&
+     git clone -q --mirror master.git shared.git &&

Why "-q"?

Quoted 7 lines
Quoted 1 line

To make verbose output cleaner.

What Junio really meant by asking that question was that you should not do this. When something goes wrong with a test, we want as much output as possible to help diagnose the problem, so suppressing output is undesirable. To summarize, don't use -q, --no-progress, or any other such option and don't redirect to /dev/null.

Thanks.

author
#32 lore raw

Re: [PATCH v7 1/6] t5323: test cases for git-pack-redundant

Quoted 27 lines · Jiang Xin <[email protected]> 于2019年2月1日周五 下午3:23写道:
Quoted 21 lines · Eric Sunshine <[email protected]> 于2019年2月1日周五 下午2:11写道:
Quoted 10 lines

Nice chart, will edit test_description as follows:

test_description='git pack-redundant test

In order to test git-pack-redundant, we will create a number of
redundant
packs in the repository master.git. The relationship between
packs (P1-P8)
and objects (T,A-R) is show in the following chart:

Quoted 10 lines

| T A B C D E F G H I J K L M N O P Q R
---+--------------------------------------
P1 | x x x x x x x x
P2 | x x x x x x x
P3 | x x x x x x
P4 | x x x x x
P5 | x x x x
P6 | x x x
P7 | x x
P8 | x

Quoted 3 lines

test_description should be a meaningful one-liner; it should not contain this other information, but this information should appear as comments in the test script.

Quoted 5 lines

In 't/t0000-basic.sh', there is also a very long test_description. After read 't/test-lib.sh', the only usage of test_description is showing it as help, when runing:

    sh ./t0000-basic.sh

sh ./t0000-basic.sh --help

author
#33 lore raw

Re: [PATCH v7 1/6] t5323: test cases for git-pack-redundant

In reply to Eric Sunshine
Quoted 13 lines · Eric Sunshine <[email protected]> 于2019年2月1日周五 下午2:11写道:
Quoted 7 lines

Everything below will be done inside master.git? Avoid cd'ing

Quoted 3 lines

around in random places in the test script, as a failure in any of the steps that does cd would start later tests in an unexpected place, if you can.

The first 10 test cases will run inside master.git, and others will run inside shared.git. Only run cd inside the two setup test cases.

That's not what Junio meant. It's okay for tests to 'cd', but each test which does so _must_ ensure that the 'cd' is undone at the end of the test, even if the test fails. The correct way to do this within each test is by using 'cd' in a subhsell, like this:

Quoted 7 lines
    test_expect_success 'setup master.git' '
        git init --bare master.git &&
        (
            cd master.git &&
            create_commits
        )
    '

create_commits should not run in sub-shell, or variables set are lost. I write a commit_commits_in function :

    # Usage: create_commits_in <repo> A B C ...
    # Note: DO NOT run it in sub shell, or variables are not set
    create_commits_in () {
           repo="$1" &&
            parent=$(git -C "$repo" rev-parse HEAD^{} 2>/dev/null) || parent=
           T=$(git -C "$repo" write-tree) &&
            shift &&
            while test $# -gt 0
            do
                    name=$1 &&
                    test_tick &&
                    if test -z "$parent"
                    then
                           oid=$(echo $name | git -C "$repo" commit-tree $T)
                    else
                            oid=$(echo $name | git -C "$repo"
commit-tree -p $parent $T)
                    fi &&
                    eval $name=$oid &&
                    parent=$oid &&
                    shift ||
                    return 1
            done
            git -C "$repo" update-ref refs/heads/master $oid
    }

and use it to create commits like:

create_commits_in master.git A B C D E F G ...

author
#37 lore raw
In reply to Junio C Hamano
Quoted 23 lines · Junio C Hamano <[email protected]> 于2019年2月1日周五 上午3:30写道:
Quoted 6 lines · Jiang Xin

From: Sun Chao <[email protected]>

When calling git pack-redundant --all, if there are too many local packs and too many redundant objects within them, the too deep iteration of get_permutations will exhaust all the resources, and the process of git pack-redundant will be killed.

Build each step of the series with "make DEVELOPER=YesPlease" and your compiler hopefully would notice...

CC builtin/pack-redundant.o
builtin/pack-redundant.c:347:12: error: 'is_superset' defined but not used [-Werror=unused-function]
static int is_superset(struct pack_list *pl, struct llist *list)

            ^~~~~~~~~~~
builtin/pack-redundant.c:313:21: error: 'get_permutations' defined but not used [-Werror=unused-function]
 static struct pll * get_permutations(struct pack_list *list, int n)
                     ^~~~~~~~~~~~~~~~
builtin/pack-redundant.c:293:13: error: 'pll_free' defined but not used [-Werror=unused-function]
 static void pll_free(struct pll *l)
             ^~~~~~~~
Quoted 4 lines

cc1: all warnings being treated as errors Makefile:2302: recipe for target 'builtin/pack-redundant.o' failed
make: *** [builtin/pack-redundant.o] Error 1

These unused code are deleted in another, and will be squashed to previous patch in next reroll.

author
#10 lore raw

v9 cover pack-redundant: new algorithm to find min packs

In reply to Jiang Xin

Sun Chao (my former colleague at Huawei) found a bug of git-pack-redundant. If there are too many packs and many of them overlap each other, running git pack-redundant --all will exhaust all memories and the process will be killed by kernel.

There is a script in commit log of commit 3/6, which can be used to create a repository with lots of redundant packs. Running git pack-redundant --all in it can reproduce this issue.

## Changes since reroll v7

1. Rewrite [PATCH v9 1/6] (t5323: test cases for git-pack-redundant)

* Add many tables for relationship of packs and objects.
* Change dir in subshell and fixed other issues.

2. New patch file from Sun Chao: [PATCH v9 3/6] (pack-redundant: delete redundant code)

3. Squash patches (remove unused functions) to patch 4/6 (new algorithm to find min packs).

## Range diff

1:  799e804d5e < -:  ---------- t5323: test cases for git-pack-redundant
-:  ---------- > 1:  c8dbf8cef2 t5323: test cases for git-pack-redundant
2:  520f6277fb = 2:  a6300516d7 pack-redundant: delay creation of unique_objects
-:  ---------- > 3:  fb71973df5 pack-redundant: delete redundant code
3:  ab1c2c4950 ! 4:  9963d1c49f pack-redundant: new algorithm to find min packs
    @@ -76,6 +76,113 @@
      diff --git a/builtin/pack-redundant.c b/builtin/pack-redundant.c
      --- a/builtin/pack-redundant.c
      +++ b/builtin/pack-redundant.c
    +@@
    + 	struct llist *all_objects;
    + } *local_packs = NULL, *altodb_packs = NULL;
    + 
    +-struct pll {
    +-	struct pll *next;
    +-	struct pack_list *pl;
    +-};
    +-
    + static struct llist_item *free_nodes;
    + 
    + static inline void llist_item_put(struct llist_item *item)
    +@@
    + 	return new_item;
    + }
    + 
    +-static void llist_free(struct llist *list)
    +-{
    +-	while ((list->back = list->front)) {
    +-		list->front = list->front->next;
    +-		llist_item_put(list->back);
    +-	}
    +-	free(list);
    +-}
    +-
    + static inline void llist_init(struct llist **list)
    + {
    + 	*list = xmalloc(sizeof(struct llist));
    +@@
    + 	}
    + }
    + 
    +-static void pll_free(struct pll *l)
    +-{
    +-	struct pll *old;
    +-	struct pack_list *opl;
    +-
    +-	while (l) {
    +-		old = l;
    +-		while (l->pl) {
    +-			opl = l->pl;
    +-			l->pl = opl->next;
    +-			free(opl);
    +-		}
    +-		l = l->next;
    +-		free(old);
    +-	}
    +-}
    +-
    +-/* all the permutations have to be free()d at the same time,
    +- * since they refer to each other
    +- */
    +-static struct pll * get_permutations(struct pack_list *list, int n)
    +-{
    +-	struct pll *subset, *ret = NULL, *new_pll = NULL;
    +-
    +-	if (list == NULL || pack_list_size(list) < n || n == 0)
    +-		return NULL;
    +-
    +-	if (n == 1) {
    +-		while (list) {
    +-			new_pll = xmalloc(sizeof(*new_pll));
    +-			new_pll->pl = NULL;
    +-			pack_list_insert(&new_pll->pl, list);
    +-			new_pll->next = ret;
    +-			ret = new_pll;
    +-			list = list->next;
    +-		}
    +-		return ret;
    +-	}
    +-
    +-	while (list->next) {
    +-		subset = get_permutations(list->next, n - 1);
    +-		while (subset) {
    +-			new_pll = xmalloc(sizeof(*new_pll));
    +-			new_pll->pl = subset->pl;
    +-			pack_list_insert(&new_pll->pl, list);
    +-			new_pll->next = ret;
    +-			ret = new_pll;
    +-			subset = subset->next;
    +-		}
    +-		list = list->next;
    +-	}
    +-	return ret;
    +-}
    +-
    +-static int is_superset(struct pack_list *pl, struct llist *list)
    +-{
    +-	struct llist *diff;
    +-
    +-	diff = llist_copy(list);
    +-
    +-	while (pl) {
    +-		llist_sorted_difference_inplace(diff, pl->all_objects);
    +-		if (diff->size == 0) { /* we're done */
    +-			llist_free(diff);
    +-			return 1;
    +-		}
    +-		pl = pl->next;
    +-	}
    +-	llist_free(diff);
    +-	return 0;
    +-}
    +-
    + static size_t sizeof_union(struct packed_git *p1, struct packed_git *p2)
    + {
    + 	size_t ret = 0;
     @@
      	return ret;
      }
    @@ -221,56 +328,56 @@
      --- a/t/t5323-pack-redundant.sh
      +++ b/t/t5323-pack-redundant.sh
     @@
    - P2:$P2
    - EOF
    - 
    + #     ALL | x x x x x x x x x x x x x x x x x   x
    + #
    + #############################################################################
     -test_expect_success 'one of pack-2/pack-3 is redundant' '
    -+test_expect_failure 'one of pack-2/pack-3 is redundant' '
    - 	git pack-redundant --all >out &&
    - 	format_packfiles <out >actual &&
    - 	test_cmp expected actual
    ++test_expect_failure 'one of pack-2/pack-3 is redundant (failed on Mac)' '
    + 	(
    + 		cd "$master_repo" &&
    + 		cat >expect <<-EOF &&
     @@
    - P6:$P6
    - EOF
    - 
    + #     ALL | x x x x x x x x x x x x x x x x x x x
    + #
    + #############################################################################
     -test_expect_success 'pack 2, 4, and 6 are redundant' '
    -+test_expect_failure 'pack 2, 4, and 6 are redundant' '
    - 	git pack-redundant --all >out &&
    - 	format_packfiles <out >actual &&
    - 	test_cmp expected actual
    ++test_expect_failure 'pack 2, 4, and 6 are redundant (failed on Mac)' '
    + 	(
    + 		cd "$master_repo" &&
    + 		cat >expect <<-EOF &&
     @@
    - P8:$P8
    - EOF
    - 
    + #     ALL | x x x x x x x x x x x x x x x x x x x
    + #
    + #############################################################################
     -test_expect_success 'pack-8 (subset of pack-1) is also redundant' '
    -+test_expect_failure 'pack-8 (subset of pack-1) is also redundant' '
    - 	git pack-redundant --all >out &&
    - 	format_packfiles <out >actual &&
    - 	test_cmp expected actual
    ++test_expect_failure 'pack-8 (subset of pack-1) is also redundant (failed on Mac)' '
    + 	(
    + 		cd "$master_repo" &&
    + 		cat >expect <<-EOF &&
     @@
    - 	test_must_be_empty out
    + 	)
      '
      
     -test_expect_success 'remove redundant packs and pass fsck' '
    -+test_expect_failure 'remove redundant packs and pass fsck' '
    - 	git pack-redundant --all | xargs rm &&
    - 	git fsck --no-progress &&
    - 	git pack-redundant --all >out &&
    ++test_expect_failure 'remove redundant packs and pass fsck (failed on Mac)' '
    + 	(
    + 		cd "$master_repo" &&
    + 		git pack-redundant --all | xargs rm &&
     @@
    - 	printf "../../master.git/objects" >objects/info/alternates
    + 	)
      '
      
     -test_expect_success 'no redundant packs without --alt-odb' '
    -+test_expect_failure 'no redundant packs without --alt-odb' '
    - 	git pack-redundant --all >out &&
    - 	test_must_be_empty out
    - '
    ++test_expect_failure 'no redundant packs without --alt-odb (failed on Mac)' '
    + 	(
    + 		cd "$shared_repo" &&
    + 		git pack-redundant --all >out &&
     @@
    - P7:$P7
    - EOF
    - 
    + #     ALL | x x x x x x x x x x x x x x x x x x x
    + #
    + #############################################################################
     -test_expect_success 'pack-redundant --verbose: show duplicate packs in stderr' '
    -+test_expect_failure 'pack-redundant --verbose: show duplicate packs in stderr' '
    - 	git pack-redundant --all --verbose >out 2>out.err &&
    - 	test_must_be_empty out &&
    - 	grep "pack$" out.err | format_packfiles >actual &&
    ++test_expect_failure 'pack-redundant --verbose: show duplicate packs in stderr (failed on Mac)' '
    + 	(
    + 		cd "$shared_repo" &&
    + 		cat >expect <<-EOF &&
4:  3c3a7ea40f < -:  ---------- pack-redundant: remove unused functions
5:  bc4b681f40 ! 5:  b8f80ad454 pack-redundant: rename pack_list.all_objects
    @@ -115,11 +115,7 @@
     +							alt->remaining_objects);
      			local = local->next;
      		}
    --		llist_sorted_difference_inplace(all_objects, alt->all_objects);
    -+		llist_sorted_difference_inplace(all_objects, alt->remaining_objects);
      		alt = alt->next;
    - 	}
    - }
     @@
      		return NULL;

6: 6cfba5b4b2 ! 6: 8a12ad699e pack-redundant: consistent sort method
@@ -83,60 +83,71 @@
--- a/t/t5323-pack-redundant.sh
+++ b/t/t5323-pack-redundant.sh
@@
- '
-
- cat >expected <<EOF
--P2:$P2
-+P3:$P3
- EOF
-
--test_expect_failure 'one of pack-2/pack-3 is redundant' '
+ # | T A B C D E F G H I J K L M N O P Q R
+ # ----+--------------------------------------
+ # P1 | x x x x x x x x
+-# P2* | ! ! ! ! ! ! !
+-# P3 | x x x x x x
++# P2 | x x x x x x x
++# P3* | ! ! ! ! ! !
+ # P4 | x x x x x
+ # P5 | x x x x
+ # ----+--------------------------------------
+ # ALL | x x x x x x x x x x x x x x x x x x
+ #
+ ############################################################################# +-test_expect_failure 'one of pack-2/pack-3 is redundant (failed on Mac)' ' +test_expect_success 'one of pack-2/pack-3 is redundant' '
- git pack-redundant --all >out &&
- format_packfiles <out >actual &&
- test_cmp expected actual
+ (
+ cd "$master_repo" &&
+ cat >expect <<-EOF &&
+- P2:$P2
++ P3:$P3
+ EOF
+ git pack-redundant --all >out &&
+ format_packfiles <out >actual &&
@@
- P6:$P6
- EOF
-
--test_expect_failure 'pack 2, 4, and 6 are redundant' '
+ # ALL | x x x x x x x x x x x x x x x x x x x
+ #
+ ############################################################################# +-test_expect_failure 'pack 2, 4, and 6 are redundant (failed on Mac)' ' +test_expect_success 'pack 2, 4, and 6 are redundant' '
- git pack-redundant --all >out &&
- format_packfiles <out >actual &&
- test_cmp expected actual
+ (
+ cd "$master_repo" &&
+ cat >expect <<-EOF &&
@@
- P8:$P8
- EOF
-
--test_expect_failure 'pack-8 (subset of pack-1) is also redundant' '
+ # ALL | x x x x x x x x x x x x x x x x x x x
+ #
+ ############################################################################# +-test_expect_failure 'pack-8 (subset of pack-1) is also redundant (failed on Mac)' ' +test_expect_success 'pack-8 (subset of pack-1) is also redundant' '
- git pack-redundant --all >out &&
- format_packfiles <out >actual &&
- test_cmp expected actual
+ (
+ cd "$master_repo" &&
+ cat >expect <<-EOF &&
@@
- test_must_be_empty out
+ )
'

--test_expect_failure 'remove redundant packs and pass fsck' ' +-test_expect_failure 'remove redundant packs and pass fsck (failed on Mac)' '
+test_expect_success 'remove redundant packs and pass fsck' '
- git pack-redundant --all | xargs rm &&
- git fsck --no-progress &&
- git pack-redundant --all >out &&
+ (
+ cd "$master_repo" &&
+ git pack-redundant --all | xargs rm &&
@@
- printf "../../master.git/objects" >objects/info/alternates
+ )
'

--test_expect_failure 'no redundant packs without --alt-odb' ' +-test_expect_failure 'no redundant packs without --alt-odb (failed on Mac)' '
+test_expect_success 'no redundant packs without --alt-odb' '
- git pack-redundant --all >out &&
- test_must_be_empty out
- '
+ (
+ cd "$shared_repo" &&
+ git pack-redundant --all >out &&
@@
- P7:$P7
- EOF
-
--test_expect_failure 'pack-redundant --verbose: show duplicate packs in stderr' '
+ # ALL | x x x x x x x x x x x x x x x x x x x
+ #
+ ############################################################################# +-test_expect_failure 'pack-redundant --verbose: show duplicate packs in stderr (failed on Mac)' ' +test_expect_success 'pack-redundant --verbose: show duplicate packs in stderr' '
- git pack-redundant --all --verbose >out 2>out.err &&
- test_must_be_empty out &&
- grep "pack$" out.err | format_packfiles >actual &&
+ (
+ cd "$shared_repo" &&
+ cat >expect <<-EOF &&

Jiang Xin (4):
t5323: test cases for git-pack-redundant
pack-redundant: delay creation of unique_objects
pack-redundant: rename pack_list.all_objects
pack-redundant: consistent sort method

Sun Chao (2):
pack-redundant: delete redundant code
pack-redundant: new algorithm to find min packs

 builtin/pack-redundant.c  | 232 +++++++----------
 t/t5323-pack-redundant.sh | 510 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 602 insertions(+), 140 deletions(-)
 create mode 100755 t/t5323-pack-redundant.sh
Signature · 1 line
2.20.1.103.ged0fc2ca7b
author
#11 lore raw

PATCH v9 1/6 t5323: test cases for git-pack-redundant

In reply to Jiang Xin

From: Jiang Xin <[email protected]>

Add test cases for git pack-redundant to validate new algorithm for git pack-redundant.

---
 t/t5323-pack-redundant.sh | 510 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 510 insertions(+)
 create mode 100755 t/t5323-pack-redundant.sh
t/t5323-pack-redundant.sh new file +510 -0
new file mode 100755
index 0000000000..d224ff3c50
@@ -0,0 +1,510 @@
+#!/bin/sh
+#
+# Copyright (c) 2018 Jiang Xin
+#
+
+test_description='Test git pack-redundant
+
+In order to test git-pack-redundant, we will create a number of objects and
+packs in the repository `master.git`. The relationship between packs (P1-P8)
+and objects (T, A-R) is showed in the following chart. Objects of a pack will
+be marked with letter x, while objects of redundant packs will be marked with
+exclamation point, and redundant pack itself will be marked with asterisk.
+
+        | T A B C D E F G H I J K L M N O P Q R
+    ----+--------------------------------------
+    P1  | x x x x x x x                       x
+    P2* |     ! ! ! !   ! ! !
+    P3  |             x     x x x x x
+    P4* |                     ! ! ! !     !
+    P5  |               x x           x x
+    P6* |                             ! !   !
+    P7  |                                 x x
+    P8* |   !
+    ----+--------------------------------------
+    ALL | x x x x x x x x x x x x x x x x x x x
+
+Another repository `shared.git` has unique objects (X-Z), while other objects
+(marked with letter s) are shared through alt-odb (of `master.git`). The
+relationship between packs and objects is as follows:
+
+        | T A B C D E F G H I J K L M N O P Q R   X Y Z
+    ----+----------------------------------------------
+    Px1 |   s s s                                 x x x
+    Px2 |         s s s                           x x x
+'
+
+. ./test-lib.sh
+
+master_repo=master.git
+shared_repo=shared.git
+
+# Note: DO NOT run it in a subshell, otherwise the variables will not be set
+# Usage: create_commits_in <repo> A B C ...
+create_commits_in () {
+	repo="$1" &&
+	parent=$(git -C "$repo" rev-parse HEAD^{} 2>/dev/null) || parent=
+	T=$(git -C "$repo" write-tree) &&
+	shift &&
+	while test $# -gt 0
+	do
+		name=$1 &&
+		test_tick &&
+		if test -z "$parent"
+		then
+			oid=$(echo $name | git -C "$repo" commit-tree $T)
+		else
+			oid=$(echo $name | git -C "$repo" commit-tree -p $parent $T)
+		fi &&
+		eval $name=$oid &&
+		parent=$oid &&
+		shift ||
+		return 1
+	done
+	git -C "$repo" update-ref refs/heads/master $oid
+}
+
+# Note: DO NOT run it in a subshell, otherwise the variables will not be set
+create_pack_1 () {
+	P1=$(git -C "$master_repo/objects/pack" pack-objects -q pack <<-EOF
+		$T
+		$A
+		$B
+		$C
+		$D
+		$E
+		$F
+		$R
+		EOF
+	) &&
+	eval P$P1=P1:$P1
+}
+
+create_pack_2 () {
+	P2=$(git -C "$master_repo/objects/pack" pack-objects -q pack <<-EOF
+		$B
+		$C
+		$D
+		$E
+		$G
+		$H
+		$I
+		EOF
+	) &&
+	eval P$P2=P2:$P2
+}
+
+create_pack_3 () {
+	P3=$(git -C "$master_repo/objects/pack" pack-objects -q pack <<-EOF
+		$F
+		$I
+		$J
+		$K
+		$L
+		$M
+		EOF
+	) &&
+	eval P$P3=P3:$P3
+}
+
+create_pack_4 () {
+	P4=$(git -C "$master_repo/objects/pack" pack-objects -q pack <<-EOF
+		$J
+		$K
+		$L
+		$M
+		$P
+		EOF
+	) &&
+	eval P$P4=P4:$P4
+}
+
+create_pack_5 () {
+	P5=$(git -C "$master_repo/objects/pack" pack-objects -q pack <<-EOF
+		$G
+		$H
+		$N
+		$O
+		EOF
+	) &&
+	eval P$P5=P5:$P5
+}
+
+create_pack_6 () {
+	P6=$(git -C "$master_repo/objects/pack" pack-objects -q pack <<-EOF
+		$N
+		$O
+		$Q
+		EOF
+	) &&
+	eval P$P6=P6:$P6
+}
+
+create_pack_7 () {
+	P7=$(git -C "$master_repo/objects/pack" pack-objects -q pack <<-EOF
+		$P
+		$Q
+		EOF
+	) &&
+	eval P$P7=P7:$P7
+}
+
+create_pack_8 () {
+	P8=$(git -C "$master_repo/objects/pack" pack-objects -q pack <<-EOF
+		$A
+		EOF
+	) &&
+	eval P$P8=P8:$P8
+}
+
+format_packfiles () {
+	sed \
+		-e "s#.*/pack-\(.*\)\.idx#\1#" \
+		-e "s#.*/pack-\(.*\)\.pack#\1#" |
+	sort -u |
+	while read p
+	do
+		if test -z "$(eval echo \${P$p})"
+		then
+			echo $p
+		else
+			eval echo "\${P$p}"
+		fi
+	done |
+	sort
+}
+
+test_expect_success 'setup master repo' '
+	git init --bare "$master_repo" &&
+	create_commits_in "$master_repo" A B C D E F G H I J K L M N O P Q R
+'
+
+#############################################################################
+# Chart of packs and objects for this test case
+#
+#         | T A B C D E F G H I J K L M N O P Q R
+#     ----+--------------------------------------
+#     P1  | x x x x x x x                       x
+#     P2  |     x x x x   x x x
+#     P3  |             x     x x x x x
+#     ----+--------------------------------------
+#     ALL | x x x x x x x x x x x x x x         x
+#
+#############################################################################
+test_expect_success 'no redundant for pack 1, 2, 3' '
+	create_pack_1 && create_pack_2 && create_pack_3 &&
+	(
+		cd "$master_repo" &&
+		git pack-redundant --all >out &&
+		test_must_be_empty out
+	)
+'
+
+test_expect_success 'create pack 4, 5' '
+	create_pack_4 && create_pack_5
+'
+
+#############################################################################
+# Chart of packs and objects for this test case
+#
+#         | T A B C D E F G H I J K L M N O P Q R
+#     ----+--------------------------------------
+#     P1  | x x x x x x x                       x
+#     P2* |     ! ! ! !   ! ! !
+#     P3  |             x     x x x x x
+#     P4  |                     x x x x     x
+#     P5  |               x x           x x
+#     ----+--------------------------------------
+#     ALL | x x x x x x x x x x x x x x x x x   x
+#
+#############################################################################
+test_expect_success 'one of pack-2/pack-3 is redundant' '
+	(
+		cd "$master_repo" &&
+		cat >expect <<-EOF &&
+			P2:$P2
+			EOF
+		git pack-redundant --all >out &&
+		format_packfiles <out >actual &&
+		test_cmp expect actual
+	)
+'
+
+test_expect_success 'create pack 6, 7' '
+	create_pack_6 && create_pack_7
+'
+
+#############################################################################
+# Chart of packs and objects for this test case
+#
+#         | T A B C D E F G H I J K L M N O P Q R
+#     ----+--------------------------------------
+#     P1  | x x x x x x x                       x
+#     P2* |     ! ! ! !   ! ! !
+#     P3  |             x     x x x x x
+#     P4* |                     ! ! ! !     !
+#     P5  |               x x           x x
+#     P6* |                             ! !   !
+#     P7  |                                 x x
+#     ----+--------------------------------------
+#     ALL | x x x x x x x x x x x x x x x x x x x
+#
+#############################################################################
+test_expect_success 'pack 2, 4, and 6 are redundant' '
+	(
+		cd "$master_repo" &&
+		cat >expect <<-EOF &&
+			P2:$P2
+			P4:$P4
+			P6:$P6
+			EOF
+		git pack-redundant --all >out &&
+		format_packfiles <out >actual &&
+		test_cmp expect actual
+	)
+'
+
+test_expect_success 'create pack 8' '
+	create_pack_8
+'
+
+#############################################################################
+# Chart of packs and objects for this test case
+#
+#         | T A B C D E F G H I J K L M N O P Q R
+#     ----+--------------------------------------
+#     P1  | x x x x x x x                       x
+#     P2* |     ! ! ! !   ! ! !
+#     P3  |             x     x x x x x
+#     P4* |                     ! ! ! !     !
+#     P5  |               x x           x x
+#     P6* |                             ! !   !
+#     P7  |                                 x x
+#     P8* |   !
+#     ----+--------------------------------------
+#     ALL | x x x x x x x x x x x x x x x x x x x
+#
+#############################################################################
+test_expect_success 'pack-8 (subset of pack-1) is also redundant' '
+	(
+		cd "$master_repo" &&
+		cat >expect <<-EOF &&
+			P2:$P2
+			P4:$P4
+			P6:$P6
+			P8:$P8
+			EOF
+		git pack-redundant --all >out &&
+		format_packfiles <out >actual &&
+		test_cmp expect actual
+	)
+'
+
+test_expect_success 'clean loose objects' '
+	(
+		cd "$master_repo" &&
+		git prune-packed &&
+		find objects -type f | sed -e "/objects\/pack\//d" >out &&
+		test_must_be_empty out
+	)
+'
+
+test_expect_success 'remove redundant packs and pass fsck' '
+	(
+		cd "$master_repo" &&
+		git pack-redundant --all | xargs rm &&
+		git fsck &&
+		git pack-redundant --all >out &&
+		test_must_be_empty out
+	)
+'
+
+# The following test cases will execute inside `shared.git`, instead of
+# inside `master.git`.
+test_expect_success 'setup shared.git' '
+	git clone --mirror "$master_repo" "$shared_repo" &&
+	(
+		cd "$shared_repo" &&
+		printf "../../$master_repo/objects\n" >objects/info/alternates
+	)
+'
+
+test_expect_success 'no redundant packs without --alt-odb' '
+	(
+		cd "$shared_repo" &&
+		git pack-redundant --all >out &&
+		test_must_be_empty out
+	)
+'
+
+#############################################################################
+# Chart of packs and objects for this test case
+#
+#     ================ master.git ===============
+#         | T A B C D E F G H I J K L M N O P Q R  <----------+
+#     ----+--------------------------------------             |
+#     P1  | x x x x x x x                       x             |
+#     P3  |             x     x x x x x                       |
+#     P5  |               x x           x x                   |
+#     P7  |                                 x x               |
+#     ----+--------------------------------------             |
+#     ALL | x x x x x x x x x x x x x x x x x x x             |
+#                                                             |
+#                                                             |
+#     ================ shared.git ===============             |
+#         | T A B C D E F G H I J K L M N O P Q R  <objects/info/alternates>
+#     ----+--------------------------------------
+#     P1* | s s s s s s s                       s
+#     P3* |             s     s s s s s
+#     P5* |               s s           s s
+#     P7* |                                 s s
+#     ----+--------------------------------------
+#     ALL | x x x x x x x x x x x x x x x x x x x
+#
+#############################################################################
+test_expect_success 'pack-redundant --verbose: show duplicate packs in stderr' '
+	(
+		cd "$shared_repo" &&
+		cat >expect <<-EOF &&
+			P1:$P1
+			P3:$P3
+			P5:$P5
+			P7:$P7
+			EOF
+		git pack-redundant --all --verbose >out 2>out.err &&
+		test_must_be_empty out &&
+		grep "pack$" out.err | format_packfiles >actual &&
+		test_cmp expect actual
+	)
+'
+
+test_expect_success 'remove redundant packs by alt-odb, no packs left' '
+	(
+		cd "$shared_repo" &&
+		cat >expect <<-EOF &&
+			fatal: Zero packs found!
+			EOF
+		git pack-redundant --all --alt-odb | xargs rm &&
+		git fsck &&
+		test_must_fail git pack-redundant --all --alt-odb >actual 2>&1 &&
+		test_cmp expect actual
+	)
+'
+
+# Note: DO NOT run function `create_pack_*` in sub shell, or variables are not set
+create_pack_x1_in () {
+	repo="$1" &&
+	Px1=$(git -C "$repo/objects/pack" pack-objects -q pack <<-EOF
+		$X
+		$Y
+		$Z
+		$A
+		$B
+		$C
+		EOF
+	) &&
+	eval P${Px1}=Px1:${Px1}
+}
+
+create_pack_x2_in () {
+	repo="$1" &&
+	Px2=$(git -C "$repo/objects/pack" pack-objects -q pack <<-EOF
+		$X
+		$Y
+		$Z
+		$D
+		$E
+		$F
+		EOF
+	) &&
+	eval P${Px2}=Px2:${Px2}
+}
+
+test_expect_success 'create new objects and packs in shared.git' '
+	create_commits_in "$shared_repo" X Y Z &&
+	create_pack_x1_in "$shared_repo" &&
+	create_pack_x2_in "$shared_repo"
+'
+
+test_expect_success 'no redundant without --alt-odb' '
+	(
+		cd "$shared_repo" &&
+		git pack-redundant --all >out &&
+		test_must_be_empty out
+	)
+'
+
+#############################################################################
+# Chart of packs and objects for this test case
+#
+#     ================ master.git ===============
+#         | T A B C D E F G H I J K L M N O P Q R  <----------------+
+#     ----+--------------------------------------                   |
+#     P1  | x x x x x x x                       x                   |
+#     P3  |             x     x x x x x                             |
+#     P5  |               x x           x x                         |
+#     P7  |                                 x x                     |
+#     ----+--------------------------------------                   |
+#     ALL | x x x x x x x x x x x x x x x x x x x                   |
+#                                                                   |
+#                                                                   |
+#     ================ shared.git =======================           |
+#         | T A B C D E F G H I J K L M N O P Q R   X Y Z <objects/info/alternates>
+#     ----+----------------------------------------------
+#     Px1 |   s s s                                 x x x
+#     Px2*|         s s s                           ! ! !
+#     ----+----------------------------------------------
+#     ALL | s s s s s s s s s s s s s s s s s s s   x x x
+#
+#############################################################################
+test_expect_success 'one pack is redundant' '
+	(
+		cd "$shared_repo" &&
+		git pack-redundant --all --alt-odb >out &&
+		format_packfiles <out >actual &&
+		test_line_count = 1 actual
+	)
+'
+
+#############################################################################
+# Chart of packs and objects for this test case
+#
+#     ================ master.git ===============
+#         | T A B C D E F G H I J K L M N O P Q R  <----------------+
+#     ----+--------------------------------------                   |
+#     P1  | x x x x x x x                       x                   |
+#     P3  |             x     x x x x x                             |
+#     P5  |               x x           x x                         |
+#     P7  |                                 x x                     |
+#     ----+--------------------------------------                   |
+#     ALL | x x x x x x x x x x x x x x x x x x x                   |
+#                                                                   |
+#                                                                   |
+#     ================ shared.git =======================           |
+#         | T A B C D E F G H I J K L M N O P Q R   X Y Z <objects/info/alternates>
+#     ----+----------------------------------------------
+#     Px1*|   s s s                                 i i i
+#     Px2*|         s s s                           i i i
+#     ----+----------------------------------------------
+#     ALL | s s s s s s s s s s s s s s s s s s s   i i i
+#                                                  (ignored objects, marked with i)
+#
+#############################################################################
+test_expect_success 'set ignore objects and all two packs are redundant' '
+	(
+		cd "$shared_repo" &&
+		cat >expect <<-EOF &&
+			Px1:$Px1
+			Px2:$Px2
+			EOF
+		git pack-redundant --all --alt-odb >out <<-EOF &&
+			$X
+			$Y
+			$Z
+			EOF
+		format_packfiles <out >actual &&
+		test_cmp expect actual
+	)
+'
+
+test_done
Signature · 1 line
2.20.1.103.ged0fc2ca7b
author
#22 lore raw

PATCH v9 2/6 pack-redundant: delay creation of unique_objects

In reply to Jiang Xin

From: Jiang Xin <[email protected]>

Instead of initializing unique_objects in add_pack(), copy from all_objects in cmp_two_packs(), when unwanted objects are removed from all_objects.

This will save memory (no allocate memory for alt-odb packs), and run llist_sorted_difference_inplace() only once when removing ignored objects and removing objects in alt-odb in scan_alt_odb_packs().

---
 builtin/pack-redundant.c | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)
builtin/pack-redundant.c +10 -5
index cf9a9aabd4..f7dab0ec60 100644
@@ -254,6 +254,11 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
 	struct llist_item *p1_hint = NULL, *p2_hint = NULL;
 	const unsigned int hashsz = the_hash_algo->rawsz;
 
+	if (!p1->unique_objects)
+		p1->unique_objects = llist_copy(p1->all_objects);
+	if (!p2->unique_objects)
+		p2->unique_objects = llist_copy(p2->all_objects);
+
 	p1_base = p1->pack->index_data;
 	p2_base = p2->pack->index_data;
 	p1_base += 256 * 4 + ((p1->pack->index_version < 2) ? 4 : 8);
@@ -536,7 +541,7 @@ static void scan_alt_odb_packs(void)
 	while (alt) {
 		local = local_packs;
 		while (local) {
-			llist_sorted_difference_inplace(local->unique_objects,
+			llist_sorted_difference_inplace(local->all_objects,
 							alt->all_objects);
 			local = local->next;
 		}
@@ -567,8 +572,7 @@ static struct pack_list * add_pack(struct packed_git *p)
 		llist_insert_back(l.all_objects, (const struct object_id *)(base + off));
 		off += step;
 	}
-	/* this list will be pruned in cmp_two_packs later */
-	l.unique_objects = llist_copy(l.all_objects);
+	l.unique_objects = NULL;
 	if (p->pack_local)
 		return pack_list_insert(&local_packs, &l);
 	else
@@ -646,7 +650,6 @@ int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 
 	load_all_objects();
 
-	cmp_local_packs();
 	if (alt_odb)
 		scan_alt_odb_packs();
 
@@ -663,10 +666,12 @@ int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 	llist_sorted_difference_inplace(all_objects, ignore);
 	pl = local_packs;
 	while (pl) {
-		llist_sorted_difference_inplace(pl->unique_objects, ignore);
+		llist_sorted_difference_inplace(pl->all_objects, ignore);
 		pl = pl->next;
 	}
 
+	cmp_local_packs();
+
 	minimize(&min);
 
 	if (verbose) {
Signature · 1 line
2.20.1.103.ged0fc2ca7b
author
#23 lore raw

PATCH v9 3/6 pack-redundant: delete redundant code

In reply to Jiang Xin

From: Sun Chao <[email protected]>

The objects in alt-odb are removed from all_objects twice in load_all_objects and scan_alt_odb_packs, remove it from the later function.

---
 builtin/pack-redundant.c | 1 -
 1 file changed, 1 deletion(-)
builtin/pack-redundant.c +0 -1
index f7dab0ec60..4a06f057dd 100644
@@ -545,7 +545,6 @@ static void scan_alt_odb_packs(void)
 							alt->all_objects);
 			local = local->next;
 		}
-		llist_sorted_difference_inplace(all_objects, alt->all_objects);
 		alt = alt->next;
 	}
 }
Signature · 1 line
2.20.1.103.ged0fc2ca7b
author
#24 lore raw

PATCH v9 4/6 pack-redundant: new algorithm to find min packs

In reply to Jiang Xin

From: Sun Chao <[email protected]>

When calling git pack-redundant --all, if there are too many local packs and too many redundant objects within them, the too deep iteration of get_permutations will exhaust all the resources, and the process of git pack-redundant will be killed.

The following script could create a repository with too many redundant packs, and running git pack-redundant --all in the test.git repo will die soon.

    #!/bin/sh

    repo="$(pwd)/test.git"
    work="$(pwd)/test"
    i=1
    max=199

    if test -d "$repo" || test -d "$work"; then
    	echo >&2 "ERROR: '$repo' or '$work' already exist"
    	exit 1
    fi

    git init -q --bare "$repo"
    git --git-dir="$repo" config gc.auto 0
    git --git-dir="$repo" config transfer.unpackLimit 0
    git clone -q "$repo" "$work" 2>/dev/null

    while :; do
        cd "$work"
        echo "loop $i: $(date +%s)" >$i
        git add $i
        git commit -q -sm "loop $i"
        git push -q origin HEAD:master
        printf "\rCreate pack %4d/%d\t" $i $max
        if test $i -ge $max; then break; fi

        cd "$repo"
        git repack -q
        if test $(($i % 2)) -eq 0; then
            git repack -aq
            pack=$(ls -t $repo/objects/pack/*.pack | head -1)
            touch "${pack%.pack}.keep"
        fi
        i=$((i+1))
    done
    printf "\ndone\n"

To get the min unique pack list, we can replace the iteration in minimize function with a new algorithm, and this could solve this issue:

1. Get the unique and non_uniqe packs, add the unique packs to the min list.

2. Remove the objects of unique packs from non_unique packs, then each object left in the non_unique packs will have at least two copies.

3. Sort the non_unique packs by the objects' size, more objects first, and add the first non_unique pack to min list.

4. Drop the duplicated objects from other packs in the ordered non_unique pack list, and repeat step 3.

Some test cases will fail on Mac OS X. Mark them and will resolve in later commit.

Original PR and discussions: https://github.com/jiangxin/git/pull/25

---
 builtin/pack-redundant.c  | 194 +++++++++++++-------------------------
 t/t5323-pack-redundant.sh |  12 +--
 2 files changed, 73 insertions(+), 133 deletions(-)
builtin/pack-redundant.c +67 -127
index 4a06f057dd..d6d9a66e46 100644
@@ -35,11 +35,6 @@ static struct pack_list {
 	struct llist *all_objects;
 } *local_packs = NULL, *altodb_packs = NULL;
 
-struct pll {
-	struct pll *next;
-	struct pack_list *pl;
-};
-
 static struct llist_item *free_nodes;
 
 static inline void llist_item_put(struct llist_item *item)
@@ -63,15 +58,6 @@ static inline struct llist_item *llist_item_get(void)
 	return new_item;
 }
 
-static void llist_free(struct llist *list)
-{
-	while ((list->back = list->front)) {
-		list->front = list->front->next;
-		llist_item_put(list->back);
-	}
-	free(list);
-}
-
 static inline void llist_init(struct llist **list)
 {
 	*list = xmalloc(sizeof(struct llist));
@@ -290,78 +276,6 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
 	}
 }
 
-static void pll_free(struct pll *l)
-{
-	struct pll *old;
-	struct pack_list *opl;
-
-	while (l) {
-		old = l;
-		while (l->pl) {
-			opl = l->pl;
-			l->pl = opl->next;
-			free(opl);
-		}
-		l = l->next;
-		free(old);
-	}
-}
-
-/* all the permutations have to be free()d at the same time,
- * since they refer to each other
- */
-static struct pll * get_permutations(struct pack_list *list, int n)
-{
-	struct pll *subset, *ret = NULL, *new_pll = NULL;
-
-	if (list == NULL || pack_list_size(list) < n || n == 0)
-		return NULL;
-
-	if (n == 1) {
-		while (list) {
-			new_pll = xmalloc(sizeof(*new_pll));
-			new_pll->pl = NULL;
-			pack_list_insert(&new_pll->pl, list);
-			new_pll->next = ret;
-			ret = new_pll;
-			list = list->next;
-		}
-		return ret;
-	}
-
-	while (list->next) {
-		subset = get_permutations(list->next, n - 1);
-		while (subset) {
-			new_pll = xmalloc(sizeof(*new_pll));
-			new_pll->pl = subset->pl;
-			pack_list_insert(&new_pll->pl, list);
-			new_pll->next = ret;
-			ret = new_pll;
-			subset = subset->next;
-		}
-		list = list->next;
-	}
-	return ret;
-}
-
-static int is_superset(struct pack_list *pl, struct llist *list)
-{
-	struct llist *diff;
-
-	diff = llist_copy(list);
-
-	while (pl) {
-		llist_sorted_difference_inplace(diff, pl->all_objects);
-		if (diff->size == 0) { /* we're done */
-			llist_free(diff);
-			return 1;
-		}
-		pl = pl->next;
-	}
-	llist_free(diff);
-	return 0;
-}
-
 static size_t sizeof_union(struct packed_git *p1, struct packed_git *p2)
 {
 	size_t ret = 0;
@@ -426,14 +340,52 @@ static inline off_t pack_set_bytecount(struct pack_list *pl)
 	return ret;
 }
 
+static int cmp_pack_list_reverse(const void *a, const void *b)
+{
+	struct pack_list *pl_a = *((struct pack_list **)a);
+	struct pack_list *pl_b = *((struct pack_list **)b);
+	size_t sz_a = pl_a->all_objects->size;
+	size_t sz_b = pl_b->all_objects->size;
+
+	if (sz_a == sz_b)
+		return 0;
+	else if (sz_a < sz_b)
+		return 1;
+	else
+		return -1;
+}
+
+/* Sort pack_list, greater size of all_objects first */
+static void sort_pack_list(struct pack_list **pl)
+{
+	struct pack_list **ary, *p;
+	int i;
+	size_t n = pack_list_size(*pl);
+
+	if (n < 2)
+		return;
+
+	/* prepare an array of packed_list for easier sorting */
+	ary = xcalloc(n, sizeof(struct pack_list *));
+	for (n = 0, p = *pl; p; p = p->next)
+		ary[n++] = p;
+
+	QSORT(ary, n, cmp_pack_list_reverse);
+
+	/* link them back again */
+	for (i = 0; i < n - 1; i++)
+		ary[i]->next = ary[i + 1];
+	ary[n - 1]->next = NULL;
+	*pl = ary[0];
+
+	free(ary);
+}
+
+
 static void minimize(struct pack_list **min)
 {
-	struct pack_list *pl, *unique = NULL,
-		*non_unique = NULL, *min_perm = NULL;
-	struct pll *perm, *perm_all, *perm_ok = NULL, *new_perm;
-	struct llist *missing;
-	off_t min_perm_size = 0, perm_size;
-	int n;
+	struct pack_list *pl, *unique = NULL, *non_unique = NULL;
+	struct llist *missing, *unique_pack_objects;
 
 	pl = local_packs;
 	while (pl) {
@@ -451,49 +403,37 @@ static void minimize(struct pack_list **min)
 		pl = pl->next;
 	}
 
+	*min = unique;
+
 	/* return if there are no objects missing from the unique set */
 	if (missing->size == 0) {
-		*min = unique;
 		free(missing);
 		return;
 	}
 
-	/* find the permutations which contain all missing objects */
-	for (n = 1; n <= pack_list_size(non_unique) && !perm_ok; n++) {
-		perm_all = perm = get_permutations(non_unique, n);
-		while (perm) {
-			if (is_superset(perm->pl, missing)) {
-				new_perm = xmalloc(sizeof(struct pll));
-				memcpy(new_perm, perm, sizeof(struct pll));
-				new_perm->next = perm_ok;
-				perm_ok = new_perm;
-			}
-			perm = perm->next;
-		}
-		if (perm_ok)
-			break;
-		pll_free(perm_all);
-	}
-	if (perm_ok == NULL)
-		die("Internal error: No complete sets found!");
-
-	/* find the permutation with the smallest size */
-	perm = perm_ok;
-	while (perm) {
-		perm_size = pack_set_bytecount(perm->pl);
-		if (!min_perm_size || min_perm_size > perm_size) {
-			min_perm_size = perm_size;
-			min_perm = perm->pl;
-		}
-		perm = perm->next;
-	}
-	*min = min_perm;
-	/* add the unique packs to the list */
-	pl = unique;
+	unique_pack_objects = llist_copy(all_objects);
+	llist_sorted_difference_inplace(unique_pack_objects, missing);
+
+	/* remove unique pack objects from the non_unique packs */
+	pl = non_unique;
 	while (pl) {
-		pack_list_insert(min, pl);
+		llist_sorted_difference_inplace(pl->all_objects, unique_pack_objects);
 		pl = pl->next;
 	}
+
+	while (non_unique) {
+		/* sort the non_unique packs, greater size of all_objects first */
+		sort_pack_list(&non_unique);
+		if (non_unique->all_objects->size == 0)
+			break;
+
+		pack_list_insert(min, non_unique);
+
+		for (pl = non_unique->next; pl && pl->all_objects->size > 0;  pl = pl->next)
+			llist_sorted_difference_inplace(pl->all_objects, non_unique->all_objects);
+
+		non_unique = non_unique->next;
+	}
 }
 
 static void load_all_objects(void)
@@ -606,7 +546,7 @@ static void load_all(void)
 int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 {
 	int i;
-	struct pack_list *min, *red, *pl;
+	struct pack_list *min = NULL, *red, *pl;
 	struct llist *ignore;
 	struct object_id *oid;
 	char buf[GIT_MAX_HEXSZ + 2]; /* hex hash + \n + \0 */
t/t5323-pack-redundant.sh +6 -6
index d224ff3c50..897acaa365 100755
@@ -218,7 +218,7 @@ test_expect_success 'create pack 4, 5' '
 #     ALL | x x x x x x x x x x x x x x x x x   x
 #
 #############################################################################
-test_expect_success 'one of pack-2/pack-3 is redundant' '
+test_expect_failure 'one of pack-2/pack-3 is redundant (failed on Mac)' '
 	(
 		cd "$master_repo" &&
 		cat >expect <<-EOF &&
@@ -250,7 +250,7 @@ test_expect_success 'create pack 6, 7' '
 #     ALL | x x x x x x x x x x x x x x x x x x x
 #
 #############################################################################
-test_expect_success 'pack 2, 4, and 6 are redundant' '
+test_expect_failure 'pack 2, 4, and 6 are redundant (failed on Mac)' '
 	(
 		cd "$master_repo" &&
 		cat >expect <<-EOF &&
@@ -285,7 +285,7 @@ test_expect_success 'create pack 8' '
 #     ALL | x x x x x x x x x x x x x x x x x x x
 #
 #############################################################################
-test_expect_success 'pack-8 (subset of pack-1) is also redundant' '
+test_expect_failure 'pack-8 (subset of pack-1) is also redundant (failed on Mac)' '
 	(
 		cd "$master_repo" &&
 		cat >expect <<-EOF &&
@@ -309,7 +309,7 @@ test_expect_success 'clean loose objects' '
 	)
 '
 
-test_expect_success 'remove redundant packs and pass fsck' '
+test_expect_failure 'remove redundant packs and pass fsck (failed on Mac)' '
 	(
 		cd "$master_repo" &&
 		git pack-redundant --all | xargs rm &&
@@ -329,7 +329,7 @@ test_expect_success 'setup shared.git' '
 	)
 '
 
-test_expect_success 'no redundant packs without --alt-odb' '
+test_expect_failure 'no redundant packs without --alt-odb (failed on Mac)' '
 	(
 		cd "$shared_repo" &&
 		git pack-redundant --all >out &&
@@ -362,7 +362,7 @@ test_expect_success 'no redundant packs without --alt-odb' '
 #     ALL | x x x x x x x x x x x x x x x x x x x
 #
 #############################################################################
-test_expect_success 'pack-redundant --verbose: show duplicate packs in stderr' '
+test_expect_failure 'pack-redundant --verbose: show duplicate packs in stderr (failed on Mac)' '
 	(
 		cd "$shared_repo" &&
 		cat >expect <<-EOF &&
Signature · 1 line
2.20.1.103.ged0fc2ca7b
author
#25 lore raw

PATCH v9 5/6 pack-redundant: rename pack_list.all_objects

In reply to Jiang Xin

From: Jiang Xin <[email protected]>

New algorithm uses pack_list.all_objects to track remaining objects, so rename it to pack_list.remaining_objects.

---
 builtin/pack-redundant.c | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)
builtin/pack-redundant.c +19 -19
index d6d9a66e46..15cdf233c4 100644
@@ -32,7 +32,7 @@ static struct pack_list {
 	struct pack_list *next;
 	struct packed_git *pack;
 	struct llist *unique_objects;
-	struct llist *all_objects;
+	struct llist *remaining_objects;
 } *local_packs = NULL, *altodb_packs = NULL;
 
 static struct llist_item *free_nodes;
@@ -241,9 +241,9 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
 	const unsigned int hashsz = the_hash_algo->rawsz;
 
 	if (!p1->unique_objects)
-		p1->unique_objects = llist_copy(p1->all_objects);
+		p1->unique_objects = llist_copy(p1->remaining_objects);
 	if (!p2->unique_objects)
-		p2->unique_objects = llist_copy(p2->all_objects);
+		p2->unique_objects = llist_copy(p2->remaining_objects);
 
 	p1_base = p1->pack->index_data;
 	p2_base = p2->pack->index_data;
@@ -344,8 +344,8 @@ static int cmp_pack_list_reverse(const void *a, const void *b)
 {
 	struct pack_list *pl_a = *((struct pack_list **)a);
 	struct pack_list *pl_b = *((struct pack_list **)b);
-	size_t sz_a = pl_a->all_objects->size;
-	size_t sz_b = pl_b->all_objects->size;
+	size_t sz_a = pl_a->remaining_objects->size;
+	size_t sz_b = pl_b->remaining_objects->size;
 
 	if (sz_a == sz_b)
 		return 0;
@@ -355,7 +355,7 @@ static int cmp_pack_list_reverse(const void *a, const void *b)
 		return -1;
 }
 
-/* Sort pack_list, greater size of all_objects first */
+/* Sort pack_list, greater size of remaining_objects first */
 static void sort_pack_list(struct pack_list **pl)
 {
 	struct pack_list **ary, *p;
@@ -399,7 +399,7 @@ static void minimize(struct pack_list **min)
 	missing = llist_copy(all_objects);
 	pl = unique;
 	while (pl) {
-		llist_sorted_difference_inplace(missing, pl->all_objects);
+		llist_sorted_difference_inplace(missing, pl->remaining_objects);
 		pl = pl->next;
 	}
 
@@ -417,20 +417,20 @@ static void minimize(struct pack_list **min)
 	/* remove unique pack objects from the non_unique packs */
 	pl = non_unique;
 	while (pl) {
-		llist_sorted_difference_inplace(pl->all_objects, unique_pack_objects);
+		llist_sorted_difference_inplace(pl->remaining_objects, unique_pack_objects);
 		pl = pl->next;
 	}
 
 	while (non_unique) {
-		/* sort the non_unique packs, greater size of all_objects first */
+		/* sort the non_unique packs, greater size of remaining_objects first */
 		sort_pack_list(&non_unique);
-		if (non_unique->all_objects->size == 0)
+		if (non_unique->remaining_objects->size == 0)
 			break;
 
 		pack_list_insert(min, non_unique);
 
-		for (pl = non_unique->next; pl && pl->all_objects->size > 0;  pl = pl->next)
-			llist_sorted_difference_inplace(pl->all_objects, non_unique->all_objects);
+		for (pl = non_unique->next; pl && pl->remaining_objects->size > 0;  pl = pl->next)
+			llist_sorted_difference_inplace(pl->remaining_objects, non_unique->remaining_objects);
 
 		non_unique = non_unique->next;
 	}
@@ -445,7 +445,7 @@ static void load_all_objects(void)
 
 	while (pl) {
 		hint = NULL;
-		l = pl->all_objects->front;
+		l = pl->remaining_objects->front;
 		while (l) {
 			hint = llist_insert_sorted_unique(all_objects,
 							  l->oid, hint);
@@ -456,7 +456,7 @@ static void load_all_objects(void)
 	/* remove objects present in remote packs */
 	pl = altodb_packs;
 	while (pl) {
-		llist_sorted_difference_inplace(all_objects, pl->all_objects);
+		llist_sorted_difference_inplace(all_objects, pl->remaining_objects);
 		pl = pl->next;
 	}
 }
@@ -481,8 +481,8 @@ static void scan_alt_odb_packs(void)
 	while (alt) {
 		local = local_packs;
 		while (local) {
-			llist_sorted_difference_inplace(local->all_objects,
-							alt->all_objects);
+			llist_sorted_difference_inplace(local->remaining_objects,
+							alt->remaining_objects);
 			local = local->next;
 		}
 		alt = alt->next;
@@ -499,7 +499,7 @@ static struct pack_list * add_pack(struct packed_git *p)
 		return NULL;
 
 	l.pack = p;
-	llist_init(&l.all_objects);
+	llist_init(&l.remaining_objects);
 
 	if (open_pack_index(p))
 		return NULL;
@@ -508,7 +508,7 @@ static struct pack_list * add_pack(struct packed_git *p)
 	base += 256 * 4 + ((p->index_version < 2) ? 4 : 8);
 	step = the_hash_algo->rawsz + ((p->index_version < 2) ? 4 : 0);
 	while (off < p->num_objects * step) {
-		llist_insert_back(l.all_objects, (const struct object_id *)(base + off));
+		llist_insert_back(l.remaining_objects, (const struct object_id *)(base + off));
 		off += step;
 	}
 	l.unique_objects = NULL;
@@ -605,7 +605,7 @@ int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 	llist_sorted_difference_inplace(all_objects, ignore);
 	pl = local_packs;
 	while (pl) {
-		llist_sorted_difference_inplace(pl->all_objects, ignore);
+		llist_sorted_difference_inplace(pl->remaining_objects, ignore);
 		pl = pl->next;
 	}
 
Signature · 1 line
2.20.1.103.ged0fc2ca7b
author
#26 lore raw

PATCH v9 6/6 pack-redundant: consistent sort method

In reply to Jiang Xin

From: Jiang Xin <[email protected]>

SZEDER reported that test case t5323 has different test result on MacOS. This is because cmp_pack_list_reverse cannot give identical result when two pack being sorted has the same size of remaining_objects.

Changes to the sorting function will make consistent test result for t5323.

The new algorithm to find redundant packs is a trade-off to save memory resources, and the result of it may be different with old one, and may be not the best result sometimes. Update t5323 for the new algorithm.

---
 builtin/pack-redundant.c  | 24 ++++++++++++++++--------
 t/t5323-pack-redundant.sh | 18 +++++++++---------
 2 files changed, 25 insertions(+), 17 deletions(-)
builtin/pack-redundant.c +16 -8
index 15cdf233c4..29ff5e99cb 100644
@@ -33,6 +33,7 @@ static struct pack_list {
 	struct packed_git *pack;
 	struct llist *unique_objects;
 	struct llist *remaining_objects;
+	size_t all_objects_size;
 } *local_packs = NULL, *altodb_packs = NULL;
 
 static struct llist_item *free_nodes;
@@ -340,19 +341,25 @@ static inline off_t pack_set_bytecount(struct pack_list *pl)
 	return ret;
 }
 
-static int cmp_pack_list_reverse(const void *a, const void *b)
+static int cmp_remaining_objects(const void *a, const void *b)
 {
 	struct pack_list *pl_a = *((struct pack_list **)a);
 	struct pack_list *pl_b = *((struct pack_list **)b);
-	size_t sz_a = pl_a->remaining_objects->size;
-	size_t sz_b = pl_b->remaining_objects->size;
 
-	if (sz_a == sz_b)
-		return 0;
-	else if (sz_a < sz_b)
+	if (pl_a->remaining_objects->size == pl_b->remaining_objects->size) {
+		/* have the same remaining_objects, big pack first */
+		if (pl_a->all_objects_size == pl_b->all_objects_size)
+			return 0;
+		else if (pl_a->all_objects_size < pl_b->all_objects_size)
+			return 1;
+		else
+			return -1;
+	} else if (pl_a->remaining_objects->size < pl_b->remaining_objects->size) {
+		/* sort by remaining objects, more objects first */
 		return 1;
-	else
+	} else {
 		return -1;
+	}
 }
 
 /* Sort pack_list, greater size of remaining_objects first */
@@ -370,7 +377,7 @@ static void sort_pack_list(struct pack_list **pl)
 	for (n = 0, p = *pl; p; p = p->next)
 		ary[n++] = p;
 
-	QSORT(ary, n, cmp_pack_list_reverse);
+	QSORT(ary, n, cmp_remaining_objects);
 
 	/* link them back again */
 	for (i = 0; i < n - 1; i++)
@@ -511,6 +518,7 @@ static struct pack_list * add_pack(struct packed_git *p)
 		llist_insert_back(l.remaining_objects, (const struct object_id *)(base + off));
 		off += step;
 	}
+	l.all_objects_size = l.remaining_objects->size;
 	l.unique_objects = NULL;
 	if (p->pack_local)
 		return pack_list_insert(&local_packs, &l);
t/t5323-pack-redundant.sh +9 -9
index 897acaa365..e642240d71 100755
@@ -210,19 +210,19 @@ test_expect_success 'create pack 4, 5' '
 #         | T A B C D E F G H I J K L M N O P Q R
 #     ----+--------------------------------------
 #     P1  | x x x x x x x                       x
-#     P2* |     ! ! ! !   ! ! !
-#     P3  |             x     x x x x x
+#     P2  |     x x x x   x x x
+#     P3* |             !     ! ! ! ! !
 #     P4  |                     x x x x     x
 #     P5  |               x x           x x
 #     ----+--------------------------------------
 #     ALL | x x x x x x x x x x x x x x x x x   x
 #
 #############################################################################
-test_expect_failure 'one of pack-2/pack-3 is redundant (failed on Mac)' '
+test_expect_success 'one of pack-2/pack-3 is redundant' '
 	(
 		cd "$master_repo" &&
 		cat >expect <<-EOF &&
-			P2:$P2
+			P3:$P3
 			EOF
 		git pack-redundant --all >out &&
 		format_packfiles <out >actual &&
@@ -250,7 +250,7 @@ test_expect_success 'create pack 6, 7' '
 #     ALL | x x x x x x x x x x x x x x x x x x x
 #
 #############################################################################
-test_expect_failure 'pack 2, 4, and 6 are redundant (failed on Mac)' '
+test_expect_success 'pack 2, 4, and 6 are redundant' '
 	(
 		cd "$master_repo" &&
 		cat >expect <<-EOF &&
@@ -285,7 +285,7 @@ test_expect_success 'create pack 8' '
 #     ALL | x x x x x x x x x x x x x x x x x x x
 #
 #############################################################################
-test_expect_failure 'pack-8 (subset of pack-1) is also redundant (failed on Mac)' '
+test_expect_success 'pack-8 (subset of pack-1) is also redundant' '
 	(
 		cd "$master_repo" &&
 		cat >expect <<-EOF &&
@@ -309,7 +309,7 @@ test_expect_success 'clean loose objects' '
 	)
 '
 
-test_expect_failure 'remove redundant packs and pass fsck (failed on Mac)' '
+test_expect_success 'remove redundant packs and pass fsck' '
 	(
 		cd "$master_repo" &&
 		git pack-redundant --all | xargs rm &&
@@ -329,7 +329,7 @@ test_expect_success 'setup shared.git' '
 	)
 '
 
-test_expect_failure 'no redundant packs without --alt-odb (failed on Mac)' '
+test_expect_success 'no redundant packs without --alt-odb' '
 	(
 		cd "$shared_repo" &&
 		git pack-redundant --all >out &&
@@ -362,7 +362,7 @@ test_expect_failure 'no redundant packs without --alt-odb (failed on Mac)' '
 #     ALL | x x x x x x x x x x x x x x x x x x x
 #
 #############################################################################
-test_expect_failure 'pack-redundant --verbose: show duplicate packs in stderr (failed on Mac)' '
+test_expect_success 'pack-redundant --verbose: show duplicate packs in stderr' '
 	(
 		cd "$shared_repo" &&
 		cat >expect <<-EOF &&
Signature · 1 line
2.20.1.103.ged0fc2ca7b
#12 lore raw

Re: [PATCH v9 1/6] t5323: test cases for git-pack-redundant

In reply to Jiang Xin

t/t5323-pack-redundant.sh line 1

Quoted 8 lines · Jiang Xin

Add test cases for git pack-redundant to validate new algorithm for git pack-redundant.

---

t/t5323-pack-redundant.sh +1 -0
@@ -0,0 +1,510 @@
+# Note: DO NOT run it in a subshell, otherwise the variables will not be set

Which variables won't be set? It's not clear what this restriction is about.

t/t5323-pack-redundant.sh line 46

Quoted 4 lines
t/t5323-pack-redundant.sh +4 -0
+# Usage: create_commits_in <repo> A B C ...
+create_commits_in () {
+       repo="$1" &&
+       parent=$(git -C "$repo" rev-parse HEAD^{} 2>/dev/null) || parent=

Broken &&-chain. Instead, perhaps:

    if ! parent=$(git -C "$repo" rev-parse HEAD^{} 2>/dev/null)
    then
        parent=
    fi &&

or something simpler.

Quoted 7 lines
t/t5323-pack-redundant.sh +7 -0
+       T=$(git -C "$repo" write-tree) &&
+       shift &&
+       while test $# -gt 0
+       do
+               name=$1 &&
+               test_tick &&
+               if test -z "$parent"

t/t5323-pack-redundant.sh line 63

Quoted 10 lines
t/t5323-pack-redundant.sh +10 -0
+               then
+                       oid=$(echo $name | git -C "$repo" commit-tree $T)
+               else
+                       oid=$(echo $name | git -C "$repo" commit-tree -p $parent $T)
+               fi &&
+               eval $name=$oid &&
+               parent=$oid &&
+               shift ||
+               return 1
+       done

Broken &&-chain. Use:

    done &&

t/t5323-pack-redundant.sh line 69

Quoted 6 lines
t/t5323-pack-redundant.sh +6 -0
+       git -C "$repo" update-ref refs/heads/master $oid
+}
+
+# Note: DO NOT run it in a subshell, otherwise the variables will not be set
+create_pack_1 () {
+       P1=$(git -C "$master_repo/objects/pack" pack-objects -q pack <<-EOF

Which variables? Note that you can capture output of a subshell into a variable, if necessary.

#13 lore raw

Re: [PATCH v9 1/6] t5323: test cases for git-pack-redundant

Quoted 12 lines · Eric Sunshine

t/t5323-pack-redundant.sh line 1

Quoted 8 lines · Jiang Xin

Add test cases for git pack-redundant to validate new algorithm for git pack-redundant.

---

t/t5323-pack-redundant.sh +1 -0
@@ -0,0 +1,510 @@
+# Note: DO NOT run it in a subshell, otherwise the variables will not be set

Which variables won't be set? It's not clear what this restriction is about.

Quoted 9 lines

t/t5323-pack-redundant.sh line 69

Quoted 6 lines
t/t5323-pack-redundant.sh +6 -0
+       git -C "$repo" update-ref refs/heads/master $oid
+}
+
+# Note: DO NOT run it in a subshell, otherwise the variables will not be set
+create_pack_1 () {
+       P1=$(git -C "$master_repo/objects/pack" pack-objects -q pack <<-EOF

Which variables? Note that you can capture output of a subshell into a variable, if necessary.

These helper functions set a bunch of variables $P1, $P2, etc. as well as variables whose name begin with P and followed by 40-hex. The script wants to use them later when preparing expected output, and with the most natural way to organize the code, that "later" happens in the process that would have spawned a subshell to run this function.

It would have been easier for you to grok if the note instead said "this function sets two global shell variables" or something, perhaps? Such a variable would certainly not be visible if this function is called inside a subshell to the main process.

#14 lore raw

Re: [PATCH v9 1/6] t5323: test cases for git-pack-redundant

Quoted 20 lines · Junio C Hamano
Quoted 11 lines · Eric Sunshine
Quoted 1 line · Jiang Xin

+# Note: DO NOT run it in a subshell, otherwise the variables will not be set

Which variables won't be set? It's not clear what this restriction is about.

t/t5323-pack-redundant.sh line 69

Quoted 3 lines
t/t5323-pack-redundant.sh new file +3 -0
+# Note: DO NOT run it in a subshell, otherwise the variables will not be set
+create_pack_1 () {
+       P1=$(git -C "$master_repo/objects/pack" pack-objects -q pack <<-EOF

Which variables? Note that you can capture output of a subshell into a variable, if necessary.

These helper functions set a bunch of variables $P1, $P2, etc. as well as variables whose name begin with P and followed by 40-hex. The script wants to use them later when preparing expected output, and with the most natural way to organize the code, that "later" happens in the process that would have spawned a subshell to run this function.

Quoted 4 lines

It would have been easier for you to grok if the note instead said "this function sets two global shell variables" or something, perhaps? Such a variable would certainly not be visible if this function is called inside a subshell to the main process.

Yes, better function comments would facilitate comprehension both for the reviewer and those working on the code in the future. For instance:

# Create commit for each argument [...with blah properties...] and # assign [...] to shell variable of same name as argument. # NOTE: Avoid calling this function from a subshell since variable # assignments will disappear when subshell exits.

or something.

But, looking more closely at the patch, I'm wondering why the various create_pack_#() functions are defined at all since they are each only ever called from a single place.

    create_pack_4 () {
        ...
        eval P$P4=P4:$P4
    }
   ...
   test_expect_success 'create pack 4, 5' '
        create_pack_4 && create_pack_5
    '

I haven't been able to convince myself that this helps readability -- especially since the function definition is often far removed from the single point of use -- over merely inlining the function body directly in the sole test which calls it.

Anyhow, this is all pretty minor, not necessarily worth a re-roll.

author
#15 lore raw

v10 cover pack-redundant: new algorithm to find min packs

Sun Chao (my former colleague at Huawei) found a bug of git-pack-redundant. If there are too many packs and many of them overlap each other, running git pack-redundant --all will exhaust all memories and the process will be killed by kernel.

There is a script in commit log of commit 3/6, which can be used to create a repository with lots of redundant packs. Running git pack-redundant --all in it can reproduce this issue.

## Changes since re-roll v9

Quoted 20 lines · Eric Sunshine <[email protected]> 于2019年2月2日周六 上午3:43写道:

t/t5323-pack-redundant.sh line 1

Quoted 8 lines · Jiang Xin

Add test cases for git pack-redundant to validate new algorithm for git pack-redundant.

---

t/t5323-pack-redundant.sh +1 -0
@@ -0,0 +1,510 @@
+# Note: DO NOT run it in a subshell, otherwise the variables will not be set

Which variables won't be set? It's not clear what this restriction is about.

t/t5323-pack-redundant.sh line 46

Quoted 4 lines
t/t5323-pack-redundant.sh +4 -0
+# Usage: create_commits_in <repo> A B C ...
+create_commits_in () {
+       repo="$1" &&
+       parent=$(git -C "$repo" rev-parse HEAD^{} 2>/dev/null) || parent=

Broken &&-chain. Instead, perhaps:

Quoted 6 lines
    if ! parent=$(git -C "$repo" rev-parse HEAD^{} 2>/dev/null)
    then
        parent=
    fi &&

or something simpler.

Fixed.

Quoted 18 lines
Quoted 7 lines
t/t5323-pack-redundant.sh +7 -0
+       T=$(git -C "$repo" write-tree) &&
+       shift &&
+       while test $# -gt 0
+       do
+               name=$1 &&
+               test_tick &&
+               if test -z "$parent"

t/t5323-pack-redundant.sh line 63

Quoted 10 lines
t/t5323-pack-redundant.sh +10 -0
+               then
+                       oid=$(echo $name | git -C "$repo" commit-tree $T)
+               else
+                       oid=$(echo $name | git -C "$repo" commit-tree -p $parent $T)
+               fi &&
+               eval $name=$oid &&
+               parent=$oid &&
+               shift ||
+               return 1
+       done
Quoted 4 lines

Broken &&-chain. Use:

    done &&

Fixed, thanks.

Quoted 5 lines
Quoted 4 lines

It would have been easier for you to grok if the note instead said "this function sets two global shell variables" or something, perhaps? Such a variable would certainly not be visible if this function is called inside a subshell to the main process.

Quoted 8 lines

Yes, better function comments would facilitate comprehension both for the reviewer and those working on the code in the future. For instance:

# Create commit for each argument [...with blah properties...] and # assign [...] to shell variable of same name as argument. # NOTE: Avoid calling this function from a subshell since variable # assignments will disappear when subshell exits.

Polished comments for create_commits_in and create_pack_in helper function.

Quoted 9 lines
    create_pack_4 () {
        ...
        eval P$P4=P4:$P4
    }
   ...
   test_expect_success 'create pack 4, 5' '
        create_pack_4 && create_pack_5
    '
Quoted 4 lines

I haven't been able to convince myself that this helps readability -- especially since the function definition is often far removed from the single point of use -- over merely inlining the function body directly in the sole test which calls it.

Use a new helper function create_pack_in to create packs near test functions.

## Range diff since v9:

1: c8dbf8cef2 ! 1: 4719043603 t5323: test cases for git-pack-redundant

    @@ -43,8 +43,8 @@
     +    ALL | x x x x x x x x x x x x x x x x x x x
     +
     +Another repository `shared.git` has unique objects (X-Z), while other objects
    -+(marked with letter s) are shared through alt-odb (of `master.git`). The
    -+relationship between packs and objects is as follows:
    ++(marked with letter s) can be found in the shared alt-odb (of `master.git`).
    ++The relationship between packs and objects is as follows:
     +
     +        | T A B C D E F G H I J K L M N O P Q R   X Y Z
     +    ----+----------------------------------------------
    @@ -57,11 +57,19 @@
     +master_repo=master.git
     +shared_repo=shared.git
     +
    -+# Note: DO NOT run it in a subshell, otherwise the variables will not be set
    -+# Usage: create_commits_in <repo> A B C ...
    ++# Create commits in <repo> and assign each commit's oid to shell variables
    ++# given in the arguments (A, B, and C). E.g.:
    ++#
    ++#     create_commits_in <repo> A B C
    ++#
    ++# NOTE: Avoid calling this function from a subshell since variable
    ++# assignments will disappear when subshell exits.
     +create_commits_in () {
     +	repo="$1" &&
    -+	parent=$(git -C "$repo" rev-parse HEAD^{} 2>/dev/null) || parent=
    ++	if ! parent=$(git -C "$repo" rev-parse HEAD^{} 2>/dev/null)
    ++	then
    ++		parent=
    ++	fi &&
     +	T=$(git -C "$repo" write-tree) &&
     +	shift &&
     +	while test $# -gt 0
    @@ -78,101 +86,26 @@
     +		parent=$oid &&
     +		shift ||
     +		return 1
    -+	done
    ++	done &&
     +	git -C "$repo" update-ref refs/heads/master $oid
     +}
     +
    -+# Note: DO NOT run it in a subshell, otherwise the variables will not be set
    -+create_pack_1 () {
    -+	P1=$(git -C "$master_repo/objects/pack" pack-objects -q pack <<-EOF
    -+		$T
    -+		$A
    -+		$B
    -+		$C
    -+		$D
    -+		$E
    -+		$F
    -+		$R
    -+		EOF
    -+	) &&
    -+	eval P$P1=P1:$P1
    -+}
    -+
    -+create_pack_2 () {
    -+	P2=$(git -C "$master_repo/objects/pack" pack-objects -q pack <<-EOF
    -+		$B
    -+		$C
    -+		$D
    -+		$E
    -+		$G
    -+		$H
    -+		$I
    -+		EOF
    -+	) &&
    -+	eval P$P2=P2:$P2
    -+}
    -+
    -+create_pack_3 () {
    -+	P3=$(git -C "$master_repo/objects/pack" pack-objects -q pack <<-EOF
    -+		$F
    -+		$I
    -+		$J
    -+		$K
    -+		$L
    -+		$M
    -+		EOF
    -+	) &&
    -+	eval P$P3=P3:$P3
    -+}
    -+
    -+create_pack_4 () {
    -+	P4=$(git -C "$master_repo/objects/pack" pack-objects -q pack <<-EOF
    -+		$J
    -+		$K
    -+		$L
    -+		$M
    -+		$P
    -+		EOF
    -+	) &&
    -+	eval P$P4=P4:$P4
    -+}
    -+
    -+create_pack_5 () {
    -+	P5=$(git -C "$master_repo/objects/pack" pack-objects -q pack <<-EOF
    -+		$G
    -+		$H
    -+		$N
    -+		$O
    -+		EOF
    -+	) &&
    -+	eval P$P5=P5:$P5
    -+}
    -+
    -+create_pack_6 () {
    -+	P6=$(git -C "$master_repo/objects/pack" pack-objects -q pack <<-EOF
    -+		$N
    -+		$O
    -+		$Q
    -+		EOF
    -+	) &&
    -+	eval P$P6=P6:$P6
    -+}
    -+
    -+create_pack_7 () {
    -+	P7=$(git -C "$master_repo/objects/pack" pack-objects -q pack <<-EOF
    -+		$P
    -+		$Q
    -+		EOF
    -+	) &&
    -+	eval P$P7=P7:$P7
    -+}
    -+
    -+create_pack_8 () {
    -+	P8=$(git -C "$master_repo/objects/pack" pack-objects -q pack <<-EOF
    -+		$A
    -+		EOF
    -+	) &&
    -+	eval P$P8=P8:$P8
    ++# Create pack in <repo> and assign pack id to variable given in the 2nd argument
    ++# (<name>). Commits in the pack will be read from stdin. E.g.:
    ++#
    ++#     create_pack_in <repo> <name> <<-EOF
    ++#         ...
    ++#         EOF
    ++#
    ++# NOTE: commits from stdin should be given using heredoc, not using pipe, and
    ++# avoid calling this function from a subshell since variable assignments will
    ++# disappear when subshell exits.
    ++create_pack_in () {
    ++	repo="$1" &&
    ++	name="$2" &&
    ++	pack=$(git -C "$repo/objects/pack" pack-objects -q pack) &&
    ++	eval $name=$pack &&
    ++	eval P$pack=$name:$pack
     +}
     +
     +format_packfiles () {
    @@ -209,8 +142,34 @@
     +#     ALL | x x x x x x x x x x x x x x         x
     +#
     +#############################################################################
    -+test_expect_success 'no redundant for pack 1, 2, 3' '
    -+	create_pack_1 && create_pack_2 && create_pack_3 &&
    ++test_expect_success 'master: no redundant for pack 1, 2, 3' '
    ++	create_pack_in "$master_repo" P1 <<-EOF &&
    ++		$T
    ++		$A
    ++		$B
    ++		$C
    ++		$D
    ++		$E
    ++		$F
    ++		$R
    ++		EOF
    ++	create_pack_in "$master_repo" P2 <<-EOF &&
    ++		$B
    ++		$C
    ++		$D
    ++		$E
    ++		$G
    ++		$H
    ++		$I
    ++		EOF
    ++	create_pack_in "$master_repo" P3 <<-EOF &&
    ++		$F
    ++		$I
    ++		$J
    ++		$K
    ++		$L
    ++		$M
    ++		EOF
     +	(
     +		cd "$master_repo" &&
     +		git pack-redundant --all >out &&
    @@ -218,10 +177,6 @@
     +	)
     +'
     +
    -+test_expect_success 'create pack 4, 5' '
    -+	create_pack_4 && create_pack_5
    -+'
    -+
     +#############################################################################
     +# Chart of packs and objects for this test case
     +#
    @@ -236,7 +191,20 @@
     +#     ALL | x x x x x x x x x x x x x x x x x   x
     +#
     +#############################################################################
    -+test_expect_success 'one of pack-2/pack-3 is redundant' '
    ++test_expect_success 'master: one of pack-2/pack-3 is redundant' '
    ++	create_pack_in "$master_repo" P4 <<-EOF &&
    ++		$J
    ++		$K
    ++		$L
    ++		$M
    ++		$P
    ++		EOF
    ++	create_pack_in "$master_repo" P5 <<-EOF &&
    ++		$G
    ++		$H
    ++		$N
    ++		$O
    ++		EOF
     +	(
     +		cd "$master_repo" &&
     +		cat >expect <<-EOF &&
    @@ -248,10 +216,6 @@
     +	)
     +'
     +
    -+test_expect_success 'create pack 6, 7' '
    -+	create_pack_6 && create_pack_7
    -+'
    -+
     +#############################################################################
     +# Chart of packs and objects for this test case
     +#
    @@ -268,7 +232,16 @@
     +#     ALL | x x x x x x x x x x x x x x x x x x x
     +#
     +#############################################################################
    -+test_expect_success 'pack 2, 4, and 6 are redundant' '
    ++test_expect_success 'master: pack 2, 4, and 6 are redundant' '
    ++	create_pack_in "$master_repo" P6 <<-EOF &&
    ++		$N
    ++		$O
    ++		$Q
    ++		EOF
    ++	create_pack_in "$master_repo" P7 <<-EOF &&
    ++		$P
    ++		$Q
    ++		EOF
     +	(
     +		cd "$master_repo" &&
     +		cat >expect <<-EOF &&
    @@ -282,10 +255,6 @@
     +	)
     +'
     +
    -+test_expect_success 'create pack 8' '
    -+	create_pack_8
    -+'
    -+
     +#############################################################################
     +# Chart of packs and objects for this test case
     +#
    @@ -303,7 +272,10 @@
     +#     ALL | x x x x x x x x x x x x x x x x x x x
     +#
     +#############################################################################
    -+test_expect_success 'pack-8 (subset of pack-1) is also redundant' '
    ++test_expect_success 'master: pack-8 (subset of pack-1) is also redundant' '
    ++	create_pack_in "$master_repo" P8 <<-EOF &&
    ++		$A
    ++		EOF
     +	(
     +		cd "$master_repo" &&
     +		cat >expect <<-EOF &&
    @@ -318,7 +290,7 @@
     +	)
     +'
     +
    -+test_expect_success 'clean loose objects' '
    ++test_expect_success 'master: clean loose objects' '
     +	(
     +		cd "$master_repo" &&
     +		git prune-packed &&
    @@ -327,7 +299,7 @@
     +	)
     +'
     +
    -+test_expect_success 'remove redundant packs and pass fsck' '
    ++test_expect_success 'master: remove redundant packs and pass fsck' '
     +	(
     +		cd "$master_repo" &&
     +		git pack-redundant --all | xargs rm &&
    @@ -347,7 +319,7 @@
     +	)
     +'
     +
    -+test_expect_success 'no redundant packs without --alt-odb' '
    ++test_expect_success 'shared: all packs are redundant, but no output without --alt-odb' '
     +	(
     +		cd "$shared_repo" &&
     +		git pack-redundant --all >out &&
    @@ -380,7 +352,7 @@
     +#     ALL | x x x x x x x x x x x x x x x x x x x
     +#
     +#############################################################################
    -+test_expect_success 'pack-redundant --verbose: show duplicate packs in stderr' '
    ++test_expect_success 'shared: show redundant packs in stderr for verbose mode' '
     +	(
     +		cd "$shared_repo" &&
     +		cat >expect <<-EOF &&
    @@ -396,7 +368,7 @@
     +	)
     +'
     +
    -+test_expect_success 'remove redundant packs by alt-odb, no packs left' '
    ++test_expect_success 'shared: remove redundant packs, no packs left' '
     +	(
     +		cd "$shared_repo" &&
     +		cat >expect <<-EOF &&
    @@ -409,10 +381,9 @@
     +	)
     +'
     +
    -+# Note: DO NOT run function `create_pack_*` in sub shell, or variables are not set
    -+create_pack_x1_in () {
    -+	repo="$1" &&
    -+	Px1=$(git -C "$repo/objects/pack" pack-objects -q pack <<-EOF
    ++test_expect_success 'shared: create new objects and packs' '
    ++	create_commits_in "$shared_repo" X Y Z &&
    ++	create_pack_in "$shared_repo" Px1 <<-EOF &&
     +		$X
     +		$Y
     +		$Z
    @@ -420,13 +391,7 @@
     +		$B
     +		$C
     +		EOF
    -+	) &&
    -+	eval P${Px1}=Px1:${Px1}
    -+}
    -+
    -+create_pack_x2_in () {
    -+	repo="$1" &&
    -+	Px2=$(git -C "$repo/objects/pack" pack-objects -q pack <<-EOF
    ++	create_pack_in "$shared_repo" Px2 <<-EOF
     +		$X
     +		$Y
     +		$Z
    @@ -434,17 +399,9 @@
     +		$E
     +		$F
     +		EOF
    -+	) &&
    -+	eval P${Px2}=Px2:${Px2}
    -+}
    -+
    -+test_expect_success 'create new objects and packs in shared.git' '
    -+	create_commits_in "$shared_repo" X Y Z &&
    -+	create_pack_x1_in "$shared_repo" &&
    -+	create_pack_x2_in "$shared_repo"
     +'
     +
    -+test_expect_success 'no redundant without --alt-odb' '
    ++test_expect_success 'shared: no redundant without --alt-odb' '
     +	(
     +		cd "$shared_repo" &&
     +		git pack-redundant --all >out &&
    @@ -475,7 +432,7 @@
     +#     ALL | s s s s s s s s s s s s s s s s s s s   x x x
     +#
     +#############################################################################
    -+test_expect_success 'one pack is redundant' '
    ++test_expect_success 'shared: one pack is redundant with --alt-odb' '
     +	(
     +		cd "$shared_repo" &&
     +		git pack-redundant --all --alt-odb >out &&
    @@ -508,7 +465,7 @@
     +#                                                  (ignored objects, marked with i)
     +#
     +#############################################################################
    -+test_expect_success 'set ignore objects and all two packs are redundant' '
    ++test_expect_success 'shared: ignore unique objects and all two packs are redundant' '
     +	(
     +		cd "$shared_repo" &&
     +		cat >expect <<-EOF &&
2:  a6300516d7 = 2:  4feb1eaa40 pack-redundant: delay creation of unique_objects
3:  fb71973df5 = 3:  875367d7b4 pack-redundant: delete redundant code
4:  9963d1c49f ! 4:  50cb2854f1 pack-redundant: new algorithm to find min packs

@@ -331,35 +331,35 @@
# ALL | x x x x x x x x x x x x x x x x x x
#
#############################################################################
--test_expect_success 'one of pack-2/pack-3 is redundant' ' -+test_expect_failure 'one of pack-2/pack-3 is redundant (failed on Mac)' '
- (
- cd "$master_repo" &&
- cat >expect <<-EOF &&
+-test_expect_success 'master: one of pack-2/pack-3 is redundant' ' ++test_expect_failure 'master: one of pack-2/pack-3 is redundant (failed on Mac)' '
+ create_pack_in "$master_repo" P4 <<-EOF &&
+ $J
+ $K
@@
# ALL | x x x x x x x x x x x x x x x x x x x
#
#############################################################################
--test_expect_success 'pack 2, 4, and 6 are redundant' ' -+test_expect_failure 'pack 2, 4, and 6 are redundant (failed on Mac)' '
- (
- cd "$master_repo" &&
- cat >expect <<-EOF &&
+-test_expect_success 'master: pack 2, 4, and 6 are redundant' ' ++test_expect_failure 'master: pack 2, 4, and 6 are redundant (failed on Mac)' '
+ create_pack_in "$master_repo" P6 <<-EOF &&
+ $N
+ $O
@@
# ALL | x x x x x x x x x x x x x x x x x x x
#
#############################################################################
--test_expect_success 'pack-8 (subset of pack-1) is also redundant' ' -+test_expect_failure 'pack-8 (subset of pack-1) is also redundant (failed on Mac)' '
- (
- cd "$master_repo" &&
- cat >expect <<-EOF &&
+-test_expect_success 'master: pack-8 (subset of pack-1) is also redundant' ' ++test_expect_failure 'master: pack-8 (subset of pack-1) is also redundant (failed on Mac)' '
+ create_pack_in "$master_repo" P8 <<-EOF &&
+ $A
+ EOF
@@
)
'

--test_expect_success 'remove redundant packs and pass fsck' ' -+test_expect_failure 'remove redundant packs and pass fsck (failed on Mac)' ' +-test_expect_success 'master: remove redundant packs and pass fsck' ' ++test_expect_failure 'master: remove redundant packs and pass fsck (failed on Mac)' '
(
cd "$master_repo" &&
git pack-redundant --all | xargs rm &&
@@ -367,8 +367,8 @@
)
'

--test_expect_success 'no redundant packs without --alt-odb' '
-+test_expect_failure 'no redundant packs without --alt-odb (failed on Mac)' ' +-test_expect_success 'shared: all packs are redundant, but no output without --alt-odb' ' ++test_expect_failure 'shared: all packs are redundant, but no output without --alt-odb (failed on Mac)' '
(
cd "$shared_repo" &&
git pack-redundant --all >out &&
@@ -376,8 +376,8 @@
# ALL | x x x x x x x x x x x x x x x x x x x
#
#############################################################################
--test_expect_success 'pack-redundant --verbose: show duplicate packs in stderr' ' -+test_expect_failure 'pack-redundant --verbose: show duplicate packs in stderr (failed on Mac)' ' +-test_expect_success 'shared: show redundant packs in stderr for verbose mode' ' ++test_expect_failure 'shared: show redundant packs in stderr for verbose mode (failed on Mac)' '
(
cd "$shared_repo" &&
cat >expect <<-EOF &&

5:  b8f80ad454 = 5:  4af03876d4 pack-redundant: rename pack_list.all_objects
6:  8a12ad699e ! 6:  89ed4fb2a5 pack-redundant: consistent sort method

@@ -96,8 +96,12 @@
# ALL | x x x x x x x x x x x x x x x x x x
#
#############################################################################
--test_expect_failure 'one of pack-2/pack-3 is redundant (failed on Mac)' ' -+test_expect_success 'one of pack-2/pack-3 is redundant' ' +-test_expect_failure 'master: one of pack-2/pack-3 is redundant (failed on Mac)' ' ++test_expect_success 'master: one of pack-2/pack-3 is redundant' '
+ create_pack_in "$master_repo" P4 <<-EOF &&
+ $J
+ $K
+@@
(
cd "$master_repo" &&
cat >expect <<-EOF &&
@@ -110,26 +114,26 @@
# ALL | x x x x x x x x x x x x x x x x x x x
#
#############################################################################
--test_expect_failure 'pack 2, 4, and 6 are redundant (failed on Mac)' ' -+test_expect_success 'pack 2, 4, and 6 are redundant' '
- (
- cd "$master_repo" &&
- cat >expect <<-EOF &&
+-test_expect_failure 'master: pack 2, 4, and 6 are redundant (failed on Mac)' ' ++test_expect_success 'master: pack 2, 4, and 6 are redundant' '
+ create_pack_in "$master_repo" P6 <<-EOF &&
+ $N
+ $O
@@
# ALL | x x x x x x x x x x x x x x x x x x x
#
#############################################################################
--test_expect_failure 'pack-8 (subset of pack-1) is also redundant (failed on Mac)' ' -+test_expect_success 'pack-8 (subset of pack-1) is also redundant' '
- (
- cd "$master_repo" &&
- cat >expect <<-EOF &&
+-test_expect_failure 'master: pack-8 (subset of pack-1) is also redundant (failed on Mac)' ' ++test_expect_success 'master: pack-8 (subset of pack-1) is also redundant' '
+ create_pack_in "$master_repo" P8 <<-EOF &&
+ $A
+ EOF
@@
)
'

--test_expect_failure 'remove redundant packs and pass fsck (failed on Mac)' ' -+test_expect_success 'remove redundant packs and pass fsck' ' +-test_expect_failure 'master: remove redundant packs and pass fsck (failed on Mac)' ' ++test_expect_success 'master: remove redundant packs and pass fsck' '
(
cd "$master_repo" &&
git pack-redundant --all | xargs rm &&
@@ -137,8 +141,8 @@
)
'

--test_expect_failure 'no redundant packs without --alt-odb (failed on Mac)' ' -+test_expect_success 'no redundant packs without --alt-odb' '
+-test_expect_failure 'shared: all packs are redundant, but no output without --alt-odb (failed on Mac)' ' ++test_expect_success 'shared: all packs are redundant, but no output without --alt-odb' '
(
cd "$shared_repo" &&
git pack-redundant --all >out &&
@@ -146,8 +150,8 @@
# ALL | x x x x x x x x x x x x x x x x x x x
#
#############################################################################
--test_expect_failure 'pack-redundant --verbose: show duplicate packs in stderr (failed on Mac)' ' -+test_expect_success 'pack-redundant --verbose: show duplicate packs in stderr' ' +-test_expect_failure 'shared: show redundant packs in stderr for verbose mode (failed on Mac)' ' ++test_expect_success 'shared: show redundant packs in stderr for verbose mode' '
(
cd "$shared_repo" &&
cat >expect <<-EOF &&

--

Jiang Xin (4):
t5323: test cases for git-pack-redundant
pack-redundant: delay creation of unique_objects
pack-redundant: rename pack_list.all_objects
pack-redundant: consistent sort method

Sun Chao (2):
pack-redundant: delete redundant code
pack-redundant: new algorithm to find min packs

 builtin/pack-redundant.c  | 232 ++++++++-----------
 t/t5323-pack-redundant.sh | 467 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 559 insertions(+), 140 deletions(-)
 create mode 100755 t/t5323-pack-redundant.sh
Signature · 1 line
2.20.1.103.ged0fc2ca7b
author
#16 lore raw

PATCH v10 1/6 t5323: test cases for git-pack-redundant

In reply to Eric Sunshine

From: Jiang Xin <[email protected]>

Add test cases for git pack-redundant to validate new algorithm for git pack-redundant.

---
 t/t5323-pack-redundant.sh | 467 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 467 insertions(+)
 create mode 100755 t/t5323-pack-redundant.sh
t/t5323-pack-redundant.sh new file +467 -0
new file mode 100755
index 0000000000..f18db06d91
@@ -0,0 +1,467 @@
+#!/bin/sh
+#
+# Copyright (c) 2018 Jiang Xin
+#
+
+test_description='Test git pack-redundant
+
+In order to test git-pack-redundant, we will create a number of objects and
+packs in the repository `master.git`. The relationship between packs (P1-P8)
+and objects (T, A-R) is showed in the following chart. Objects of a pack will
+be marked with letter x, while objects of redundant packs will be marked with
+exclamation point, and redundant pack itself will be marked with asterisk.
+
+        | T A B C D E F G H I J K L M N O P Q R
+    ----+--------------------------------------
+    P1  | x x x x x x x                       x
+    P2* |     ! ! ! !   ! ! !
+    P3  |             x     x x x x x
+    P4* |                     ! ! ! !     !
+    P5  |               x x           x x
+    P6* |                             ! !   !
+    P7  |                                 x x
+    P8* |   !
+    ----+--------------------------------------
+    ALL | x x x x x x x x x x x x x x x x x x x
+
+Another repository `shared.git` has unique objects (X-Z), while other objects
+(marked with letter s) are shared through alt-odb (of `master.git`). The
+relationship between packs and objects is as follows:
+
+        | T A B C D E F G H I J K L M N O P Q R   X Y Z
+    ----+----------------------------------------------
+    Px1 |   s s s                                 x x x
+    Px2 |         s s s                           x x x
+'
+
+. ./test-lib.sh
+
+master_repo=master.git
+shared_repo=shared.git
+
+# Create commits in <repo> and assign each commit's oid to shell variables
+# given in the arguments (A, B, and C). E.g.:
+#
+#     create_commits_in <repo> A B C
+#
+# NOTE: Avoid calling this function from a subshell since variable
+# assignments will disappear when subshell exits.
+create_commits_in () {
+	repo="$1" &&
+	if ! parent=$(git -C "$repo" rev-parse HEAD^{} 2>/dev/null)
+	then
+		parent=
+	fi &&
+	T=$(git -C "$repo" write-tree) &&
+	shift &&
+	while test $# -gt 0
+	do
+		name=$1 &&
+		test_tick &&
+		if test -z "$parent"
+		then
+			oid=$(echo $name | git -C "$repo" commit-tree $T)
+		else
+			oid=$(echo $name | git -C "$repo" commit-tree -p $parent $T)
+		fi &&
+		eval $name=$oid &&
+		parent=$oid &&
+		shift ||
+		return 1
+	done &&
+	git -C "$repo" update-ref refs/heads/master $oid
+}
+
+# Create pack in <repo> and assign pack id to variable given in the 2nd argument
+# (<name>). Commits in the pack will be read from stdin. E.g.:
+#
+#     create_pack_in <repo> <name> <<-EOF
+#         ...
+#         EOF
+#
+# NOTE: commits from stdin should be given using heredoc, not using pipe, and
+# avoid calling this function from a subshell since variable assignments will
+# disappear when subshell exits.
+create_pack_in () {
+	repo="$1" &&
+	name="$2" &&
+	pack=$(git -C "$repo/objects/pack" pack-objects -q pack) &&
+	eval $name=$pack &&
+	eval P$pack=$name:$pack
+}
+
+format_packfiles () {
+	sed \
+		-e "s#.*/pack-\(.*\)\.idx#\1#" \
+		-e "s#.*/pack-\(.*\)\.pack#\1#" |
+	sort -u |
+	while read p
+	do
+		if test -z "$(eval echo \${P$p})"
+		then
+			echo $p
+		else
+			eval echo "\${P$p}"
+		fi
+	done |
+	sort
+}
+
+test_expect_success 'setup master repo' '
+	git init --bare "$master_repo" &&
+	create_commits_in "$master_repo" A B C D E F G H I J K L M N O P Q R
+'
+
+#############################################################################
+# Chart of packs and objects for this test case
+#
+#         | T A B C D E F G H I J K L M N O P Q R
+#     ----+--------------------------------------
+#     P1  | x x x x x x x                       x
+#     P2  |     x x x x   x x x
+#     P3  |             x     x x x x x
+#     ----+--------------------------------------
+#     ALL | x x x x x x x x x x x x x x         x
+#
+#############################################################################
+test_expect_success 'master: no redundant for pack 1, 2, 3' '
+	create_pack_in "$master_repo" P1 <<-EOF &&
+		$T
+		$A
+		$B
+		$C
+		$D
+		$E
+		$F
+		$R
+		EOF
+	create_pack_in "$master_repo" P2 <<-EOF &&
+		$B
+		$C
+		$D
+		$E
+		$G
+		$H
+		$I
+		EOF
+	create_pack_in "$master_repo" P3 <<-EOF &&
+		$F
+		$I
+		$J
+		$K
+		$L
+		$M
+		EOF
+	(
+		cd "$master_repo" &&
+		git pack-redundant --all >out &&
+		test_must_be_empty out
+	)
+'
+
+#############################################################################
+# Chart of packs and objects for this test case
+#
+#         | T A B C D E F G H I J K L M N O P Q R
+#     ----+--------------------------------------
+#     P1  | x x x x x x x                       x
+#     P2* |     ! ! ! !   ! ! !
+#     P3  |             x     x x x x x
+#     P4  |                     x x x x     x
+#     P5  |               x x           x x
+#     ----+--------------------------------------
+#     ALL | x x x x x x x x x x x x x x x x x   x
+#
+#############################################################################
+test_expect_success 'master: one of pack-2/pack-3 is redundant' '
+	create_pack_in "$master_repo" P4 <<-EOF &&
+		$J
+		$K
+		$L
+		$M
+		$P
+		EOF
+	create_pack_in "$master_repo" P5 <<-EOF &&
+		$G
+		$H
+		$N
+		$O
+		EOF
+	(
+		cd "$master_repo" &&
+		cat >expect <<-EOF &&
+			P2:$P2
+			EOF
+		git pack-redundant --all >out &&
+		format_packfiles <out >actual &&
+		test_cmp expect actual
+	)
+'
+
+#############################################################################
+# Chart of packs and objects for this test case
+#
+#         | T A B C D E F G H I J K L M N O P Q R
+#     ----+--------------------------------------
+#     P1  | x x x x x x x                       x
+#     P2* |     ! ! ! !   ! ! !
+#     P3  |             x     x x x x x
+#     P4* |                     ! ! ! !     !
+#     P5  |               x x           x x
+#     P6* |                             ! !   !
+#     P7  |                                 x x
+#     ----+--------------------------------------
+#     ALL | x x x x x x x x x x x x x x x x x x x
+#
+#############################################################################
+test_expect_success 'master: pack 2, 4, and 6 are redundant' '
+	create_pack_in "$master_repo" P6 <<-EOF &&
+		$N
+		$O
+		$Q
+		EOF
+	create_pack_in "$master_repo" P7 <<-EOF &&
+		$P
+		$Q
+		EOF
+	(
+		cd "$master_repo" &&
+		cat >expect <<-EOF &&
+			P2:$P2
+			P4:$P4
+			P6:$P6
+			EOF
+		git pack-redundant --all >out &&
+		format_packfiles <out >actual &&
+		test_cmp expect actual
+	)
+'
+
+#############################################################################
+# Chart of packs and objects for this test case
+#
+#         | T A B C D E F G H I J K L M N O P Q R
+#     ----+--------------------------------------
+#     P1  | x x x x x x x                       x
+#     P2* |     ! ! ! !   ! ! !
+#     P3  |             x     x x x x x
+#     P4* |                     ! ! ! !     !
+#     P5  |               x x           x x
+#     P6* |                             ! !   !
+#     P7  |                                 x x
+#     P8* |   !
+#     ----+--------------------------------------
+#     ALL | x x x x x x x x x x x x x x x x x x x
+#
+#############################################################################
+test_expect_success 'master: pack-8 (subset of pack-1) is also redundant' '
+	create_pack_in "$master_repo" P8 <<-EOF &&
+		$A
+		EOF
+	(
+		cd "$master_repo" &&
+		cat >expect <<-EOF &&
+			P2:$P2
+			P4:$P4
+			P6:$P6
+			P8:$P8
+			EOF
+		git pack-redundant --all >out &&
+		format_packfiles <out >actual &&
+		test_cmp expect actual
+	)
+'
+
+test_expect_success 'master: clean loose objects' '
+	(
+		cd "$master_repo" &&
+		git prune-packed &&
+		find objects -type f | sed -e "/objects\/pack\//d" >out &&
+		test_must_be_empty out
+	)
+'
+
+test_expect_success 'master: remove redundant packs and pass fsck' '
+	(
+		cd "$master_repo" &&
+		git pack-redundant --all | xargs rm &&
+		git fsck &&
+		git pack-redundant --all >out &&
+		test_must_be_empty out
+	)
+'
+
+# The following test cases will execute inside `shared.git`, instead of
+# inside `master.git`.
+test_expect_success 'setup shared.git' '
+	git clone --mirror "$master_repo" "$shared_repo" &&
+	(
+		cd "$shared_repo" &&
+		printf "../../$master_repo/objects\n" >objects/info/alternates
+	)
+'
+
+test_expect_success 'shared: all packs are redundant, but no output without --alt-odb' '
+	(
+		cd "$shared_repo" &&
+		git pack-redundant --all >out &&
+		test_must_be_empty out
+	)
+'
+
+#############################################################################
+# Chart of packs and objects for this test case
+#
+#     ================ master.git ===============
+#         | T A B C D E F G H I J K L M N O P Q R  <----------+
+#     ----+--------------------------------------             |
+#     P1  | x x x x x x x                       x             |
+#     P3  |             x     x x x x x                       |
+#     P5  |               x x           x x                   |
+#     P7  |                                 x x               |
+#     ----+--------------------------------------             |
+#     ALL | x x x x x x x x x x x x x x x x x x x             |
+#                                                             |
+#                                                             |
+#     ================ shared.git ===============             |
+#         | T A B C D E F G H I J K L M N O P Q R  <objects/info/alternates>
+#     ----+--------------------------------------
+#     P1* | s s s s s s s                       s
+#     P3* |             s     s s s s s
+#     P5* |               s s           s s
+#     P7* |                                 s s
+#     ----+--------------------------------------
+#     ALL | x x x x x x x x x x x x x x x x x x x
+#
+#############################################################################
+test_expect_success 'shared: show redundant packs in stderr for verbose mode' '
+	(
+		cd "$shared_repo" &&
+		cat >expect <<-EOF &&
+			P1:$P1
+			P3:$P3
+			P5:$P5
+			P7:$P7
+			EOF
+		git pack-redundant --all --verbose >out 2>out.err &&
+		test_must_be_empty out &&
+		grep "pack$" out.err | format_packfiles >actual &&
+		test_cmp expect actual
+	)
+'
+
+test_expect_success 'shared: remove redundant packs, no packs left' '
+	(
+		cd "$shared_repo" &&
+		cat >expect <<-EOF &&
+			fatal: Zero packs found!
+			EOF
+		git pack-redundant --all --alt-odb | xargs rm &&
+		git fsck &&
+		test_must_fail git pack-redundant --all --alt-odb >actual 2>&1 &&
+		test_cmp expect actual
+	)
+'
+
+test_expect_success 'shared: create new objects and packs' '
+	create_commits_in "$shared_repo" X Y Z &&
+	create_pack_in "$shared_repo" Px1 <<-EOF &&
+		$X
+		$Y
+		$Z
+		$A
+		$B
+		$C
+		EOF
+	create_pack_in "$shared_repo" Px2 <<-EOF
+		$X
+		$Y
+		$Z
+		$D
+		$E
+		$F
+		EOF
+'
+
+test_expect_success 'shared: no redundant without --alt-odb' '
+	(
+		cd "$shared_repo" &&
+		git pack-redundant --all >out &&
+		test_must_be_empty out
+	)
+'
+
+#############################################################################
+# Chart of packs and objects for this test case
+#
+#     ================ master.git ===============
+#         | T A B C D E F G H I J K L M N O P Q R  <----------------+
+#     ----+--------------------------------------                   |
+#     P1  | x x x x x x x                       x                   |
+#     P3  |             x     x x x x x                             |
+#     P5  |               x x           x x                         |
+#     P7  |                                 x x                     |
+#     ----+--------------------------------------                   |
+#     ALL | x x x x x x x x x x x x x x x x x x x                   |
+#                                                                   |
+#                                                                   |
+#     ================ shared.git =======================           |
+#         | T A B C D E F G H I J K L M N O P Q R   X Y Z <objects/info/alternates>
+#     ----+----------------------------------------------
+#     Px1 |   s s s                                 x x x
+#     Px2*|         s s s                           ! ! !
+#     ----+----------------------------------------------
+#     ALL | s s s s s s s s s s s s s s s s s s s   x x x
+#
+#############################################################################
+test_expect_success 'shared: one pack is redundant with --alt-odb' '
+	(
+		cd "$shared_repo" &&
+		git pack-redundant --all --alt-odb >out &&
+		format_packfiles <out >actual &&
+		test_line_count = 1 actual
+	)
+'
+
+#############################################################################
+# Chart of packs and objects for this test case
+#
+#     ================ master.git ===============
+#         | T A B C D E F G H I J K L M N O P Q R  <----------------+
+#     ----+--------------------------------------                   |
+#     P1  | x x x x x x x                       x                   |
+#     P3  |             x     x x x x x                             |
+#     P5  |               x x           x x                         |
+#     P7  |                                 x x                     |
+#     ----+--------------------------------------                   |
+#     ALL | x x x x x x x x x x x x x x x x x x x                   |
+#                                                                   |
+#                                                                   |
+#     ================ shared.git =======================           |
+#         | T A B C D E F G H I J K L M N O P Q R   X Y Z <objects/info/alternates>
+#     ----+----------------------------------------------
+#     Px1*|   s s s                                 i i i
+#     Px2*|         s s s                           i i i
+#     ----+----------------------------------------------
+#     ALL | s s s s s s s s s s s s s s s s s s s   i i i
+#                                                  (ignored objects, marked with i)
+#
+#############################################################################
+test_expect_success 'shared: ignore unique objects and all two packs are redundant' '
+	(
+		cd "$shared_repo" &&
+		cat >expect <<-EOF &&
+			Px1:$Px1
+			Px2:$Px2
+			EOF
+		git pack-redundant --all --alt-odb >out <<-EOF &&
+			$X
+			$Y
+			$Z
+			EOF
+		format_packfiles <out >actual &&
+		test_cmp expect actual
+	)
+'
+
+test_done
Signature · 1 line
2.20.1.103.ged0fc2ca7b
author
#17 lore raw

PATCH v10 2/6 pack-redundant: delay creation of unique_objects

In reply to Eric Sunshine

From: Jiang Xin <[email protected]>

Instead of initializing unique_objects in add_pack(), copy from all_objects in cmp_two_packs(), when unwanted objects are removed from all_objects.

This will save memory (no allocate memory for alt-odb packs), and run llist_sorted_difference_inplace() only once when removing ignored objects and removing objects in alt-odb in scan_alt_odb_packs().

---
 builtin/pack-redundant.c | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)
builtin/pack-redundant.c +10 -5
index cf9a9aabd4..f7dab0ec60 100644
@@ -254,6 +254,11 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
 	struct llist_item *p1_hint = NULL, *p2_hint = NULL;
 	const unsigned int hashsz = the_hash_algo->rawsz;
 
+	if (!p1->unique_objects)
+		p1->unique_objects = llist_copy(p1->all_objects);
+	if (!p2->unique_objects)
+		p2->unique_objects = llist_copy(p2->all_objects);
+
 	p1_base = p1->pack->index_data;
 	p2_base = p2->pack->index_data;
 	p1_base += 256 * 4 + ((p1->pack->index_version < 2) ? 4 : 8);
@@ -536,7 +541,7 @@ static void scan_alt_odb_packs(void)
 	while (alt) {
 		local = local_packs;
 		while (local) {
-			llist_sorted_difference_inplace(local->unique_objects,
+			llist_sorted_difference_inplace(local->all_objects,
 							alt->all_objects);
 			local = local->next;
 		}
@@ -567,8 +572,7 @@ static struct pack_list * add_pack(struct packed_git *p)
 		llist_insert_back(l.all_objects, (const struct object_id *)(base + off));
 		off += step;
 	}
-	/* this list will be pruned in cmp_two_packs later */
-	l.unique_objects = llist_copy(l.all_objects);
+	l.unique_objects = NULL;
 	if (p->pack_local)
 		return pack_list_insert(&local_packs, &l);
 	else
@@ -646,7 +650,6 @@ int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 
 	load_all_objects();
 
-	cmp_local_packs();
 	if (alt_odb)
 		scan_alt_odb_packs();
 
@@ -663,10 +666,12 @@ int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 	llist_sorted_difference_inplace(all_objects, ignore);
 	pl = local_packs;
 	while (pl) {
-		llist_sorted_difference_inplace(pl->unique_objects, ignore);
+		llist_sorted_difference_inplace(pl->all_objects, ignore);
 		pl = pl->next;
 	}
 
+	cmp_local_packs();
+
 	minimize(&min);
 
 	if (verbose) {
Signature · 1 line
2.20.1.103.ged0fc2ca7b
author
#18 lore raw

PATCH v10 3/6 pack-redundant: delete redundant code

In reply to Eric Sunshine

From: Sun Chao <[email protected]>

The objects in alt-odb are removed from all_objects twice in load_all_objects and scan_alt_odb_packs, remove it from the later function.

---
 builtin/pack-redundant.c | 1 -
 1 file changed, 1 deletion(-)
builtin/pack-redundant.c +0 -1
index f7dab0ec60..4a06f057dd 100644
@@ -545,7 +545,6 @@ static void scan_alt_odb_packs(void)
 							alt->all_objects);
 			local = local->next;
 		}
-		llist_sorted_difference_inplace(all_objects, alt->all_objects);
 		alt = alt->next;
 	}
 }
Signature · 1 line
2.20.1.103.ged0fc2ca7b
author
#19 lore raw

PATCH v10 4/6 pack-redundant: new algorithm to find min packs

In reply to Eric Sunshine

From: Sun Chao <[email protected]>

When calling git pack-redundant --all, if there are too many local packs and too many redundant objects within them, the too deep iteration of get_permutations will exhaust all the resources, and the process of git pack-redundant will be killed.

The following script could create a repository with too many redundant packs, and running git pack-redundant --all in the test.git repo will die soon.

    #!/bin/sh

    repo="$(pwd)/test.git"
    work="$(pwd)/test"
    i=1
    max=199

    if test -d "$repo" || test -d "$work"; then
    	echo >&2 "ERROR: '$repo' or '$work' already exist"
    	exit 1
    fi

    git init -q --bare "$repo"
    git --git-dir="$repo" config gc.auto 0
    git --git-dir="$repo" config transfer.unpackLimit 0
    git clone -q "$repo" "$work" 2>/dev/null

    while :; do
        cd "$work"
        echo "loop $i: $(date +%s)" >$i
        git add $i
        git commit -q -sm "loop $i"
        git push -q origin HEAD:master
        printf "\rCreate pack %4d/%d\t" $i $max
        if test $i -ge $max; then break; fi

        cd "$repo"
        git repack -q
        if test $(($i % 2)) -eq 0; then
            git repack -aq
            pack=$(ls -t $repo/objects/pack/*.pack | head -1)
            touch "${pack%.pack}.keep"
        fi
        i=$((i+1))
    done
    printf "\ndone\n"

To get the min unique pack list, we can replace the iteration in minimize function with a new algorithm, and this could solve this issue:

1. Get the unique and non_uniqe packs, add the unique packs to the min list.

2. Remove the objects of unique packs from non_unique packs, then each object left in the non_unique packs will have at least two copies.

3. Sort the non_unique packs by the objects' size, more objects first, and add the first non_unique pack to min list.

4. Drop the duplicated objects from other packs in the ordered non_unique pack list, and repeat step 3.

Some test cases will fail on Mac OS X. Mark them and will resolve in later commit.

Original PR and discussions: https://github.com/jiangxin/git/pull/25

---
 builtin/pack-redundant.c  | 194 +++++++++++++-------------------------
 t/t5323-pack-redundant.sh |  12 +--
 2 files changed, 73 insertions(+), 133 deletions(-)
builtin/pack-redundant.c +67 -127
index 4a06f057dd..d6d9a66e46 100644
@@ -35,11 +35,6 @@ static struct pack_list {
 	struct llist *all_objects;
 } *local_packs = NULL, *altodb_packs = NULL;
 
-struct pll {
-	struct pll *next;
-	struct pack_list *pl;
-};
-
 static struct llist_item *free_nodes;
 
 static inline void llist_item_put(struct llist_item *item)
@@ -63,15 +58,6 @@ static inline struct llist_item *llist_item_get(void)
 	return new_item;
 }
 
-static void llist_free(struct llist *list)
-{
-	while ((list->back = list->front)) {
-		list->front = list->front->next;
-		llist_item_put(list->back);
-	}
-	free(list);
-}
-
 static inline void llist_init(struct llist **list)
 {
 	*list = xmalloc(sizeof(struct llist));
@@ -290,78 +276,6 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
 	}
 }
 
-static void pll_free(struct pll *l)
-{
-	struct pll *old;
-	struct pack_list *opl;
-
-	while (l) {
-		old = l;
-		while (l->pl) {
-			opl = l->pl;
-			l->pl = opl->next;
-			free(opl);
-		}
-		l = l->next;
-		free(old);
-	}
-}
-
-/* all the permutations have to be free()d at the same time,
- * since they refer to each other
- */
-static struct pll * get_permutations(struct pack_list *list, int n)
-{
-	struct pll *subset, *ret = NULL, *new_pll = NULL;
-
-	if (list == NULL || pack_list_size(list) < n || n == 0)
-		return NULL;
-
-	if (n == 1) {
-		while (list) {
-			new_pll = xmalloc(sizeof(*new_pll));
-			new_pll->pl = NULL;
-			pack_list_insert(&new_pll->pl, list);
-			new_pll->next = ret;
-			ret = new_pll;
-			list = list->next;
-		}
-		return ret;
-	}
-
-	while (list->next) {
-		subset = get_permutations(list->next, n - 1);
-		while (subset) {
-			new_pll = xmalloc(sizeof(*new_pll));
-			new_pll->pl = subset->pl;
-			pack_list_insert(&new_pll->pl, list);
-			new_pll->next = ret;
-			ret = new_pll;
-			subset = subset->next;
-		}
-		list = list->next;
-	}
-	return ret;
-}
-
-static int is_superset(struct pack_list *pl, struct llist *list)
-{
-	struct llist *diff;
-
-	diff = llist_copy(list);
-
-	while (pl) {
-		llist_sorted_difference_inplace(diff, pl->all_objects);
-		if (diff->size == 0) { /* we're done */
-			llist_free(diff);
-			return 1;
-		}
-		pl = pl->next;
-	}
-	llist_free(diff);
-	return 0;
-}
-
 static size_t sizeof_union(struct packed_git *p1, struct packed_git *p2)
 {
 	size_t ret = 0;
@@ -426,14 +340,52 @@ static inline off_t pack_set_bytecount(struct pack_list *pl)
 	return ret;
 }
 
+static int cmp_pack_list_reverse(const void *a, const void *b)
+{
+	struct pack_list *pl_a = *((struct pack_list **)a);
+	struct pack_list *pl_b = *((struct pack_list **)b);
+	size_t sz_a = pl_a->all_objects->size;
+	size_t sz_b = pl_b->all_objects->size;
+
+	if (sz_a == sz_b)
+		return 0;
+	else if (sz_a < sz_b)
+		return 1;
+	else
+		return -1;
+}
+
+/* Sort pack_list, greater size of all_objects first */
+static void sort_pack_list(struct pack_list **pl)
+{
+	struct pack_list **ary, *p;
+	int i;
+	size_t n = pack_list_size(*pl);
+
+	if (n < 2)
+		return;
+
+	/* prepare an array of packed_list for easier sorting */
+	ary = xcalloc(n, sizeof(struct pack_list *));
+	for (n = 0, p = *pl; p; p = p->next)
+		ary[n++] = p;
+
+	QSORT(ary, n, cmp_pack_list_reverse);
+
+	/* link them back again */
+	for (i = 0; i < n - 1; i++)
+		ary[i]->next = ary[i + 1];
+	ary[n - 1]->next = NULL;
+	*pl = ary[0];
+
+	free(ary);
+}
+
+
 static void minimize(struct pack_list **min)
 {
-	struct pack_list *pl, *unique = NULL,
-		*non_unique = NULL, *min_perm = NULL;
-	struct pll *perm, *perm_all, *perm_ok = NULL, *new_perm;
-	struct llist *missing;
-	off_t min_perm_size = 0, perm_size;
-	int n;
+	struct pack_list *pl, *unique = NULL, *non_unique = NULL;
+	struct llist *missing, *unique_pack_objects;
 
 	pl = local_packs;
 	while (pl) {
@@ -451,49 +403,37 @@ static void minimize(struct pack_list **min)
 		pl = pl->next;
 	}
 
+	*min = unique;
+
 	/* return if there are no objects missing from the unique set */
 	if (missing->size == 0) {
-		*min = unique;
 		free(missing);
 		return;
 	}
 
-	/* find the permutations which contain all missing objects */
-	for (n = 1; n <= pack_list_size(non_unique) && !perm_ok; n++) {
-		perm_all = perm = get_permutations(non_unique, n);
-		while (perm) {
-			if (is_superset(perm->pl, missing)) {
-				new_perm = xmalloc(sizeof(struct pll));
-				memcpy(new_perm, perm, sizeof(struct pll));
-				new_perm->next = perm_ok;
-				perm_ok = new_perm;
-			}
-			perm = perm->next;
-		}
-		if (perm_ok)
-			break;
-		pll_free(perm_all);
-	}
-	if (perm_ok == NULL)
-		die("Internal error: No complete sets found!");
-
-	/* find the permutation with the smallest size */
-	perm = perm_ok;
-	while (perm) {
-		perm_size = pack_set_bytecount(perm->pl);
-		if (!min_perm_size || min_perm_size > perm_size) {
-			min_perm_size = perm_size;
-			min_perm = perm->pl;
-		}
-		perm = perm->next;
-	}
-	*min = min_perm;
-	/* add the unique packs to the list */
-	pl = unique;
+	unique_pack_objects = llist_copy(all_objects);
+	llist_sorted_difference_inplace(unique_pack_objects, missing);
+
+	/* remove unique pack objects from the non_unique packs */
+	pl = non_unique;
 	while (pl) {
-		pack_list_insert(min, pl);
+		llist_sorted_difference_inplace(pl->all_objects, unique_pack_objects);
 		pl = pl->next;
 	}
+
+	while (non_unique) {
+		/* sort the non_unique packs, greater size of all_objects first */
+		sort_pack_list(&non_unique);
+		if (non_unique->all_objects->size == 0)
+			break;
+
+		pack_list_insert(min, non_unique);
+
+		for (pl = non_unique->next; pl && pl->all_objects->size > 0;  pl = pl->next)
+			llist_sorted_difference_inplace(pl->all_objects, non_unique->all_objects);
+
+		non_unique = non_unique->next;
+	}
 }
 
 static void load_all_objects(void)
@@ -606,7 +546,7 @@ static void load_all(void)
 int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 {
 	int i;
-	struct pack_list *min, *red, *pl;
+	struct pack_list *min = NULL, *red, *pl;
 	struct llist *ignore;
 	struct object_id *oid;
 	char buf[GIT_MAX_HEXSZ + 2]; /* hex hash + \n + \0 */
t/t5323-pack-redundant.sh +6 -6
index f18db06d91..3e62e8663f 100755
@@ -173,7 +173,7 @@ test_expect_success 'master: no redundant for pack 1, 2, 3' '
 #     ALL | x x x x x x x x x x x x x x x x x   x
 #
 #############################################################################
-test_expect_success 'master: one of pack-2/pack-3 is redundant' '
+test_expect_failure 'master: one of pack-2/pack-3 is redundant (failed on Mac)' '
 	create_pack_in "$master_repo" P4 <<-EOF &&
 		$J
 		$K
@@ -214,7 +214,7 @@ test_expect_success 'master: one of pack-2/pack-3 is redundant' '
 #     ALL | x x x x x x x x x x x x x x x x x x x
 #
 #############################################################################
-test_expect_success 'master: pack 2, 4, and 6 are redundant' '
+test_expect_failure 'master: pack 2, 4, and 6 are redundant (failed on Mac)' '
 	create_pack_in "$master_repo" P6 <<-EOF &&
 		$N
 		$O
@@ -254,7 +254,7 @@ test_expect_success 'master: pack 2, 4, and 6 are redundant' '
 #     ALL | x x x x x x x x x x x x x x x x x x x
 #
 #############################################################################
-test_expect_success 'master: pack-8 (subset of pack-1) is also redundant' '
+test_expect_failure 'master: pack-8 (subset of pack-1) is also redundant (failed on Mac)' '
 	create_pack_in "$master_repo" P8 <<-EOF &&
 		$A
 		EOF
@@ -281,7 +281,7 @@ test_expect_success 'master: clean loose objects' '
 	)
 '
 
-test_expect_success 'master: remove redundant packs and pass fsck' '
+test_expect_failure 'master: remove redundant packs and pass fsck (failed on Mac)' '
 	(
 		cd "$master_repo" &&
 		git pack-redundant --all | xargs rm &&
@@ -301,7 +301,7 @@ test_expect_success 'setup shared.git' '
 	)
 '
 
-test_expect_success 'shared: all packs are redundant, but no output without --alt-odb' '
+test_expect_failure 'shared: all packs are redundant, but no output without --alt-odb (failed on Mac)' '
 	(
 		cd "$shared_repo" &&
 		git pack-redundant --all >out &&
@@ -334,7 +334,7 @@ test_expect_success 'shared: all packs are redundant, but no output without --al
 #     ALL | x x x x x x x x x x x x x x x x x x x
 #
 #############################################################################
-test_expect_success 'shared: show redundant packs in stderr for verbose mode' '
+test_expect_failure 'shared: show redundant packs in stderr for verbose mode (failed on Mac)' '
 	(
 		cd "$shared_repo" &&
 		cat >expect <<-EOF &&
Signature · 1 line
2.20.1.103.ged0fc2ca7b
author
#20 lore raw

PATCH v10 5/6 pack-redundant: rename pack_list.all_objects

In reply to Eric Sunshine

From: Jiang Xin <[email protected]>

New algorithm uses pack_list.all_objects to track remaining objects, so rename it to pack_list.remaining_objects.

---
 builtin/pack-redundant.c | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)
builtin/pack-redundant.c +19 -19
index d6d9a66e46..15cdf233c4 100644
@@ -32,7 +32,7 @@ static struct pack_list {
 	struct pack_list *next;
 	struct packed_git *pack;
 	struct llist *unique_objects;
-	struct llist *all_objects;
+	struct llist *remaining_objects;
 } *local_packs = NULL, *altodb_packs = NULL;
 
 static struct llist_item *free_nodes;
@@ -241,9 +241,9 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
 	const unsigned int hashsz = the_hash_algo->rawsz;
 
 	if (!p1->unique_objects)
-		p1->unique_objects = llist_copy(p1->all_objects);
+		p1->unique_objects = llist_copy(p1->remaining_objects);
 	if (!p2->unique_objects)
-		p2->unique_objects = llist_copy(p2->all_objects);
+		p2->unique_objects = llist_copy(p2->remaining_objects);
 
 	p1_base = p1->pack->index_data;
 	p2_base = p2->pack->index_data;
@@ -344,8 +344,8 @@ static int cmp_pack_list_reverse(const void *a, const void *b)
 {
 	struct pack_list *pl_a = *((struct pack_list **)a);
 	struct pack_list *pl_b = *((struct pack_list **)b);
-	size_t sz_a = pl_a->all_objects->size;
-	size_t sz_b = pl_b->all_objects->size;
+	size_t sz_a = pl_a->remaining_objects->size;
+	size_t sz_b = pl_b->remaining_objects->size;
 
 	if (sz_a == sz_b)
 		return 0;
@@ -355,7 +355,7 @@ static int cmp_pack_list_reverse(const void *a, const void *b)
 		return -1;
 }
 
-/* Sort pack_list, greater size of all_objects first */
+/* Sort pack_list, greater size of remaining_objects first */
 static void sort_pack_list(struct pack_list **pl)
 {
 	struct pack_list **ary, *p;
@@ -399,7 +399,7 @@ static void minimize(struct pack_list **min)
 	missing = llist_copy(all_objects);
 	pl = unique;
 	while (pl) {
-		llist_sorted_difference_inplace(missing, pl->all_objects);
+		llist_sorted_difference_inplace(missing, pl->remaining_objects);
 		pl = pl->next;
 	}
 
@@ -417,20 +417,20 @@ static void minimize(struct pack_list **min)
 	/* remove unique pack objects from the non_unique packs */
 	pl = non_unique;
 	while (pl) {
-		llist_sorted_difference_inplace(pl->all_objects, unique_pack_objects);
+		llist_sorted_difference_inplace(pl->remaining_objects, unique_pack_objects);
 		pl = pl->next;
 	}
 
 	while (non_unique) {
-		/* sort the non_unique packs, greater size of all_objects first */
+		/* sort the non_unique packs, greater size of remaining_objects first */
 		sort_pack_list(&non_unique);
-		if (non_unique->all_objects->size == 0)
+		if (non_unique->remaining_objects->size == 0)
 			break;
 
 		pack_list_insert(min, non_unique);
 
-		for (pl = non_unique->next; pl && pl->all_objects->size > 0;  pl = pl->next)
-			llist_sorted_difference_inplace(pl->all_objects, non_unique->all_objects);
+		for (pl = non_unique->next; pl && pl->remaining_objects->size > 0;  pl = pl->next)
+			llist_sorted_difference_inplace(pl->remaining_objects, non_unique->remaining_objects);
 
 		non_unique = non_unique->next;
 	}
@@ -445,7 +445,7 @@ static void load_all_objects(void)
 
 	while (pl) {
 		hint = NULL;
-		l = pl->all_objects->front;
+		l = pl->remaining_objects->front;
 		while (l) {
 			hint = llist_insert_sorted_unique(all_objects,
 							  l->oid, hint);
@@ -456,7 +456,7 @@ static void load_all_objects(void)
 	/* remove objects present in remote packs */
 	pl = altodb_packs;
 	while (pl) {
-		llist_sorted_difference_inplace(all_objects, pl->all_objects);
+		llist_sorted_difference_inplace(all_objects, pl->remaining_objects);
 		pl = pl->next;
 	}
 }
@@ -481,8 +481,8 @@ static void scan_alt_odb_packs(void)
 	while (alt) {
 		local = local_packs;
 		while (local) {
-			llist_sorted_difference_inplace(local->all_objects,
-							alt->all_objects);
+			llist_sorted_difference_inplace(local->remaining_objects,
+							alt->remaining_objects);
 			local = local->next;
 		}
 		alt = alt->next;
@@ -499,7 +499,7 @@ static struct pack_list * add_pack(struct packed_git *p)
 		return NULL;
 
 	l.pack = p;
-	llist_init(&l.all_objects);
+	llist_init(&l.remaining_objects);
 
 	if (open_pack_index(p))
 		return NULL;
@@ -508,7 +508,7 @@ static struct pack_list * add_pack(struct packed_git *p)
 	base += 256 * 4 + ((p->index_version < 2) ? 4 : 8);
 	step = the_hash_algo->rawsz + ((p->index_version < 2) ? 4 : 0);
 	while (off < p->num_objects * step) {
-		llist_insert_back(l.all_objects, (const struct object_id *)(base + off));
+		llist_insert_back(l.remaining_objects, (const struct object_id *)(base + off));
 		off += step;
 	}
 	l.unique_objects = NULL;
@@ -605,7 +605,7 @@ int cmd_pack_redundant(int argc, const char **argv, const char *prefix)
 	llist_sorted_difference_inplace(all_objects, ignore);
 	pl = local_packs;
 	while (pl) {
-		llist_sorted_difference_inplace(pl->all_objects, ignore);
+		llist_sorted_difference_inplace(pl->remaining_objects, ignore);
 		pl = pl->next;
 	}
 
Signature · 1 line
2.20.1.103.ged0fc2ca7b
author
#21 lore raw

PATCH v10 6/6 pack-redundant: consistent sort method

In reply to Eric Sunshine

From: Jiang Xin <[email protected]>

SZEDER reported that test case t5323 has different test result on MacOS. This is because cmp_pack_list_reverse cannot give identical result when two pack being sorted has the same size of remaining_objects.

Changes to the sorting function will make consistent test result for t5323.

The new algorithm to find redundant packs is a trade-off to save memory resources, and the result of it may be different with old one, and may be not the best result sometimes. Update t5323 for the new algorithm.

---
 builtin/pack-redundant.c  | 24 ++++++++++++++++--------
 t/t5323-pack-redundant.sh | 18 +++++++++---------
 2 files changed, 25 insertions(+), 17 deletions(-)
builtin/pack-redundant.c +16 -8
index 15cdf233c4..29ff5e99cb 100644
@@ -33,6 +33,7 @@ static struct pack_list {
 	struct packed_git *pack;
 	struct llist *unique_objects;
 	struct llist *remaining_objects;
+	size_t all_objects_size;
 } *local_packs = NULL, *altodb_packs = NULL;
 
 static struct llist_item *free_nodes;
@@ -340,19 +341,25 @@ static inline off_t pack_set_bytecount(struct pack_list *pl)
 	return ret;
 }
 
-static int cmp_pack_list_reverse(const void *a, const void *b)
+static int cmp_remaining_objects(const void *a, const void *b)
 {
 	struct pack_list *pl_a = *((struct pack_list **)a);
 	struct pack_list *pl_b = *((struct pack_list **)b);
-	size_t sz_a = pl_a->remaining_objects->size;
-	size_t sz_b = pl_b->remaining_objects->size;
 
-	if (sz_a == sz_b)
-		return 0;
-	else if (sz_a < sz_b)
+	if (pl_a->remaining_objects->size == pl_b->remaining_objects->size) {
+		/* have the same remaining_objects, big pack first */
+		if (pl_a->all_objects_size == pl_b->all_objects_size)
+			return 0;
+		else if (pl_a->all_objects_size < pl_b->all_objects_size)
+			return 1;
+		else
+			return -1;
+	} else if (pl_a->remaining_objects->size < pl_b->remaining_objects->size) {
+		/* sort by remaining objects, more objects first */
 		return 1;
-	else
+	} else {
 		return -1;
+	}
 }
 
 /* Sort pack_list, greater size of remaining_objects first */
@@ -370,7 +377,7 @@ static void sort_pack_list(struct pack_list **pl)
 	for (n = 0, p = *pl; p; p = p->next)
 		ary[n++] = p;
 
-	QSORT(ary, n, cmp_pack_list_reverse);
+	QSORT(ary, n, cmp_remaining_objects);
 
 	/* link them back again */
 	for (i = 0; i < n - 1; i++)
@@ -511,6 +518,7 @@ static struct pack_list * add_pack(struct packed_git *p)
 		llist_insert_back(l.remaining_objects, (const struct object_id *)(base + off));
 		off += step;
 	}
+	l.all_objects_size = l.remaining_objects->size;
 	l.unique_objects = NULL;
 	if (p->pack_local)
 		return pack_list_insert(&local_packs, &l);
t/t5323-pack-redundant.sh +9 -9
index 3e62e8663f..384b244314 100755
@@ -165,15 +165,15 @@ test_expect_success 'master: no redundant for pack 1, 2, 3' '
 #         | T A B C D E F G H I J K L M N O P Q R
 #     ----+--------------------------------------
 #     P1  | x x x x x x x                       x
-#     P2* |     ! ! ! !   ! ! !
-#     P3  |             x     x x x x x
+#     P2  |     x x x x   x x x
+#     P3* |             !     ! ! ! ! !
 #     P4  |                     x x x x     x
 #     P5  |               x x           x x
 #     ----+--------------------------------------
 #     ALL | x x x x x x x x x x x x x x x x x   x
 #
 #############################################################################
-test_expect_failure 'master: one of pack-2/pack-3 is redundant (failed on Mac)' '
+test_expect_success 'master: one of pack-2/pack-3 is redundant' '
 	create_pack_in "$master_repo" P4 <<-EOF &&
 		$J
 		$K
@@ -190,7 +190,7 @@ test_expect_failure 'master: one of pack-2/pack-3 is redundant (failed on Mac)'
 	(
 		cd "$master_repo" &&
 		cat >expect <<-EOF &&
-			P2:$P2
+			P3:$P3
 			EOF
 		git pack-redundant --all >out &&
 		format_packfiles <out >actual &&
@@ -214,7 +214,7 @@ test_expect_failure 'master: one of pack-2/pack-3 is redundant (failed on Mac)'
 #     ALL | x x x x x x x x x x x x x x x x x x x
 #
 #############################################################################
-test_expect_failure 'master: pack 2, 4, and 6 are redundant (failed on Mac)' '
+test_expect_success 'master: pack 2, 4, and 6 are redundant' '
 	create_pack_in "$master_repo" P6 <<-EOF &&
 		$N
 		$O
@@ -254,7 +254,7 @@ test_expect_failure 'master: pack 2, 4, and 6 are redundant (failed on Mac)' '
 #     ALL | x x x x x x x x x x x x x x x x x x x
 #
 #############################################################################
-test_expect_failure 'master: pack-8 (subset of pack-1) is also redundant (failed on Mac)' '
+test_expect_success 'master: pack-8 (subset of pack-1) is also redundant' '
 	create_pack_in "$master_repo" P8 <<-EOF &&
 		$A
 		EOF
@@ -281,7 +281,7 @@ test_expect_success 'master: clean loose objects' '
 	)
 '
 
-test_expect_failure 'master: remove redundant packs and pass fsck (failed on Mac)' '
+test_expect_success 'master: remove redundant packs and pass fsck' '
 	(
 		cd "$master_repo" &&
 		git pack-redundant --all | xargs rm &&
@@ -301,7 +301,7 @@ test_expect_success 'setup shared.git' '
 	)
 '
 
-test_expect_failure 'shared: all packs are redundant, but no output without --alt-odb (failed on Mac)' '
+test_expect_success 'shared: all packs are redundant, but no output without --alt-odb' '
 	(
 		cd "$shared_repo" &&
 		git pack-redundant --all >out &&
@@ -334,7 +334,7 @@ test_expect_failure 'shared: all packs are redundant, but no output without --al
 #     ALL | x x x x x x x x x x x x x x x x x x x
 #
 #############################################################################
-test_expect_failure 'shared: show redundant packs in stderr for verbose mode (failed on Mac)' '
+test_expect_success 'shared: show redundant packs in stderr for verbose mode' '
 	(
 		cd "$shared_repo" &&
 		cat >expect <<-EOF &&
Signature · 1 line
2.20.1.103.ged0fc2ca7b