Skip to contentSkip to conversation

fetch --recurse-submodules: fetch unpopulated submodules

PATCH Superseded Glen Choo 6 participants 149 messages Feb 10, 2022 – Mar 16, 2022 Documentation Page 2 of 2

lore
Share & tools
lore
author
#115 lore raw

PATCH v4 7/10 submodule: extract get_fetch_task()

In reply to Glen Choo

get_next_submodule() configures the parallel submodule fetch by performing two functions:

* iterate the index to find submodules
* configure the child processes to fetch the submodules found in the previous step

Extract the index iterating code into an iterator function, get_fetch_task(), so that get_next_submodule() is agnostic of how to find submodules. This prepares for a subsequent commit will teach the fetch machinery to also iterate through the list of changed submodules (in addition to the index).

---
 submodule.c | 61 +++++++++++++++++++++++++++++++----------------------
 1 file changed, 36 insertions(+), 25 deletions(-)
submodule.c +36 -25
index 0b9c25f9d3..7a5316b6f7 100644
@@ -1389,6 +1389,7 @@ struct fetch_task {
 	struct repository *repo;
 	const struct submodule *sub;
 	unsigned free_sub : 1; /* Do we need to free the submodule? */
+	const char *default_argv; /* The default fetch mode. */
 
 	struct oid_array *commits; /* Ensure these commits are fetched */
 };
@@ -1466,14 +1467,11 @@ static struct repository *get_submodule_repo_for(struct repository *r,
 	return ret;
 }
 
-static int get_next_submodule(struct child_process *cp,
-			      struct strbuf *err, void *data, void **task_cb)
+static struct fetch_task *
+get_fetch_task(struct submodule_parallel_fetch *spf, struct strbuf *err)
 {
-	struct submodule_parallel_fetch *spf = data;
-
 	for (; spf->count < spf->r->index->cache_nr; spf->count++) {
 		const struct cache_entry *ce = spf->r->index->cache[spf->count];
-		const char *default_argv;
 		struct fetch_task *task;
 
 		if (!S_ISGITLINK(ce->ce_mode))
@@ -1493,10 +1491,10 @@ static int get_next_submodule(struct child_process *cp,
 					&spf->changed_submodule_names,
 					task->sub->name))
 				continue;
-			default_argv = "on-demand";
+			task->default_argv = "on-demand";
 			break;
 		case RECURSE_SUBMODULES_ON:
-			default_argv = "yes";
+			task->default_argv = "yes";
 			break;
 		case RECURSE_SUBMODULES_OFF:
 			continue;
@@ -1504,29 +1502,12 @@ static int get_next_submodule(struct child_process *cp,
 
 		task->repo = get_submodule_repo_for(spf->r, task->sub->path, null_oid());
 		if (task->repo) {
-			struct strbuf submodule_prefix = STRBUF_INIT;
-			child_process_init(cp);
-			cp->dir = task->repo->gitdir;
-			prepare_submodule_repo_env_in_gitdir(&cp->env_array);
-			cp->git_cmd = 1;
 			if (!spf->quiet)
 				strbuf_addf(err, _("Fetching submodule %s%s\n"),
 					    spf->prefix, ce->name);
-			strvec_init(&cp->args);
-			strvec_pushv(&cp->args, spf->args.v);
-			strvec_push(&cp->args, default_argv);
-			strvec_push(&cp->args, "--submodule-prefix");
-
-			strbuf_addf(&submodule_prefix, "%s%s/",
-						       spf->prefix,
-						       task->sub->path);
-			strvec_push(&cp->args, submodule_prefix.buf);
 
 			spf->count++;
-			*task_cb = task;
-
-			strbuf_release(&submodule_prefix);
-			return 1;
+			return task;
 		} else {
 			struct strbuf empty_submodule_path = STRBUF_INIT;
 
@@ -1550,6 +1531,36 @@ static int get_next_submodule(struct child_process *cp,
 			strbuf_release(&empty_submodule_path);
 		}
 	}
+	return NULL;
+}
+
+static int get_next_submodule(struct child_process *cp, struct strbuf *err,
+			      void *data, void **task_cb)
+{
+	struct submodule_parallel_fetch *spf = data;
+	struct fetch_task *task = get_fetch_task(spf, err);
+
+	if (task) {
+		struct strbuf submodule_prefix = STRBUF_INIT;
+
+		child_process_init(cp);
+		cp->dir = task->repo->gitdir;
+		prepare_submodule_repo_env_in_gitdir(&cp->env_array);
+		cp->git_cmd = 1;
+		strvec_init(&cp->args);
+		strvec_pushv(&cp->args, spf->args.v);
+		strvec_push(&cp->args, task->default_argv);
+		strvec_push(&cp->args, "--submodule-prefix");
+
+		strbuf_addf(&submodule_prefix, "%s%s/",
+						spf->prefix,
+						task->sub->path);
+		strvec_push(&cp->args, submodule_prefix.buf);
+		*task_cb = task;
+
+		strbuf_release(&submodule_prefix);
+		return 1;
+	}
 
 	if (spf->oid_fetch_tasks_nr) {
 		struct fetch_task *task =
Signature · 1 line
2.33.GIT
author
#116 lore raw

PATCH v4 8/10 submodule: move logic into fetch_task_create()

In reply to Glen Choo

get_fetch_task() gets a fetch task by iterating the index; a future commit will introduce a similar function, get_fetch_task_from_changed(), that gets a fetch task from the list of changed submodules. Both functions are similar in that they need to:

* create a fetch task
* initialize the submodule repo for the fetch task
* determine the default recursion mode

Move all of this logic into fetch_task_create() so that it is no longer split between fetch_task_create() and get_fetch_task(). This will make it easier to share code with get_fetch_task_from_changed().

---
 submodule.c | 99 ++++++++++++++++++++++++++++-------------------------
 1 file changed, 52 insertions(+), 47 deletions(-)
submodule.c +52 -47
index 7a5316b6f7..b36ef26752 100644
@@ -1415,32 +1415,6 @@ static const struct submodule *get_non_gitmodules_submodule(const char *path)
 	return (const struct submodule *) ret;
 }
 
-static struct fetch_task *fetch_task_create(struct repository *r,
-					    const char *path,
-					    const struct object_id *treeish_name)
-{
-	struct fetch_task *task = xmalloc(sizeof(*task));
-	memset(task, 0, sizeof(*task));
-
-	task->sub = submodule_from_path(r, treeish_name, path);
-	if (!task->sub) {
-		/*
-		 * No entry in .gitmodules? Technically not a submodule,
-		 * but historically we supported repositories that happen to be
-		 * in-place where a gitlink is. Keep supporting them.
-		 */
-		task->sub = get_non_gitmodules_submodule(path);
-		if (!task->sub) {
-			free(task);
-			return NULL;
-		}
-
-		task->free_sub = 1;
-	}
-
-	return task;
-}
-
 static void fetch_task_release(struct fetch_task *p)
 {
 	if (p->free_sub)
@@ -1467,6 +1441,57 @@ static struct repository *get_submodule_repo_for(struct repository *r,
 	return ret;
 }
 
+static struct fetch_task *fetch_task_create(struct submodule_parallel_fetch *spf,
+					    const char *path,
+					    const struct object_id *treeish_name)
+{
+	struct fetch_task *task = xmalloc(sizeof(*task));
+	memset(task, 0, sizeof(*task));
+
+	task->sub = submodule_from_path(spf->r, treeish_name, path);
+
+	if (!task->sub) {
+		/*
+		 * No entry in .gitmodules? Technically not a submodule,
+		 * but historically we supported repositories that happen to be
+		 * in-place where a gitlink is. Keep supporting them.
+		 */
+		task->sub = get_non_gitmodules_submodule(path);
+		if (!task->sub)
+			goto cleanup;
+
+		task->free_sub = 1;
+	}
+
+	switch (get_fetch_recurse_config(task->sub, spf))
+	{
+	default:
+	case RECURSE_SUBMODULES_DEFAULT:
+	case RECURSE_SUBMODULES_ON_DEMAND:
+		if (!task->sub ||
+			!string_list_lookup(
+				&spf->changed_submodule_names,
+				task->sub->name))
+			goto cleanup;
+		task->default_argv = "on-demand";
+		break;
+	case RECURSE_SUBMODULES_ON:
+		task->default_argv = "yes";
+		break;
+	case RECURSE_SUBMODULES_OFF:
+		goto cleanup;
+	}
+
+	task->repo = get_submodule_repo_for(spf->r, path, treeish_name);
+
+	return task;
+
+ cleanup:
+	fetch_task_release(task);
+	free(task);
+	return NULL;
+}
+
 static struct fetch_task *
 get_fetch_task(struct submodule_parallel_fetch *spf, struct strbuf *err)
 {
@@ -1477,30 +1502,10 @@ get_fetch_task(struct submodule_parallel_fetch *spf, struct strbuf *err)
 		if (!S_ISGITLINK(ce->ce_mode))
 			continue;
 
-		task = fetch_task_create(spf->r, ce->name, null_oid());
+		task = fetch_task_create(spf, ce->name, null_oid());
 		if (!task)
 			continue;
 
-		switch (get_fetch_recurse_config(task->sub, spf))
-		{
-		default:
-		case RECURSE_SUBMODULES_DEFAULT:
-		case RECURSE_SUBMODULES_ON_DEMAND:
-			if (!task->sub ||
-			    !string_list_lookup(
-					&spf->changed_submodule_names,
-					task->sub->name))
-				continue;
-			task->default_argv = "on-demand";
-			break;
-		case RECURSE_SUBMODULES_ON:
-			task->default_argv = "yes";
-			break;
-		case RECURSE_SUBMODULES_OFF:
-			continue;
-		}
-
-		task->repo = get_submodule_repo_for(spf->r, task->sub->path, null_oid());
 		if (task->repo) {
 			if (!spf->quiet)
 				strbuf_addf(err, _("Fetching submodule %s%s\n"),
Signature · 1 line
2.33.GIT
author
#117 lore raw

PATCH v4 9/10 fetch: fetch unpopulated, changed submodules

In reply to Glen Choo

"git fetch --recurse-submodules" only considers populated submodules (i.e. submodules that can be found by iterating the index), which makes "git fetch" behave differently based on which commit is checked out. As a result, even if the user has initialized all submodules correctly, they may not fetch the necessary submodule commits, and commands like "git checkout --recurse-submodules" might fail.

Teach "git fetch" to fetch cloned, changed submodules regardless of whether they are populated. This is in addition to the current behavior of fetching populated submodules (which is always attempted regardless of what was fetched in the superproject, or even if nothing was fetched in the superproject).

A submodule may be encountered multiple times (via the list of populated submodules or via the list of changed submodules). When this happens, "git fetch" only reads the 'populated copy' and ignores the 'changed copy'. Amend the verify_fetch_result() test helper so that we can assert on which 'copy' is being read.

---
 Documentation/fetch-options.txt |  26 ++--
 Documentation/git-fetch.txt     |  10 +-
 builtin/fetch.c                 |  14 +-
 submodule.c                     | 178 +++++++++++++++++++--
 submodule.h                     |  12 +-
 t/t5526-fetch-submodules.sh     | 263 +++++++++++++++++++++++++++++++-
 6 files changed, 457 insertions(+), 46 deletions(-)
Documentation/fetch-options.txt +17 -9
index f903683189..6cdd9d43c5 100644
@@ -186,15 +186,23 @@ endif::git-pull[]
 ifndef::git-pull[]
 --recurse-submodules[=yes|on-demand|no]::
 	This option controls if and under what conditions new commits of
-	populated submodules should be fetched too. It can be used as a
-	boolean option to completely disable recursion when set to 'no' or to
-	unconditionally recurse into all populated submodules when set to
-	'yes', which is the default when this option is used without any
-	value. Use 'on-demand' to only recurse into a populated submodule
-	when the superproject retrieves a commit that updates the submodule's
-	reference to a commit that isn't already in the local submodule
-	clone. By default, 'on-demand' is used, unless
-	`fetch.recurseSubmodules` is set (see linkgit:git-config[1]).
+	submodules should be fetched too. When recursing through submodules,
+	`git fetch` always attempts to fetch "changed" submodules, that is, a
+	submodule that has commits that are referenced by a newly fetched
+	superproject commit but are missing in the local submodule clone. A
+	changed submodule can be fetched as long as it is present locally e.g.
+	in `$GIT_DIR/modules/` (see linkgit:gitsubmodules[7]); if the upstream
+	adds a new submodule, that submodule cannot be fetched until it is
+	cloned e.g. by `git submodule update`.
++
+When set to 'on-demand', only changed submodules are fetched. When set
+to 'yes', all populated submodules are fetched and submodules that are
+both unpopulated and changed are fetched. When set to 'no', submodules
+are never fetched.
++
+When unspecified, this uses the value of `fetch.recurseSubmodules` if it
+is set (see linkgit:git-config[1]), defaulting to 'on-demand' if unset.
+When this option is used without any value, it defaults to 'yes'.
 endif::git-pull[]
 
 -j::
Documentation/git-fetch.txt +4 -6
index 550c16ca61..e9d364669a 100644
@@ -287,12 +287,10 @@ include::transfer-data-leaks.txt[]
 
 BUGS
 ----
-Using --recurse-submodules can only fetch new commits in already checked
-out submodules right now. When e.g. upstream added a new submodule in the
-just fetched commits of the superproject the submodule itself cannot be
-fetched, making it impossible to check out that submodule later without
-having to do a fetch again. This is expected to be fixed in a future Git
-version.
+Using --recurse-submodules can only fetch new commits in submodules that are
+present locally e.g. in `$GIT_DIR/modules/`. If the upstream adds a new
+submodule, that submodule cannot be fetched until it is cloned e.g. by `git
+submodule update`. This is expected to be fixed in a future Git version.
 
 SEE ALSO
 --------
builtin/fetch.c +7 -7
index 95832ba1df..97a89763c8 100644
@@ -2178,13 +2178,13 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
 			max_children = fetch_parallel_config;
 
 		add_options_to_argv(&options);
-		result = fetch_populated_submodules(the_repository,
-						    &options,
-						    submodule_prefix,
-						    recurse_submodules,
-						    recurse_submodules_default,
-						    verbosity < 0,
-						    max_children);
+		result = fetch_submodules(the_repository,
+					  &options,
+					  submodule_prefix,
+					  recurse_submodules,
+					  recurse_submodules_default,
+					  verbosity < 0,
+					  max_children);
 		strvec_clear(&options);
 	}
 
submodule.c +162 -16

289 lines not on this page · raw message

submodule.h +6 -6
index 784ceffc0e..61bebde319 100644
@@ -88,12 +88,12 @@ int should_update_submodules(void);
  */
 const struct submodule *submodule_from_ce(const struct cache_entry *ce);
 void check_for_new_submodule_commits(struct object_id *oid);
-int fetch_populated_submodules(struct repository *r,
-			       const struct strvec *options,
-			       const char *prefix,
-			       int command_line_option,
-			       int default_option,
-			       int quiet, int max_parallel_jobs);
+int fetch_submodules(struct repository *r,
+		     const struct strvec *options,
+		     const char *prefix,
+		     int command_line_option,
+		     int default_option,
+		     int quiet, int max_parallel_jobs);
 unsigned is_submodule_modified(const char *path, int ignore_untracked);
 int submodule_uses_gitfile(const char *path);
 
t/t5526-fetch-submodules.sh +261 -2

293 lines not on this page · raw message

Signature · 1 line
2.33.GIT
#118 lore raw

Re: [PATCH v4 09/10] fetch: fetch unpopulated, changed submodules

submodule.c static void collect_changed_submodules_cb(struct diff_queue_struct *q, · line 889

Quoted 11 lines · Glen Choo
submodule.c +7 -2
 		item = string_list_insert(changed, name);
-		if (!item->util)
+		if (item->util)
+			cs_data = item->util;
+		else {
 			item->util = xcalloc(1, sizeof(struct changed_submodule_data));
-		cs_data = item->util;
+			cs_data = item->util;
+			cs_data->super_oid = commit_oid;
+			cs_data->path = xstrdup(p->two->path);
+		}

I do not quite get this change.

collect_changed_submodules() walks a range of revisions in the superproject, doing an equivalent of "git log --raw" and feeding the differences to this callback function. The above code looks at the path and uses the "changed" string list to record which submodule was modified, what commit in the submodule is needed, etc.

What happens when the range has more than one change to the same submodule? cs_data has only one room for recording .super_oid (which commit in the superproject touches the submodule) and .path (where in the superproject's tree the submodule exists). "git mv" of a submodule might be rare and it may not hurt too much that only a single .path can be kept, but it looks somewhat iffy.

Quoted 1 line

oid_array_append(&cs_data->new_commits, &p->two->oid);

At least, we are not losing any submodule commit even when the same submodule is touched more than once by the superproject, but it is dubious why we have cs_data.super_oid and cs_data.path in the first place.

How are they used, or are they something that seemed useful when the code was first written but it turned out that they weren't and left unused?

Or do we need to make cs_data an array of 3-tuple { .super_oid, .submodule_oid, .path } for each submodule name?

author
#119 lore raw

Re: [PATCH v4 09/10] fetch: fetch unpopulated, changed submodules

Quoted 22 lines · Junio C Hamano

submodule.c static void collect_changed_submodules_cb(struct diff_queue_struct *q, · line 889

Quoted 11 lines · Glen Choo
submodule.c +7 -2
 		item = string_list_insert(changed, name);
-		if (!item->util)
+		if (item->util)
+			cs_data = item->util;
+		else {
 			item->util = xcalloc(1, sizeof(struct changed_submodule_data));
-		cs_data = item->util;
+			cs_data = item->util;
+			cs_data->super_oid = commit_oid;
+			cs_data->path = xstrdup(p->two->path);
+		}

I do not quite get this change.

collect_changed_submodules() walks a range of revisions in the superproject, doing an equivalent of "git log --raw" and feeding the differences to this callback function. The above code looks at the path and uses the "changed" string list to record which submodule was modified, what commit in the submodule is needed, etc.

Quoted 6 lines

What happens when the range has more than one change to the same submodule? cs_data has only one room for recording .super_oid (which commit in the superproject touches the submodule) and .path (where in the superproject's tree the submodule exists). "git mv" of a submodule might be rare and it may not hurt too much that only a single .path can be kept, but it looks somewhat iffy.

Yes, I agree that it looks odd, which is why I added this comment to hopefully make it less opaque:

+ * (super_oid, path) allows the submodule config to be read from _some_
+ * .gitmodules file. We store this information the first time we find a
+ * superproject commit that points to the submodule, but this is
+ * arbitrary - we can choose any (super_oid, path) that matches the
+ * submodule's name.

I guess this only says that it is ok to store .super_oid and .path from any commit, but doesn't go in depth into _why_. It's ok because we only need (.super_oid, .path) because repo_submodule_init(..., path, treeish_name) maps these args to the submodule's name and gitdir (i.e. .git/modules/<name>).

This means we don't worry about 'git mv' (super_oid's .gitmodules will tell us the correct name even if the path changed relative to some other commit), nor seeing the submodule more than once (it doesn't matter whose .gitmodules we look at so long as repo_submodule_init() derives the correct gitdir).

And now that you've pointed this out, I realize that we could do away with (.super_oid, .path) altogether if we had a variant of repo_submodule_init() that takes the submodule name instead of (path, treeish_name). (We have a similar submodule_from_name(), but that only reads the submodule config, not a struct repository.) I would prefer not to introduce such a function so late into the review cycle, but I could clean this up later.

Quoted 6 lines
Quoted 1 line

oid_array_append(&cs_data->new_commits, &p->two->oid);

At least, we are not losing any submodule commit even when the same submodule is touched more than once by the superproject, but it is dubious why we have cs_data.super_oid and cs_data.path in the first place.

On the hand, we actually need to record every submodule commit, so yes.

Quoted 6 lines

How are they used, or are they something that seemed useful when the code was first written but it turned out that they weren't and left unused?

Or do we need to make cs_data an array of 3-tuple { .super_oid, .submodule_oid, .path } for each submodule name?

To conclude:

- The changed_submodules string_list is basically a map that tells us, for a given submodule _name_, which commits we need to fetch and where repo_submodule_init() can read the submodule name from.
- We only use cs_data as a string_list_item.util, and the string_list_item.string is the submodule name itself.
- .new_commits tells us which commits to fetch.
- .super_oid and .path tells repo_submodule_init() how to get the name of the submodule.

So we don't need to make this a 3-tuple.

#120 lore raw

Re: [PATCH v4 09/10] fetch: fetch unpopulated, changed submodules

Quoted 7 lines · Glen Choo

And now that you've pointed this out, I realize that we could do away with (.super_oid, .path) altogether if we had a variant of repo_submodule_init() that takes the submodule name instead of (path, treeish_name). (We have a similar submodule_from_name(), but that only reads the submodule config, not a struct repository.) I would prefer not to introduce such a function so late into the review cycle, but I could clean this up later.

I am puzzled. What do you exactly mean by "late into the review cycle"?

Quoted 10 lines

- The changed_submodules string_list is basically a map that tells us, for a given submodule _name_, which commits we need to fetch and where repo_submodule_init() can read the submodule name from.
- We only use cs_data as a string_list_item.util, and the string_list_item.string is the submodule name itself.
- .new_commits tells us which commits to fetch.
- .super_oid and .path tells repo_submodule_init() how to get the name of the submodule.

So we don't need to make this a 3-tuple.

OK. We need to learn in which local repository houses the submodule we discover in cs_data resides. It may or may not have a checkout in the current checkout of the superorject commit. And just one <.super_oid, .path> tuple should be sufficient to tell us that, because the mapping from submodule name to path may change as "git mv" moves it around, but the mapping from submodule name to where the submodule repository is stored in the .git/ directory of the superproject should not change. Am I following you so far correctly?

I am wondering if we need even one <.super_oid, .path> tuple. Looking at the implementation of repo_submodule_init(), I have a feeling that a version of "initialize named submodule in a given tree-ish in the superproject" would be rather trivial. We already have submodule name, so submodule_name_to_gitdir() would be all we need, no? After all, we are only interested in fetching objects to fill missing commits (and possibly update the remote tracking branches) and do not care about touching its working tree. And once we learn that .git/modules/<name>/ directory, we can fetch the necessary commits into it, right?

Or am I oversimplifying the problem?

author
#121 lore raw

Re: [PATCH v4 09/10] fetch: fetch unpopulated, changed submodules

Quoted 10 lines · Junio C Hamano
Quoted 7 lines · Glen Choo

And now that you've pointed this out, I realize that we could do away with (.super_oid, .path) altogether if we had a variant of repo_submodule_init() that takes the submodule name instead of (path, treeish_name). (We have a similar submodule_from_name(), but that only reads the submodule config, not a struct repository.) I would prefer not to introduce such a function so late into the review cycle, but I could clean this up later.

Quoted 2 lines

I am puzzled. What do you exactly mean by "late into the review cycle"?

I mean that reviewers have already seen several iterations of this, and I'm afraid that a refactor might introduce unnecessary cognitive overhead.

But of course, we might decide that the refactor is a good enough idea that we want to do it anyway :)

Quoted 11 lines
Quoted 10 lines

- The changed_submodules string_list is basically a map that tells us, for a given submodule _name_, which commits we need to fetch and where repo_submodule_init() can read the submodule name from.
- We only use cs_data as a string_list_item.util, and the string_list_item.string is the submodule name itself.
- .new_commits tells us which commits to fetch.
- .super_oid and .path tells repo_submodule_init() how to get the name of the submodule.

So we don't need to make this a 3-tuple.

Quoted 9 lines

OK. We need to learn in which local repository houses the submodule we discover in cs_data resides. It may or may not have a checkout in the current checkout of the superorject commit. And just one <.super_oid, .path> tuple should be sufficient to tell us that, because the mapping from submodule name to path may change as "git mv" moves it around, but the mapping from submodule name to where the submodule repository is stored in the .git/ directory of the superproject should not change. Am I following you so far correctly?

Yes, that's correct.

Quoted 11 lines

I am wondering if we need even one <.super_oid, .path> tuple. Looking at the implementation of repo_submodule_init(), I have a feeling that a version of "initialize named submodule in a given tree-ish in the superproject" would be rather trivial. We already have submodule name, so submodule_name_to_gitdir() would be all we need, no? After all, we are only interested in fetching objects to fill missing commits (and possibly update the remote tracking branches) and do not care about touching its working tree. And once we learn that .git/modules/<name>/ directory, we can fetch the necessary commits into it, right?

Quoted 1 line

Or am I oversimplifying the problem?

I don't think you are oversimplifying. Now that I look at it again, it really _does_ seem trivial. Doing this refactor saves me the headache of explaining why we need a single <.super_oid, .path> tuple, and saves readers the headache of figuring out if I'm right.

I'll try it and see if it really makes things simpler or not.

#122 lore raw

Re: [PATCH v4 09/10] fetch: fetch unpopulated, changed submodules

Quoted 8 lines · Glen Choo
Quoted 1 line

Or am I oversimplifying the problem?

I don't think you are oversimplifying. Now that I look at it again, it really _does_ seem trivial. Doing this refactor saves me the headache of explaining why we need a single <.super_oid, .path> tuple, and saves readers the headache of figuring out if I'm right.

I'll try it and see if it really makes things simpler or not.

Thanks.

#123 lore raw

Re: [PATCH v4 09/10] fetch: fetch unpopulated, changed submodules

In reply to Glen Choo
Quoted 18 lines · Glen Choo
submodule.c +18 -0
+		/*
+		 * NEEDSWORK: Submodules set/unset a value for
+		 * core.worktree when they are populated/unpopulated by
+		 * "git checkout" (and similar commands, see
+		 * submodule_move_head() and
+		 * connect_work_tree_and_git_dir()), but if the
+		 * submodule is unpopulated in another way (e.g. "git
+		 * rm", "rm -r"), core.worktree will still be set even
+		 * though the directory doesn't exist, and the child
+		 * process will crash while trying to chdir into the
+		 * nonexistent directory.
+		 *
+		 * In this case, we know that the submodule has no
+		 * working tree, so we can work around this by
+		 * setting "--work-tree=." (--bare does not work because
+		 * worktree settings take precedence over bare-ness).
+		 * However, this is not necessarily true in other cases,
+		 * so a generalized solution is still necessary.

submodule.c get_fetch_task(struct submodule_parallel_fetch *spf, struct strbuf *err) · line 1677

Quoted 10 lines
submodule.c +10 -0
+		 *
+		 * Possible solutions:
+		 * - teach "git [add|rm]" to unset core.worktree and
+		 *   discourage users from removing submodules without
+		 *   using a Git command.
+		 * - teach submodule child processes to ignore stale
+		 *   core.worktree values.
+		 */
+		strvec_push(&task->git_args, "--work-tree=.");
+		return task;

Thanks - this is a good comment.

I've also written other comments on this patch that are easier to describe in the context of an earlier conversation, so I've written these as replies to an earlier email:

https://lore.kernel.org/git/[email protected]/
https://lore.kernel.org/git/[email protected]/

author
#124 lore raw

PATCH v4 10/10 submodule: fix latent check_has_commit() bug

In reply to Glen Choo

When check_has_commit() is called on a missing submodule, initialization of the struct repository fails, but it attempts to clear the struct anyway (which is a fatal error). This bug is masked by its only caller, submodule_has_commits(), first calling add_submodule_odb(). The latter fails if the submodule does not exist, making submodule_has_commits() exit early and not invoke check_has_commit().

Fix this bug, and because calling add_submodule_odb() is no longer necessary as of 13a2f620b2 (submodule: pass repo to check_has_commit(), 2021-10-08), remove that call too.

This is the last caller of add_submodule_odb(), so remove that function. (Submodule ODBs are still added as alternates via add_submodule_odb_by_path().)

---
 submodule.c | 35 ++---------------------------------
 submodule.h |  9 ++++-----
 2 files changed, 6 insertions(+), 38 deletions(-)
submodule.c +2 -33
index 1f5f39ce18..6e6b2d04e4 100644
@@ -167,26 +167,6 @@ void stage_updated_gitmodules(struct index_state *istate)
 
 static struct string_list added_submodule_odb_paths = STRING_LIST_INIT_NODUP;
 
-/* TODO: remove this function, use repo_submodule_init instead. */
-int add_submodule_odb(const char *path)
-{
-	struct strbuf objects_directory = STRBUF_INIT;
-	int ret = 0;
-
-	ret = strbuf_git_path_submodule(&objects_directory, path, "objects/");
-	if (ret)
-		goto done;
-	if (!is_directory(objects_directory.buf)) {
-		ret = -1;
-		goto done;
-	}
-	string_list_insert(&added_submodule_odb_paths,
-			   strbuf_detach(&objects_directory, NULL));
-done:
-	strbuf_release(&objects_directory);
-	return ret;
-}
-
 void add_submodule_odb_by_path(const char *path)
 {
 	string_list_insert(&added_submodule_odb_paths, xstrdup(path));
@@ -971,7 +951,8 @@ static int check_has_commit(const struct object_id *oid, void *data)
 
 	if (repo_submodule_init(&subrepo, cb->repo, cb->path, cb->super_oid)) {
 		cb->result = 0;
-		goto cleanup;
+		/* subrepo failed to init, so don't clean it up. */
+		return 0;
 	}
 
 	type = oid_object_info(&subrepo, oid, NULL);
@@ -1007,18 +988,6 @@ static int submodule_has_commits(struct repository *r,
 		.super_oid = super_oid
 	};
 
-	/*
-	 * Perform a cheap, but incorrect check for the existence of 'commits'.
-	 * This is done by adding the submodule's object store to the in-core
-	 * object store, and then querying for each commit's existence.  If we
-	 * do not have the commit object anywhere, there is no chance we have
-	 * it in the object store of the correct submodule and have it
-	 * reachable from a ref, so we can fail early without spawning rev-list
-	 * which is expensive.
-	 */
-	if (add_submodule_odb(path))
-		return 0;
-
 	oid_array_for_each_unique(commits, check_has_commit, &has_commit);
 
 	if (has_commit.result) {
submodule.h +4 -5
index 61bebde319..40c1445237 100644
@@ -103,12 +103,11 @@ int submodule_uses_gitfile(const char *path);
 int bad_to_remove_submodule(const char *path, unsigned flags);
 
 /*
- * Call add_submodule_odb() to add the submodule at the given path to a list.
- * When register_all_submodule_odb_as_alternates() is called, the object stores
- * of all submodules in that list will be added as alternates in
- * the_repository.
+ * Call add_submodule_odb_by_path() to add the submodule at the given
+ * path to a list. When register_all_submodule_odb_as_alternates() is
+ * called, the object stores of all submodules in that list will be
+ * added as alternates in the_repository.
  */
-int add_submodule_odb(const char *path);
 void add_submodule_odb_by_path(const char *path);
 int register_all_submodule_odb_as_alternates(void);
 
Signature · 1 line
2.33.GIT
#125 lore raw

Re: [PATCH v4 10/10] submodule: fix latent check_has_commit() bug

Quoted 15 lines · Glen Choo

When check_has_commit() is called on a missing submodule, initialization of the struct repository fails, but it attempts to clear the struct anyway (which is a fatal error). This bug is masked by its only caller, submodule_has_commits(), first calling add_submodule_odb(). The latter fails if the submodule does not exist, making submodule_has_commits() exit early and not invoke check_has_commit().

Fix this bug, and because calling add_submodule_odb() is no longer necessary as of 13a2f620b2 (submodule: pass repo to check_has_commit(), 2021-10-08), remove that call too.

This is the last caller of add_submodule_odb(), so remove that function. (Submodule ODBs are still added as alternates via add_submodule_odb_by_path().)

Quoted 5 lines
---
 submodule.c | 35 ++---------------------------------
 submodule.h |  9 ++++-----
 2 files changed, 6 insertions(+), 38 deletions(-)

Looks reasonable. Will queue.

Thanks.

#126 lore raw
In reply to Glen Choo
Quoted 3 lines · Glen Choo

== Since v3
- Numerous style fixes + improved comments.
- Fix sed portability issues.

Good.

Quoted 1 line

- Fix failing test due to default branch name assumptions.

OK. Sprinkling "-b main" all over to "git init" makes a bit noisy patch, but the conversion looks good.

author
#127 lore raw

v5 cover fetch --recurse-submodules: fetch unpopulated submodules

In reply to Glen Choo

Original cover letter: https://lore.kernel.org/git/[email protected]

Based off 'master'.

Thanks as always for the kind feedback :) I believe I incorporated all of feedback from the previous round except for the following:

- <[email protected]> In a recursive fetch, we thought that we could use the submodule name instead of the superproject commit oid and path.

It's true that we don't need <.super_oid, .path> in order to init the subrepo, but it turns out that recursive fetch reads some configuration values from .gitmodules (via submodule_from_path()), so we still need to store super_oid in order to read the correct .gitmodules file.

- <[email protected]> I've described the differences between the no-submodule-in-index test and the other-submodule-in-index test (their comments now refer to one another, so the contrast is more obvious), but didn't reorder them because I thought that made the test setup less intuitive to read.

- <[email protected]> I added expect.err.sub2 to verify_test_result() but didn't change write_expected_super() to account for sub2. It turned out to be tricky to predict the output when 'super' fetches >1 branch because each fetched branch can affect the formatting. e.g.

    	   OLD_HEAD..super  super           -> origin/super

can become

    	   OLD_HEAD..super  super                   -> origin/super
    	   OLD_HEAD..super  some-other-branch       -> origin/some-other-branch

(I could work around this by replacing the whitespace with sed, but it seemed like too much overhead for a one-off test).

= Patch organization

- Patches 1-3 are quality-of-life improvements to the test suite that make it easier to write the tests in patch 9.
- Patches 4-6 are preparation for "git fetch" to read .gitmodules from the superproject commit in patch 7.
- Patches 7-8 refactor out the logic of "finding which submodules to fetch" and "fetching the submodules", making it easier to tell "git fetch" to fetch unpopulated submodules.
- Patch 9 teaches "git fetch" to fetch changed, unpopulated submodules in addition to populated submodules.
- Patch 10 is an optional bugfix + cleanup of the "git fetch" code that removes the last caller of the deprecated "add_submodule_odb()".

= Changes

== Since v4
- Rename test helpers (s/check_/write_expected_)
- Test style fixes
- Update test comments
- Remove the manual test_cmp in the test that checks sub2 (but we still construct expect.err.super manually).

== Since v3
- Numerous style fixes + improved comments.
- Fix sed portability issues.
- Fix failing test due to default branch name assumptions.
- Patch 3: change a test so that it no longer depends on state from the previous test.
- Patch 9: fix memory leak when recording super_oid and path + add explanatory comment.

== Since v2
- Numerous small fixes to the code and commit message (thanks to all who helped spot these :))
- In patch 2, use test_cmp + sed to assert on test output, effectively reverting the "use grep" approach of v1-2 (see patch 2's description).
- New patch 3: introduce a test helper that creates the expected superproject commit (instead of copy-pasting the code over and over).
- I did not get rid of "git fetch" inside the test helper (as Jonathan suggested) though, because that requires a bigger change in the test setup, and I think the test helper makes the test straightforward enough.
- New patch 8: refactor some shared logic out into fetch_task_create(). This reduces code duplication between the get_fetch_task_from_* functions.
- In patch 9, add additional tests for 'submodules with the same name'.
- In patch 9, handle a bug where a submodule that is unpopulated by "git rm" still has "core.worktree" set and cannot be fetched (see patch 9's description).
- Remove the "git fetch --update-shallow" patch (I'll try to send it separately).

== Since v1
- Numerous style fixes suggested by Jonathan (thanks!)
- In patch 3, don't prematurely read submodules from the superproject commit (see:
<[email protected]>).
- In patch 7, stop using "git checkout" and "! grep" in tests.
- In patch 7, stop doing the "find changed submodules" rev walk unconditionally. Instead, continue to check for .gitmodules, but also check for submodules in $GIT_DIR/modules.
- I'm not entirely happy with the helper function name, see "---" for details.
- Move "git fetch --update-shallow" bugfix to patch 8.
- Because the "find changed submodules" rev walk is no longer unconditional, this fix is no longer needed for tests to pass.
- Rename fetch_populated_submodules() to fetch_submodules().

Glen Choo (10):
t5526: introduce test helper to assert on fetches
t5526: stop asserting on stderr literally
t5526: create superproject commits with test helper
submodule: make static functions read submodules from commits
submodule: inline submodule_commits() into caller
submodule: store new submodule commits oid_array in a struct
submodule: extract get_fetch_task()
submodule: move logic into fetch_task_create()
fetch: fetch unpopulated, changed submodules
submodule: fix latent check_has_commit() bug

 Documentation/fetch-options.txt |  26 +-
 Documentation/git-fetch.txt     |  10 +-
 builtin/fetch.c                 |  14 +-
 submodule.c                     | 442 +++++++++++++++++---------
 submodule.h                     |  21 +-
 t/t5526-fetch-submodules.sh     | 545 ++++++++++++++++++++++++--------
 6 files changed, 746 insertions(+), 312 deletions(-)

Range-diff against v4:
1: 57cd31afc2 ! 1: f22f992e2b t5526: introduce test helper to assert on fetches

    @@ t/t5526-fetch-submodules.sh: add_upstream_commit() {
     +#
     +# If a repo should not be fetched in the test, its corresponding
     +# expect.err file should be rm-ed.
    -+verify_fetch_result() {
    ++verify_fetch_result () {
     +	ACTUAL_ERR=$1 &&
     +	rm -f expect.err.combined &&
     +	if test -f expect.err.super
 2:  b70c894cff !  2:  f6ee125e16 t5526: stop asserting on stderr literally
    @@ Commit message
     
         Stop asserting on $head1 by replacing it with a dummy value in the
         actual and expected output. Do this by introducing test
    -    helpers (check_*()) that make it easier to construct the expected
    -    output, and use sed to munge the actual output.
    +    helpers (write_expected_*()) that make it easier to construct the
    +    expected output, and use sed to munge the actual output.
     
         Signed-off-by: Glen Choo <[email protected]>
     
    @@ t/t5526-fetch-submodules.sh: export GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB
      
      pwd=$(pwd)
      
    -+check_sub () {
    ++write_expected_sub () {
     +	NEW_HEAD=$1 &&
    -+	cat >$pwd/expect.err.sub <<-EOF
    ++	cat >"$pwd/expect.err.sub" <<-EOF
     +	Fetching submodule submodule
     +	From $pwd/submodule
     +	   OLD_HEAD..$NEW_HEAD  sub        -> origin/sub
     +	EOF
     +}
     +
    -+check_deep () {
    ++write_expected_deep () {
     +	NEW_HEAD=$1 &&
    -+	cat >$pwd/expect.err.deep <<-EOF
    ++	cat >"$pwd/expect.err.deep" <<-EOF
     +	Fetching submodule submodule/subdir/deepsubmodule
     +	From $pwd/deepsubmodule
     +	   OLD_HEAD..$NEW_HEAD  deep       -> origin/deep
     +	EOF
     +}
     +
    -+check_super () {
    ++write_expected_super () {
     +	NEW_HEAD=$1 &&
    -+	cat >$pwd/expect.err.super <<-EOF
    ++	cat >"$pwd/expect.err.super" <<-EOF
     +	From $pwd/.
     +	   OLD_HEAD..$NEW_HEAD  super      -> origin/super
     +	EOF
    @@ t/t5526-fetch-submodules.sh: pwd=$(pwd)
     -		echo "From $pwd/submodule" >> ../expect.err.sub &&
     -		echo "   $head1..$head2  sub        -> origin/sub" >> ../expect.err.sub
     +		new_head=$(git rev-parse --short HEAD) &&
    -+		check_sub $new_head
    ++		write_expected_sub $new_head
      	) &&
      	(
      		cd deepsubmodule &&
    @@ t/t5526-fetch-submodules.sh: pwd=$(pwd)
     -		echo "From $pwd/deepsubmodule" >> ../expect.err.deep &&
     -		echo "   $head1..$head2  deep       -> origin/deep" >> ../expect.err.deep
     +		new_head=$(git rev-parse --short HEAD) &&
    -+		check_deep $new_head
    ++		write_expected_deep $new_head
      	)
      }
      
    -@@ t/t5526-fetch-submodules.sh: add_upstream_commit() {
    - #
    - # If a repo should not be fetched in the test, its corresponding
    - # expect.err file should be rm-ed.
    --verify_fetch_result() {
    -+verify_fetch_result () {
    - 	ACTUAL_ERR=$1 &&
    - 	rm -f expect.err.combined &&
    - 	if test -f expect.err.super
    -@@ t/t5526-fetch-submodules.sh: verify_fetch_result() {
    +@@ t/t5526-fetch-submodules.sh: verify_fetch_result () {
      	then
      		cat expect.err.deep >>expect.err.combined
      	fi &&
    @@ t/t5526-fetch-submodules.sh: test_expect_success "Recursion doesn't happen when
     -	echo "From $pwd/." > expect.err.super &&
     -	echo "   $head1..$head2  super      -> origin/super" >>expect.err.super &&
     +	new_head=$(git rev-parse --short HEAD) &&
    -+	check_super $new_head &&
    ++	write_expected_super $new_head &&
      	rm expect.err.deep &&
      	(
      		cd downstream &&
    @@ t/t5526-fetch-submodules.sh: test_expect_success "Recursion stops when no new su
     -	echo "From $pwd/." > expect.err.super &&
     -	echo "   $head1..$head2  super      -> origin/super" >> expect.err.super &&
     +	new_head=$(git rev-parse --short HEAD) &&
    -+	check_super $new_head &&
    ++	write_expected_super $new_head &&
      	rm expect.err.sub &&
      	rm expect.err.deep &&
      	(
    @@ t/t5526-fetch-submodules.sh: test_expect_success "Recursion picks up config in s
     -	echo "From $pwd/." > expect.err.super &&
     -	echo "   $head1..$head2  super      -> origin/super" >> expect.err.super &&
     +	new_head=$(git rev-parse --short HEAD) &&
    -+	check_super $new_head &&
    ++	write_expected_super $new_head &&
      	(
      		cd downstream &&
      		git fetch >../actual.out 2>../actual.err &&
    @@ t/t5526-fetch-submodules.sh: test_expect_success "Recursion picks up all submodu
     -		echo "From $pwd/submodule" >> ../expect.err.sub &&
     -		echo "   $head1..$head2  sub        -> origin/sub" >> ../expect.err.sub
     +		new_head=$(git rev-parse --short HEAD) &&
    -+		check_sub $new_head
    ++		write_expected_sub $new_head
      	) &&
     -	head1=$(git rev-parse --short HEAD) &&
      	git add submodule &&
    @@ t/t5526-fetch-submodules.sh: test_expect_success "Recursion picks up all submodu
     -	echo "From $pwd/." > expect.err.super &&
     -	echo "   $head1..$head2  super      -> origin/super" >> expect.err.super &&
     +	new_head=$(git rev-parse --short HEAD) &&
    -+	check_super $new_head &&
    ++	write_expected_super $new_head &&
      	(
      		cd downstream &&
      		git fetch >../actual.out 2>../actual.err
    @@ t/t5526-fetch-submodules.sh: test_expect_success "'--recurse-submodules=on-deman
     -		echo "From $pwd/submodule" >> ../expect.err.sub &&
     -		echo "   $head1..$head2  sub        -> origin/sub" >> ../expect.err.sub
     +		new_head=$(git rev-parse --short HEAD) &&
    -+		check_sub $new_head
    ++		write_expected_sub $new_head
      	) &&
      	(
      		cd downstream &&
    @@ t/t5526-fetch-submodules.sh: test_expect_success "'--recurse-submodules=on-deman
     -	echo "From $pwd/." > expect.err.super &&
     -	echo "   $head1..$head2  super      -> origin/super" >>expect.err.super &&
     +	new_head=$(git rev-parse --short HEAD) &&
    -+	check_super $new_head &&
    ++	write_expected_super $new_head &&
      	(
      		cd downstream &&
      		git config fetch.recurseSubmodules false &&
    @@ t/t5526-fetch-submodules.sh: test_expect_success "'--recurse-submodules=on-deman
     -	echo "From $pwd/." > expect.err.super &&
     -	echo "   $head1..$head2  super      -> origin/super" >> expect.err.super &&
     +	new_head=$(git rev-parse --short HEAD) &&
    -+	check_super $new_head &&
    ++	write_expected_super $new_head &&
      	rm expect.err.sub &&
      	rm expect.err.deep &&
      	(
    @@ t/t5526-fetch-submodules.sh: test_expect_success "'fetch.recurseSubmodules=on-de
     -	echo "From $pwd/." > expect.err.super &&
     -	echo "   $head1..$head2  super      -> origin/super" >>expect.err.super &&
     +	new_head=$(git rev-parse --short HEAD) &&
    -+	check_super $new_head &&
    ++	write_expected_super $new_head &&
      	rm expect.err.deep &&
      	(
      		cd downstream &&
    @@ t/t5526-fetch-submodules.sh: test_expect_success "'submodule.<sub>.fetchRecurseS
     -	echo "From $pwd/." > expect.err.super &&
     -	echo "   $head1..$head2  super      -> origin/super" >>expect.err.super &&
     +	new_head=$(git rev-parse --short HEAD) &&
    -+	check_super $new_head &&
    ++	write_expected_super $new_head &&
      	rm expect.err.deep &&
      	(
      		cd downstream &&
    @@ t/t5526-fetch-submodules.sh: test_expect_success "don't fetch submodule when new
     -	echo "From $pwd/." > expect.err.super &&
     -	echo "   $head1..$head2  super      -> origin/super" >> expect.err.super &&
     +	new_head=$(git rev-parse --short HEAD) &&
    -+	check_super $new_head &&
    ++	write_expected_super $new_head &&
      	rm expect.err.sub &&
      	# This file does not exist, but rm -f for readability
      	rm -f expect.err.deep &&
    @@ t/t5526-fetch-submodules.sh: test_expect_success "'fetch.recurseSubmodules=on-de
     -	echo "From $pwd/." >expect.err.super &&
     -	echo "   $head1..$head2  super      -> origin/super" >>expect.err.super &&
     +	new_head=$(git rev-parse --short HEAD) &&
    -+	check_super $new_head &&
    ++	write_expected_super $new_head &&
      	rm expect.err.deep &&
      	(
      		cd downstream &&
 3:  7e2a01164e !  3:  17ccae2933 t5526: create superproject commits with test helper
    @@ Commit message
         Signed-off-by: Glen Choo <[email protected]>
     
      ## t/t5526-fetch-submodules.sh ##
    -@@ t/t5526-fetch-submodules.sh: check_super () {
    +@@ t/t5526-fetch-submodules.sh: write_expected_super () {
      # a file that contains the expected err if that new commit were fetched.
      # These output files get concatenated in the right order by
      # verify_fetch_result().
    @@ t/t5526-fetch-submodules.sh: add_upstream_commit() {
     +	git commit -m "new submodule" &&
     +	super_head=$(git rev-parse --short HEAD) &&
     +	sub_head=$(git -C submodule rev-parse --short HEAD) &&
    -+	check_super $super_head &&
    -+	check_sub $sub_head
    ++	write_expected_super $super_head &&
    ++	write_expected_sub $sub_head
     +}
     +
      # Verifies that the expected repositories were fetched. This is done by
    @@ t/t5526-fetch-submodules.sh: test_expect_success "Recursion picks up config in s
     -		git add subdir/deepsubmodule &&
     -		git commit -m "new deepsubmodule" &&
     -		new_head=$(git rev-parse --short HEAD) &&
    --		check_sub $new_head
    +-		write_expected_sub $new_head
     -	) &&
     -	git add submodule &&
     -	git commit -m "new submodule" &&
     -	new_head=$(git rev-parse --short HEAD) &&
    --	check_super $new_head &&
    +-	write_expected_super $new_head &&
     +	add_submodule_commits &&
     +	add_superproject_commits &&
      	(
    @@ t/t5526-fetch-submodules.sh: test_expect_success "Recursion picks up all submodu
     -		git add subdir/deepsubmodule &&
     -		git commit -m "new deepsubmodule" &&
     -		new_head=$(git rev-parse --short HEAD) &&
    --		check_sub $new_head
    +-		write_expected_sub $new_head
     -	) &&
     +	add_submodule_commits &&
      	(
    @@ t/t5526-fetch-submodules.sh: test_expect_success "'--recurse-submodules=on-deman
     -	git add submodule &&
     -	git commit -m "new submodule" &&
     -	new_head=$(git rev-parse --short HEAD) &&
    --	check_super $new_head &&
    +-	write_expected_super $new_head &&
    ++	add_submodule_commits &&
     +	add_superproject_commits &&
      	(
      		cd downstream &&
 4:  88112ee225 =  4:  b220dd32c1 submodule: make static functions read submodules from commits
 5:  007cd97aba =  5:  da346aa12a submodule: inline submodule_commits() into caller
 6:  f34ea88fe9 =  6:  c1fd9c3abf submodule: store new submodule commits oid_array in a struct
 7:  f66ab663c5 =  7:  dbb931fe30 submodule: extract get_fetch_task()
 8:  4e3db1bc9d =  8:  7242236df9 submodule: move logic into fetch_task_create()
 9:  9e7b1c1bbe !  9:  0dada865d4 fetch: fetch unpopulated, changed submodules
    @@ submodule.h: int should_update_submodules(void);
      ## t/t5526-fetch-submodules.sh ##
     @@ t/t5526-fetch-submodules.sh: pwd=$(pwd)
      
    - check_sub () {
    + write_expected_sub () {
      	NEW_HEAD=$1 &&
     +	SUPER_HEAD=$2 &&
    - 	cat >$pwd/expect.err.sub <<-EOF
    + 	cat >"$pwd/expect.err.sub" <<-EOF
     -	Fetching submodule submodule
     +	Fetching submodule submodule${SUPER_HEAD:+ at commit $SUPER_HEAD}
      	From $pwd/submodule
      	   OLD_HEAD..$NEW_HEAD  sub        -> origin/sub
      	EOF
    -@@ t/t5526-fetch-submodules.sh: check_sub () {
    + }
      
    - check_deep () {
    ++write_expected_sub2 () {
    ++	NEW_HEAD=$1 &&
    ++	SUPER_HEAD=$2 &&
    ++	cat >"$pwd/expect.err.sub2" <<-EOF
    ++	Fetching submodule submodule2${SUPER_HEAD:+ at commit $SUPER_HEAD}
    ++	From $pwd/submodule2
    ++	   OLD_HEAD..$NEW_HEAD  sub2       -> origin/sub2
    ++	EOF
    ++}
    ++
    + write_expected_deep () {
      	NEW_HEAD=$1 &&
     +	SUB_HEAD=$2 &&
    - 	cat >$pwd/expect.err.deep <<-EOF
    + 	cat >"$pwd/expect.err.deep" <<-EOF
     -	Fetching submodule submodule/subdir/deepsubmodule
     +	Fetching submodule submodule/subdir/deepsubmodule${SUB_HEAD:+ at commit $SUB_HEAD}
      	From $pwd/deepsubmodule
      	   OLD_HEAD..$NEW_HEAD  deep       -> origin/deep
      	EOF
    -@@ t/t5526-fetch-submodules.sh: test_expect_success "'--recurse-submodules=on-demand' doesn't recurse when no ne
    - '
    - 
    - test_expect_success "'--recurse-submodules=on-demand' recurses as deep as necessary (and ignores config)" '
    -+	add_submodule_commits &&
    - 	add_superproject_commits &&
    - 	(
    - 		cd downstream &&
    +@@ t/t5526-fetch-submodules.sh: verify_fetch_result () {
    + 	then
    + 		cat expect.err.deep >>expect.err.combined
    + 	fi &&
    ++	if test -f expect.err.sub2
    ++	then
    ++		cat expect.err.sub2 >>expect.err.combined
    ++	fi &&
    + 	sed -e 's/[0-9a-f][0-9a-f]*\.\./OLD_HEAD\.\./' "$ACTUAL_ERR" >actual.err.cmp &&
    + 	test_cmp expect.err.combined actual.err.cmp
    + }
     @@ t/t5526-fetch-submodules.sh: test_expect_success "'--recurse-submodules=on-demand' recurses as deep as necess
      	verify_fetch_result actual.err
      '
      
    -+# Test that we can fetch submodules in other branches by running fetch
    -+# in a commit that has no submodules.
    ++# These tests verify that we can fetch submodules that aren't in the
    ++# index.
    ++#
    ++# First, test the simple case where the index is empty and we only fetch
    ++# submodules that are not in the index.
     +test_expect_success 'setup downstream branch without submodules' '
     +	(
     +		cd downstream &&
    @@ t/t5526-fetch-submodules.sh: test_expect_success "'--recurse-submodules=on-deman
     +	deep_head=$(git -C submodule/subdir/deepsubmodule rev-parse --short HEAD) &&
     +
     +	# assert that these are fetched from commits, not the index
    -+	check_sub $sub_head $super_head &&
    -+	check_deep $deep_head $sub_head &&
    ++	write_expected_sub $sub_head $super_head &&
    ++	write_expected_deep $deep_head $sub_head &&
     +
     +	test_must_be_empty actual.out &&
     +	verify_fetch_result actual.err
    @@ t/t5526-fetch-submodules.sh: test_expect_success "'--recurse-submodules=on-deman
     +	deep_head=$(git -C submodule/subdir/deepsubmodule rev-parse --short HEAD) &&
     +
     +	# assert that these are fetched from commits, not the index
    -+	check_sub $sub_head $super_head &&
    -+	check_deep $deep_head $sub_head &&
    ++	write_expected_sub $sub_head $super_head &&
    ++	write_expected_deep $deep_head $sub_head &&
     +
     +	test_must_be_empty actual.out &&
     +	verify_fetch_result actual.err
    @@ t/t5526-fetch-submodules.sh: test_expect_success "'--recurse-submodules=on-deman
     +	) &&
     +	test_must_be_empty actual.out &&
     +	super_head=$(git rev-parse --short HEAD) &&
    -+	check_super $super_head &&
    ++	write_expected_super $super_head &&
     +	# Neither should be fetched because the submodule is inactive
     +	rm expect.err.sub &&
     +	rm expect.err.deep &&
     +	verify_fetch_result actual.err
     +'
     +
    -+# In downstream, init "submodule2", but do not check it out while
    -+# fetching. This lets us assert that unpopulated submodules can be
    -+# fetched.
    ++# Now that we know we can fetch submodules that are not in the index,
    ++# test that we can fetch index and non-index submodules in the same
    ++# operation.
     +test_expect_success 'setup downstream branch with other submodule' '
     +	mkdir submodule2 &&
     +	(
    @@ t/t5526-fetch-submodules.sh: test_expect_success "'--recurse-submodules=on-deman
     +'
     +
     +test_expect_success "'--recurse-submodules' should fetch submodule commits in changed submodules and the index" '
    ++	test_when_finished "rm expect.err.sub2" &&
     +	# Create new commit in origin/super
     +	add_submodule_commits &&
     +	add_superproject_commits &&
    @@ t/t5526-fetch-submodules.sh: test_expect_success "'--recurse-submodules=on-deman
     +		git fetch --recurse-submodules >../actual.out 2>../actual.err
     +	) &&
     +	test_must_be_empty actual.out &&
    -+	deep_head=$(git -C submodule/subdir/deepsubmodule rev-parse --short HEAD) &&
    -+	sub_head=$(git -C submodule rev-parse --short HEAD) &&
     +	sub2_head=$(git -C submodule2 rev-parse --short HEAD) &&
    -+	super_head=$(git rev-parse --short HEAD) &&
    ++	super_head=$(git rev-parse --short super) &&
     +	super_sub2_only_head=$(git rev-parse --short super-sub2-only) &&
    ++	write_expected_sub2 $sub2_head $super_sub2_only_head &&
     +
    -+	# Use test_cmp manually because verify_fetch_result does not
    -+	# consider submodule2. All the repos should be fetched, but only
    -+	# submodule2 should be read from a commit
    -+	cat > expect.err.combined <<-EOF &&
    ++	# write_expected_super cannot handle >1 branch. Since this is a
    ++	# one-off, construct expect.err.super manually.
    ++	cat >"$pwd/expect.err.super" <<-EOF &&
     +	From $pwd/.
     +	   OLD_HEAD..$super_head  super           -> origin/super
     +	   OLD_HEAD..$super_sub2_only_head  super-sub2-only -> origin/super-sub2-only
    -+	Fetching submodule submodule
    -+	From $pwd/submodule
    -+	   OLD_HEAD..$sub_head  sub        -> origin/sub
    -+	Fetching submodule submodule/subdir/deepsubmodule
    -+	From $pwd/deepsubmodule
    -+	   OLD_HEAD..$deep_head  deep       -> origin/deep
    -+	Fetching submodule submodule2 at commit $super_sub2_only_head
    -+	From $pwd/submodule2
    -+	   OLD_HEAD..$sub2_head  sub2       -> origin/sub2
     +	EOF
    -+	sed -e "s/[0-9a-f][0-9a-f]*\.\./OLD_HEAD\.\./" actual.err >actual.err.cmp &&
    -+	test_cmp expect.err.combined actual.err.cmp
    ++	verify_fetch_result actual.err
     +'
     +
      test_expect_success "'--recurse-submodules=on-demand' stops when no new submodule commits are found in the superproject (and ignores config)" '

10: 362ce3c7f8 = 10: 71bb456041 submodule: fix latent check_has_commit() bug

base-commit: 715d08a9e51251ad8290b181b6ac3b9e1f9719d7

Signature · 1 line
2.33.GIT
author
#128 lore raw

PATCH v5 1/10 t5526: introduce test helper to assert on fetches

Tests in t/t5526-fetch-submodules.sh are unnecessarily noisy:

* The tests have extra logic in order to reproduce the expected stderr literally, but not all of these details (e.g. the head of the remote-tracking branch before the fetch) are relevant to the test.

* The expect.err file is constructed by the add_upstream_commit() helper as input into test_cmp, but most tests fetch a different combination of repos from expect.err. This results in noisy tests that modify parts of that expect.err to generate the expected output.

To address both of these issues, introduce a verify_fetch_result() helper to t/t5526-fetch-submodules.sh that asserts on the output of "git fetch --recurse-submodules" and handles the ordering of expect.err.

As a result, the tests no longer construct expect.err manually. Tests still consider the old head of the remote-tracking branch ("$head1"), but that will be fixed in a later commit.

---
 t/t5526-fetch-submodules.sh | 139 ++++++++++++++++++++++--------------
 1 file changed, 84 insertions(+), 55 deletions(-)
t/t5526-fetch-submodules.sh +84 -55
index 840c89cc8b..c3a67270b1 100755
@@ -10,6 +10,10 @@ export GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB
 
 pwd=$(pwd)
 
+# For each submodule in the test setup, this creates a commit and writes
+# a file that contains the expected err if that new commit were fetched.
+# These output files get concatenated in the right order by
+# verify_fetch_result().
 add_upstream_commit() {
 	(
 		cd submodule &&
@@ -19,9 +23,9 @@ add_upstream_commit() {
 		git add subfile &&
 		git commit -m new subfile &&
 		head2=$(git rev-parse --short HEAD) &&
-		echo "Fetching submodule submodule" > ../expect.err &&
-		echo "From $pwd/submodule" >> ../expect.err &&
-		echo "   $head1..$head2  sub        -> origin/sub" >> ../expect.err
+		echo "Fetching submodule submodule" > ../expect.err.sub &&
+		echo "From $pwd/submodule" >> ../expect.err.sub &&
+		echo "   $head1..$head2  sub        -> origin/sub" >> ../expect.err.sub
 	) &&
 	(
 		cd deepsubmodule &&
@@ -31,12 +35,36 @@ add_upstream_commit() {
 		git add deepsubfile &&
 		git commit -m new deepsubfile &&
 		head2=$(git rev-parse --short HEAD) &&
-		echo "Fetching submodule submodule/subdir/deepsubmodule" >> ../expect.err
-		echo "From $pwd/deepsubmodule" >> ../expect.err &&
-		echo "   $head1..$head2  deep       -> origin/deep" >> ../expect.err
+		echo "Fetching submodule submodule/subdir/deepsubmodule" > ../expect.err.deep
+		echo "From $pwd/deepsubmodule" >> ../expect.err.deep &&
+		echo "   $head1..$head2  deep       -> origin/deep" >> ../expect.err.deep
 	)
 }
 
+# Verifies that the expected repositories were fetched. This is done by
+# concatenating the files expect.err.[super|sub|deep] in the correct
+# order and comparing it to the actual stderr.
+#
+# If a repo should not be fetched in the test, its corresponding
+# expect.err file should be rm-ed.
+verify_fetch_result () {
+	ACTUAL_ERR=$1 &&
+	rm -f expect.err.combined &&
+	if test -f expect.err.super
+	then
+		cat expect.err.super >>expect.err.combined
+	fi &&
+	if test -f expect.err.sub
+	then
+		cat expect.err.sub >>expect.err.combined
+	fi &&
+	if test -f expect.err.deep
+	then
+		cat expect.err.deep >>expect.err.combined
+	fi &&
+	test_cmp expect.err.combined $ACTUAL_ERR
+}
+
 test_expect_success setup '
 	mkdir deepsubmodule &&
 	(
@@ -74,7 +102,7 @@ test_expect_success "fetch --recurse-submodules recurses into submodules" '
 		git fetch --recurse-submodules >../actual.out 2>../actual.err
 	) &&
 	test_must_be_empty actual.out &&
-	test_cmp expect.err actual.err
+	verify_fetch_result actual.err
 '
 
 test_expect_success "submodule.recurse option triggers recursive fetch" '
@@ -84,7 +112,7 @@ test_expect_success "submodule.recurse option triggers recursive fetch" '
 		git -c submodule.recurse fetch >../actual.out 2>../actual.err
 	) &&
 	test_must_be_empty actual.out &&
-	test_cmp expect.err actual.err
+	verify_fetch_result actual.err
 '
 
 test_expect_success "fetch --recurse-submodules -j2 has the same output behaviour" '
@@ -94,7 +122,7 @@ test_expect_success "fetch --recurse-submodules -j2 has the same output behaviou
 		GIT_TRACE="$TRASH_DIRECTORY/trace.out" git fetch --recurse-submodules -j2 2>../actual.err
 	) &&
 	test_must_be_empty actual.out &&
-	test_cmp expect.err actual.err &&
+	verify_fetch_result actual.err &&
 	grep "2 tasks" trace.out
 '
 
@@ -124,7 +152,7 @@ test_expect_success "using fetchRecurseSubmodules=true in .gitmodules recurses i
 		git fetch >../actual.out 2>../actual.err
 	) &&
 	test_must_be_empty actual.out &&
-	test_cmp expect.err actual.err
+	verify_fetch_result actual.err
 '
 
 test_expect_success "--no-recurse-submodules overrides .gitmodules config" '
@@ -155,7 +183,7 @@ test_expect_success "--recurse-submodules overrides fetchRecurseSubmodules setti
 		git config --unset submodule.submodule.fetchRecurseSubmodules
 	) &&
 	test_must_be_empty actual.out &&
-	test_cmp expect.err actual.err
+	verify_fetch_result actual.err
 '
 
 test_expect_success "--quiet propagates to submodules" '
@@ -183,7 +211,7 @@ test_expect_success "--dry-run propagates to submodules" '
 		git fetch --recurse-submodules --dry-run >../actual.out 2>../actual.err
 	) &&
 	test_must_be_empty actual.out &&
-	test_cmp expect.err actual.err
+	verify_fetch_result actual.err
 '
 
 test_expect_success "Without --dry-run propagates to submodules" '
@@ -192,7 +220,7 @@ test_expect_success "Without --dry-run propagates to submodules" '
 		git fetch --recurse-submodules >../actual.out 2>../actual.err
 	) &&
 	test_must_be_empty actual.out &&
-	test_cmp expect.err actual.err
+	verify_fetch_result actual.err
 '
 
 test_expect_success "recurseSubmodules=true propagates into submodules" '
@@ -203,7 +231,7 @@ test_expect_success "recurseSubmodules=true propagates into submodules" '
 		git fetch >../actual.out 2>../actual.err
 	) &&
 	test_must_be_empty actual.out &&
-	test_cmp expect.err actual.err
+	verify_fetch_result actual.err
 '
 
 test_expect_success "--recurse-submodules overrides config in submodule" '
@@ -217,7 +245,7 @@ test_expect_success "--recurse-submodules overrides config in submodule" '
 		git fetch --recurse-submodules >../actual.out 2>../actual.err
 	) &&
 	test_must_be_empty actual.out &&
-	test_cmp expect.err actual.err
+	verify_fetch_result actual.err
 '
 
 test_expect_success "--no-recurse-submodules overrides config setting" '
@@ -250,14 +278,14 @@ test_expect_success "Recursion stops when no new submodule commits are fetched"
 	git add submodule &&
 	git commit -m "new submodule" &&
 	head2=$(git rev-parse --short HEAD) &&
-	echo "From $pwd/." > expect.err.sub &&
-	echo "   $head1..$head2  super      -> origin/super" >>expect.err.sub &&
-	head -3 expect.err >> expect.err.sub &&
+	echo "From $pwd/." > expect.err.super &&
+	echo "   $head1..$head2  super      -> origin/super" >>expect.err.super &&
+	rm expect.err.deep &&
 	(
 		cd downstream &&
 		git fetch >../actual.out 2>../actual.err
 	) &&
-	test_cmp expect.err.sub actual.err &&
+	verify_fetch_result actual.err &&
 	test_must_be_empty actual.out
 '
 
@@ -268,14 +296,16 @@ test_expect_success "Recursion doesn't happen when new superproject commits don'
 	git add file &&
 	git commit -m "new file" &&
 	head2=$(git rev-parse --short HEAD) &&
-	echo "From $pwd/." > expect.err.file &&
-	echo "   $head1..$head2  super      -> origin/super" >> expect.err.file &&
+	echo "From $pwd/." > expect.err.super &&
+	echo "   $head1..$head2  super      -> origin/super" >> expect.err.super &&
+	rm expect.err.sub &&
+	rm expect.err.deep &&
 	(
 		cd downstream &&
 		git fetch >../actual.out 2>../actual.err
 	) &&
 	test_must_be_empty actual.out &&
-	test_cmp expect.err.file actual.err
+	verify_fetch_result actual.err
 '
 
 test_expect_success "Recursion picks up config in submodule" '
@@ -292,9 +322,8 @@ test_expect_success "Recursion picks up config in submodule" '
 	git add submodule &&
 	git commit -m "new submodule" &&
 	head2=$(git rev-parse --short HEAD) &&
-	echo "From $pwd/." > expect.err.sub &&
-	echo "   $head1..$head2  super      -> origin/super" >> expect.err.sub &&
-	cat expect.err >> expect.err.sub &&
+	echo "From $pwd/." > expect.err.super &&
+	echo "   $head1..$head2  super      -> origin/super" >> expect.err.super &&
 	(
 		cd downstream &&
 		git fetch >../actual.out 2>../actual.err &&
@@ -303,7 +332,7 @@ test_expect_success "Recursion picks up config in submodule" '
 			git config --unset fetch.recurseSubmodules
 		)
 	) &&
-	test_cmp expect.err.sub actual.err &&
+	verify_fetch_result actual.err &&
 	test_must_be_empty actual.out
 '
 
@@ -328,15 +357,13 @@ test_expect_success "Recursion picks up all submodules when necessary" '
 	git add submodule &&
 	git commit -m "new submodule" &&
 	head2=$(git rev-parse --short HEAD) &&
-	echo "From $pwd/." > expect.err.2 &&
-	echo "   $head1..$head2  super      -> origin/super" >> expect.err.2 &&
-	cat expect.err.sub >> expect.err.2 &&
-	tail -3 expect.err >> expect.err.2 &&
+	echo "From $pwd/." > expect.err.super &&
+	echo "   $head1..$head2  super      -> origin/super" >> expect.err.super &&
 	(
 		cd downstream &&
 		git fetch >../actual.out 2>../actual.err
 	) &&
-	test_cmp expect.err.2 actual.err &&
+	verify_fetch_result actual.err &&
 	test_must_be_empty actual.out
 '
 
@@ -372,11 +399,8 @@ test_expect_success "'--recurse-submodules=on-demand' recurses as deep as necess
 	git add submodule &&
 	git commit -m "new submodule" &&
 	head2=$(git rev-parse --short HEAD) &&
-	tail -3 expect.err > expect.err.deepsub &&
-	echo "From $pwd/." > expect.err &&
-	echo "   $head1..$head2  super      -> origin/super" >>expect.err &&
-	cat expect.err.sub >> expect.err &&
-	cat expect.err.deepsub >> expect.err &&
+	echo "From $pwd/." > expect.err.super &&
+	echo "   $head1..$head2  super      -> origin/super" >>expect.err.super &&
 	(
 		cd downstream &&
 		git config fetch.recurseSubmodules false &&
@@ -392,7 +416,7 @@ test_expect_success "'--recurse-submodules=on-demand' recurses as deep as necess
 		)
 	) &&
 	test_must_be_empty actual.out &&
-	test_cmp expect.err actual.err
+	verify_fetch_result actual.err
 '
 
 test_expect_success "'--recurse-submodules=on-demand' stops when no new submodule commits are found in the superproject (and ignores config)" '
@@ -402,14 +426,16 @@ test_expect_success "'--recurse-submodules=on-demand' stops when no new submodul
 	git add file &&
 	git commit -m "new file" &&
 	head2=$(git rev-parse --short HEAD) &&
-	echo "From $pwd/." > expect.err.file &&
-	echo "   $head1..$head2  super      -> origin/super" >> expect.err.file &&
+	echo "From $pwd/." > expect.err.super &&
+	echo "   $head1..$head2  super      -> origin/super" >> expect.err.super &&
+	rm expect.err.sub &&
+	rm expect.err.deep &&
 	(
 		cd downstream &&
 		git fetch --recurse-submodules=on-demand >../actual.out 2>../actual.err
 	) &&
 	test_must_be_empty actual.out &&
-	test_cmp expect.err.file actual.err
+	verify_fetch_result actual.err
 '
 
 test_expect_success "'fetch.recurseSubmodules=on-demand' overrides global config" '
@@ -423,9 +449,9 @@ test_expect_success "'fetch.recurseSubmodules=on-demand' overrides global config
 	git add submodule &&
 	git commit -m "new submodule" &&
 	head2=$(git rev-parse --short HEAD) &&
-	echo "From $pwd/." > expect.err.2 &&
-	echo "   $head1..$head2  super      -> origin/super" >>expect.err.2 &&
-	head -3 expect.err >> expect.err.2 &&
+	echo "From $pwd/." > expect.err.super &&
+	echo "   $head1..$head2  super      -> origin/super" >>expect.err.super &&
+	rm expect.err.deep &&
 	(
 		cd downstream &&
 		git config fetch.recurseSubmodules on-demand &&
@@ -437,7 +463,7 @@ test_expect_success "'fetch.recurseSubmodules=on-demand' overrides global config
 		git config --unset fetch.recurseSubmodules
 	) &&
 	test_must_be_empty actual.out &&
-	test_cmp expect.err.2 actual.err
+	verify_fetch_result actual.err
 '
 
 test_expect_success "'submodule.<sub>.fetchRecurseSubmodules=on-demand' overrides fetch.recurseSubmodules" '
@@ -451,9 +477,9 @@ test_expect_success "'submodule.<sub>.fetchRecurseSubmodules=on-demand' override
 	git add submodule &&
 	git commit -m "new submodule" &&
 	head2=$(git rev-parse --short HEAD) &&
-	echo "From $pwd/." > expect.err.2 &&
-	echo "   $head1..$head2  super      -> origin/super" >>expect.err.2 &&
-	head -3 expect.err >> expect.err.2 &&
+	echo "From $pwd/." > expect.err.super &&
+	echo "   $head1..$head2  super      -> origin/super" >>expect.err.super &&
+	rm expect.err.deep &&
 	(
 		cd downstream &&
 		git config submodule.submodule.fetchRecurseSubmodules on-demand &&
@@ -465,7 +491,7 @@ test_expect_success "'submodule.<sub>.fetchRecurseSubmodules=on-demand' override
 		git config --unset submodule.submodule.fetchRecurseSubmodules
 	) &&
 	test_must_be_empty actual.out &&
-	test_cmp expect.err.2 actual.err
+	verify_fetch_result actual.err
 '
 
 test_expect_success "don't fetch submodule when newly recorded commits are already present" '
@@ -477,14 +503,17 @@ test_expect_success "don't fetch submodule when newly recorded commits are alrea
 	git add submodule &&
 	git commit -m "submodule rewound" &&
 	head2=$(git rev-parse --short HEAD) &&
-	echo "From $pwd/." > expect.err &&
-	echo "   $head1..$head2  super      -> origin/super" >> expect.err &&
+	echo "From $pwd/." > expect.err.super &&
+	echo "   $head1..$head2  super      -> origin/super" >> expect.err.super &&
+	rm expect.err.sub &&
+	# This file does not exist, but rm -f for readability
+	rm -f expect.err.deep &&
 	(
 		cd downstream &&
 		git fetch >../actual.out 2>../actual.err
 	) &&
 	test_must_be_empty actual.out &&
-	test_cmp expect.err actual.err &&
+	verify_fetch_result actual.err &&
 	(
 		cd submodule &&
 		git checkout -q sub
@@ -502,9 +531,9 @@ test_expect_success "'fetch.recurseSubmodules=on-demand' works also without .git
 	git rm .gitmodules &&
 	git commit -m "new submodule without .gitmodules" &&
 	head2=$(git rev-parse --short HEAD) &&
-	echo "From $pwd/." >expect.err.2 &&
-	echo "   $head1..$head2  super      -> origin/super" >>expect.err.2 &&
-	head -3 expect.err >>expect.err.2 &&
+	echo "From $pwd/." >expect.err.super &&
+	echo "   $head1..$head2  super      -> origin/super" >>expect.err.super &&
+	rm expect.err.deep &&
 	(
 		cd downstream &&
 		rm .gitmodules &&
@@ -520,7 +549,7 @@ test_expect_success "'fetch.recurseSubmodules=on-demand' works also without .git
 		git reset --hard
 	) &&
 	test_must_be_empty actual.out &&
-	test_cmp expect.err.2 actual.err &&
+	verify_fetch_result actual.err &&
 	git checkout HEAD^ -- .gitmodules &&
 	git add .gitmodules &&
 	git commit -m "new submodule restored .gitmodules"
Signature · 1 line
2.33.GIT
author
#129 lore raw

PATCH v5 2/10 t5526: stop asserting on stderr literally

In reply to Glen Choo

In the previous commit message, we noted that not all of the "git fetch" stderr is relevant to the tests. Most of the test setup lines are dedicated to these details of the stderr:

1. which repos (super/sub/deep) are involved in the fetch
2. the head of the remote-tracking branch before the fetch (i.e. $head1)
3. the head of the remote-tracking branch after the fetch (i.e. $head2)

1. and 3. are relevant because they tell us that the expected commit is fetched by the expected repo, but 2. is completely irrelevant.

Stop asserting on $head1 by replacing it with a dummy value in the actual and expected output. Do this by introducing test helpers (write_expected_*()) that make it easier to construct the expected output, and use sed to munge the actual output.

---
 t/t5526-fetch-submodules.sh | 117 +++++++++++++++++-------------------
 1 file changed, 56 insertions(+), 61 deletions(-)
t/t5526-fetch-submodules.sh +56 -61
index c3a67270b1..e7136b68ba 100755
@@ -10,6 +10,32 @@ export GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB
 
 pwd=$(pwd)
 
+write_expected_sub () {
+	NEW_HEAD=$1 &&
+	cat >"$pwd/expect.err.sub" <<-EOF
+	Fetching submodule submodule
+	From $pwd/submodule
+	   OLD_HEAD..$NEW_HEAD  sub        -> origin/sub
+	EOF
+}
+
+write_expected_deep () {
+	NEW_HEAD=$1 &&
+	cat >"$pwd/expect.err.deep" <<-EOF
+	Fetching submodule submodule/subdir/deepsubmodule
+	From $pwd/deepsubmodule
+	   OLD_HEAD..$NEW_HEAD  deep       -> origin/deep
+	EOF
+}
+
+write_expected_super () {
+	NEW_HEAD=$1 &&
+	cat >"$pwd/expect.err.super" <<-EOF
+	From $pwd/.
+	   OLD_HEAD..$NEW_HEAD  super      -> origin/super
+	EOF
+}
+
 # For each submodule in the test setup, this creates a commit and writes
 # a file that contains the expected err if that new commit were fetched.
 # These output files get concatenated in the right order by
@@ -17,27 +43,21 @@ pwd=$(pwd)
 add_upstream_commit() {
 	(
 		cd submodule &&
-		head1=$(git rev-parse --short HEAD) &&
 		echo new >> subfile &&
 		test_tick &&
 		git add subfile &&
 		git commit -m new subfile &&
-		head2=$(git rev-parse --short HEAD) &&
-		echo "Fetching submodule submodule" > ../expect.err.sub &&
-		echo "From $pwd/submodule" >> ../expect.err.sub &&
-		echo "   $head1..$head2  sub        -> origin/sub" >> ../expect.err.sub
+		new_head=$(git rev-parse --short HEAD) &&
+		write_expected_sub $new_head
 	) &&
 	(
 		cd deepsubmodule &&
-		head1=$(git rev-parse --short HEAD) &&
 		echo new >> deepsubfile &&
 		test_tick &&
 		git add deepsubfile &&
 		git commit -m new deepsubfile &&
-		head2=$(git rev-parse --short HEAD) &&
-		echo "Fetching submodule submodule/subdir/deepsubmodule" > ../expect.err.deep
-		echo "From $pwd/deepsubmodule" >> ../expect.err.deep &&
-		echo "   $head1..$head2  deep       -> origin/deep" >> ../expect.err.deep
+		new_head=$(git rev-parse --short HEAD) &&
+		write_expected_deep $new_head
 	)
 }
 
@@ -62,7 +82,8 @@ verify_fetch_result () {
 	then
 		cat expect.err.deep >>expect.err.combined
 	fi &&
-	test_cmp expect.err.combined $ACTUAL_ERR
+	sed -e 's/[0-9a-f][0-9a-f]*\.\./OLD_HEAD\.\./' "$ACTUAL_ERR" >actual.err.cmp &&
+	test_cmp expect.err.combined actual.err.cmp
 }
 
 test_expect_success setup '
@@ -274,12 +295,10 @@ test_expect_success "Recursion doesn't happen when no new commits are fetched in
 '
 
 test_expect_success "Recursion stops when no new submodule commits are fetched" '
-	head1=$(git rev-parse --short HEAD) &&
 	git add submodule &&
 	git commit -m "new submodule" &&
-	head2=$(git rev-parse --short HEAD) &&
-	echo "From $pwd/." > expect.err.super &&
-	echo "   $head1..$head2  super      -> origin/super" >>expect.err.super &&
+	new_head=$(git rev-parse --short HEAD) &&
+	write_expected_super $new_head &&
 	rm expect.err.deep &&
 	(
 		cd downstream &&
@@ -291,13 +310,11 @@ test_expect_success "Recursion stops when no new submodule commits are fetched"
 
 test_expect_success "Recursion doesn't happen when new superproject commits don't change any submodules" '
 	add_upstream_commit &&
-	head1=$(git rev-parse --short HEAD) &&
 	echo a > file &&
 	git add file &&
 	git commit -m "new file" &&
-	head2=$(git rev-parse --short HEAD) &&
-	echo "From $pwd/." > expect.err.super &&
-	echo "   $head1..$head2  super      -> origin/super" >> expect.err.super &&
+	new_head=$(git rev-parse --short HEAD) &&
+	write_expected_super $new_head &&
 	rm expect.err.sub &&
 	rm expect.err.deep &&
 	(
@@ -318,12 +335,10 @@ test_expect_success "Recursion picks up config in submodule" '
 		)
 	) &&
 	add_upstream_commit &&
-	head1=$(git rev-parse --short HEAD) &&
 	git add submodule &&
 	git commit -m "new submodule" &&
-	head2=$(git rev-parse --short HEAD) &&
-	echo "From $pwd/." > expect.err.super &&
-	echo "   $head1..$head2  super      -> origin/super" >> expect.err.super &&
+	new_head=$(git rev-parse --short HEAD) &&
+	write_expected_super $new_head &&
 	(
 		cd downstream &&
 		git fetch >../actual.out 2>../actual.err &&
@@ -345,20 +360,15 @@ test_expect_success "Recursion picks up all submodules when necessary" '
 			git fetch &&
 			git checkout -q FETCH_HEAD
 		) &&
-		head1=$(git rev-parse --short HEAD^) &&
 		git add subdir/deepsubmodule &&
 		git commit -m "new deepsubmodule" &&
-		head2=$(git rev-parse --short HEAD) &&
-		echo "Fetching submodule submodule" > ../expect.err.sub &&
-		echo "From $pwd/submodule" >> ../expect.err.sub &&
-		echo "   $head1..$head2  sub        -> origin/sub" >> ../expect.err.sub
+		new_head=$(git rev-parse --short HEAD) &&
+		write_expected_sub $new_head
 	) &&
-	head1=$(git rev-parse --short HEAD) &&
 	git add submodule &&
 	git commit -m "new submodule" &&
-	head2=$(git rev-parse --short HEAD) &&
-	echo "From $pwd/." > expect.err.super &&
-	echo "   $head1..$head2  super      -> origin/super" >> expect.err.super &&
+	new_head=$(git rev-parse --short HEAD) &&
+	write_expected_super $new_head &&
 	(
 		cd downstream &&
 		git fetch >../actual.out 2>../actual.err
@@ -376,13 +386,10 @@ test_expect_success "'--recurse-submodules=on-demand' doesn't recurse when no ne
 			git fetch &&
 			git checkout -q FETCH_HEAD
 		) &&
-		head1=$(git rev-parse --short HEAD^) &&
 		git add subdir/deepsubmodule &&
 		git commit -m "new deepsubmodule" &&
-		head2=$(git rev-parse --short HEAD) &&
-		echo Fetching submodule submodule > ../expect.err.sub &&
-		echo "From $pwd/submodule" >> ../expect.err.sub &&
-		echo "   $head1..$head2  sub        -> origin/sub" >> ../expect.err.sub
+		new_head=$(git rev-parse --short HEAD) &&
+		write_expected_sub $new_head
 	) &&
 	(
 		cd downstream &&
@@ -395,12 +402,10 @@ test_expect_success "'--recurse-submodules=on-demand' doesn't recurse when no ne
 '
 
 test_expect_success "'--recurse-submodules=on-demand' recurses as deep as necessary (and ignores config)" '
-	head1=$(git rev-parse --short HEAD) &&
 	git add submodule &&
 	git commit -m "new submodule" &&
-	head2=$(git rev-parse --short HEAD) &&
-	echo "From $pwd/." > expect.err.super &&
-	echo "   $head1..$head2  super      -> origin/super" >>expect.err.super &&
+	new_head=$(git rev-parse --short HEAD) &&
+	write_expected_super $new_head &&
 	(
 		cd downstream &&
 		git config fetch.recurseSubmodules false &&
@@ -421,13 +426,11 @@ test_expect_success "'--recurse-submodules=on-demand' recurses as deep as necess
 
 test_expect_success "'--recurse-submodules=on-demand' stops when no new submodule commits are found in the superproject (and ignores config)" '
 	add_upstream_commit &&
-	head1=$(git rev-parse --short HEAD) &&
 	echo a >> file &&
 	git add file &&
 	git commit -m "new file" &&
-	head2=$(git rev-parse --short HEAD) &&
-	echo "From $pwd/." > expect.err.super &&
-	echo "   $head1..$head2  super      -> origin/super" >> expect.err.super &&
+	new_head=$(git rev-parse --short HEAD) &&
+	write_expected_super $new_head &&
 	rm expect.err.sub &&
 	rm expect.err.deep &&
 	(
@@ -445,12 +448,10 @@ test_expect_success "'fetch.recurseSubmodules=on-demand' overrides global config
 	) &&
 	add_upstream_commit &&
 	git config --global fetch.recurseSubmodules false &&
-	head1=$(git rev-parse --short HEAD) &&
 	git add submodule &&
 	git commit -m "new submodule" &&
-	head2=$(git rev-parse --short HEAD) &&
-	echo "From $pwd/." > expect.err.super &&
-	echo "   $head1..$head2  super      -> origin/super" >>expect.err.super &&
+	new_head=$(git rev-parse --short HEAD) &&
+	write_expected_super $new_head &&
 	rm expect.err.deep &&
 	(
 		cd downstream &&
@@ -473,12 +474,10 @@ test_expect_success "'submodule.<sub>.fetchRecurseSubmodules=on-demand' override
 	) &&
 	add_upstream_commit &&
 	git config fetch.recurseSubmodules false &&
-	head1=$(git rev-parse --short HEAD) &&
 	git add submodule &&
 	git commit -m "new submodule" &&
-	head2=$(git rev-parse --short HEAD) &&
-	echo "From $pwd/." > expect.err.super &&
-	echo "   $head1..$head2  super      -> origin/super" >>expect.err.super &&
+	new_head=$(git rev-parse --short HEAD) &&
+	write_expected_super $new_head &&
 	rm expect.err.deep &&
 	(
 		cd downstream &&
@@ -499,12 +498,10 @@ test_expect_success "don't fetch submodule when newly recorded commits are alrea
 		cd submodule &&
 		git checkout -q HEAD^^
 	) &&
-	head1=$(git rev-parse --short HEAD) &&
 	git add submodule &&
 	git commit -m "submodule rewound" &&
-	head2=$(git rev-parse --short HEAD) &&
-	echo "From $pwd/." > expect.err.super &&
-	echo "   $head1..$head2  super      -> origin/super" >> expect.err.super &&
+	new_head=$(git rev-parse --short HEAD) &&
+	write_expected_super $new_head &&
 	rm expect.err.sub &&
 	# This file does not exist, but rm -f for readability
 	rm -f expect.err.deep &&
@@ -526,13 +523,11 @@ test_expect_success "'fetch.recurseSubmodules=on-demand' works also without .git
 		git fetch --recurse-submodules
 	) &&
 	add_upstream_commit &&
-	head1=$(git rev-parse --short HEAD) &&
 	git add submodule &&
 	git rm .gitmodules &&
 	git commit -m "new submodule without .gitmodules" &&
-	head2=$(git rev-parse --short HEAD) &&
-	echo "From $pwd/." >expect.err.super &&
-	echo "   $head1..$head2  super      -> origin/super" >>expect.err.super &&
+	new_head=$(git rev-parse --short HEAD) &&
+	write_expected_super $new_head &&
 	rm expect.err.deep &&
 	(
 		cd downstream &&
Signature · 1 line
2.33.GIT
author
#130 lore raw

PATCH v5 3/10 t5526: create superproject commits with test helper

In reply to Glen Choo

A few tests in t5526 use this pattern as part of their setup:

1. Create new commits in the upstream submodules (using add_upstream_commit()).
2. In the upstream superprojects, add the new submodule commits from the previous step.

A future commit will add more tests with this pattern, so reduce the verbosity of present and future tests by introducing a test helper that creates superproject commits. Since we now have two helpers that add upstream commits, rename add_upstream_commit() to add_submodule_commits().

---
 t/t5526-fetch-submodules.sh | 95 ++++++++++++++++++-------------------
 1 file changed, 45 insertions(+), 50 deletions(-)
t/t5526-fetch-submodules.sh +45 -50
index e7136b68ba..aa6bb9867c 100755
@@ -40,7 +40,7 @@ write_expected_super () {
 # a file that contains the expected err if that new commit were fetched.
 # These output files get concatenated in the right order by
 # verify_fetch_result().
-add_upstream_commit() {
+add_submodule_commits () {
 	(
 		cd submodule &&
 		echo new >> subfile &&
@@ -61,6 +61,30 @@ add_upstream_commit() {
 	)
 }
 
+# For each superproject in the test setup, update its submodule, add the
+# submodule and create a new commit with the submodule change.
+#
+# This requires add_submodule_commits() to be called first, otherwise
+# the submodules will not have changed and cannot be "git add"-ed.
+add_superproject_commits () {
+	(
+		cd submodule &&
+		(
+			cd subdir/deepsubmodule &&
+			git fetch &&
+			git checkout -q FETCH_HEAD
+		) &&
+		git add subdir/deepsubmodule &&
+		git commit -m "new deep submodule"
+	) &&
+	git add submodule &&
+	git commit -m "new submodule" &&
+	super_head=$(git rev-parse --short HEAD) &&
+	sub_head=$(git -C submodule rev-parse --short HEAD) &&
+	write_expected_super $super_head &&
+	write_expected_sub $sub_head
+}
+
 # Verifies that the expected repositories were fetched. This is done by
 # concatenating the files expect.err.[super|sub|deep] in the correct
 # order and comparing it to the actual stderr.
@@ -117,7 +141,7 @@ test_expect_success setup '
 '
 
 test_expect_success "fetch --recurse-submodules recurses into submodules" '
-	add_upstream_commit &&
+	add_submodule_commits &&
 	(
 		cd downstream &&
 		git fetch --recurse-submodules >../actual.out 2>../actual.err
@@ -127,7 +151,7 @@ test_expect_success "fetch --recurse-submodules recurses into submodules" '
 '
 
 test_expect_success "submodule.recurse option triggers recursive fetch" '
-	add_upstream_commit &&
+	add_submodule_commits &&
 	(
 		cd downstream &&
 		git -c submodule.recurse fetch >../actual.out 2>../actual.err
@@ -137,7 +161,7 @@ test_expect_success "submodule.recurse option triggers recursive fetch" '
 '
 
 test_expect_success "fetch --recurse-submodules -j2 has the same output behaviour" '
-	add_upstream_commit &&
+	add_submodule_commits &&
 	(
 		cd downstream &&
 		GIT_TRACE="$TRASH_DIRECTORY/trace.out" git fetch --recurse-submodules -j2 2>../actual.err
@@ -148,7 +172,7 @@ test_expect_success "fetch --recurse-submodules -j2 has the same output behaviou
 '
 
 test_expect_success "fetch alone only fetches superproject" '
-	add_upstream_commit &&
+	add_submodule_commits &&
 	(
 		cd downstream &&
 		git fetch >../actual.out 2>../actual.err
@@ -177,7 +201,7 @@ test_expect_success "using fetchRecurseSubmodules=true in .gitmodules recurses i
 '
 
 test_expect_success "--no-recurse-submodules overrides .gitmodules config" '
-	add_upstream_commit &&
+	add_submodule_commits &&
 	(
 		cd downstream &&
 		git fetch --no-recurse-submodules >../actual.out 2>../actual.err
@@ -226,7 +250,7 @@ test_expect_success "--quiet propagates to parallel submodules" '
 '
 
 test_expect_success "--dry-run propagates to submodules" '
-	add_upstream_commit &&
+	add_submodule_commits &&
 	(
 		cd downstream &&
 		git fetch --recurse-submodules --dry-run >../actual.out 2>../actual.err
@@ -245,7 +269,7 @@ test_expect_success "Without --dry-run propagates to submodules" '
 '
 
 test_expect_success "recurseSubmodules=true propagates into submodules" '
-	add_upstream_commit &&
+	add_submodule_commits &&
 	(
 		cd downstream &&
 		git config fetch.recurseSubmodules true &&
@@ -256,7 +280,7 @@ test_expect_success "recurseSubmodules=true propagates into submodules" '
 '
 
 test_expect_success "--recurse-submodules overrides config in submodule" '
-	add_upstream_commit &&
+	add_submodule_commits &&
 	(
 		cd downstream &&
 		(
@@ -270,7 +294,7 @@ test_expect_success "--recurse-submodules overrides config in submodule" '
 '
 
 test_expect_success "--no-recurse-submodules overrides config setting" '
-	add_upstream_commit &&
+	add_submodule_commits &&
 	(
 		cd downstream &&
 		git config fetch.recurseSubmodules true &&
@@ -309,7 +333,7 @@ test_expect_success "Recursion stops when no new submodule commits are fetched"
 '
 
 test_expect_success "Recursion doesn't happen when new superproject commits don't change any submodules" '
-	add_upstream_commit &&
+	add_submodule_commits &&
 	echo a > file &&
 	git add file &&
 	git commit -m "new file" &&
@@ -334,7 +358,7 @@ test_expect_success "Recursion picks up config in submodule" '
 			git config fetch.recurseSubmodules true
 		)
 	) &&
-	add_upstream_commit &&
+	add_submodule_commits &&
 	git add submodule &&
 	git commit -m "new submodule" &&
 	new_head=$(git rev-parse --short HEAD) &&
@@ -352,23 +376,8 @@ test_expect_success "Recursion picks up config in submodule" '
 '
 
 test_expect_success "Recursion picks up all submodules when necessary" '
-	add_upstream_commit &&
-	(
-		cd submodule &&
-		(
-			cd subdir/deepsubmodule &&
-			git fetch &&
-			git checkout -q FETCH_HEAD
-		) &&
-		git add subdir/deepsubmodule &&
-		git commit -m "new deepsubmodule" &&
-		new_head=$(git rev-parse --short HEAD) &&
-		write_expected_sub $new_head
-	) &&
-	git add submodule &&
-	git commit -m "new submodule" &&
-	new_head=$(git rev-parse --short HEAD) &&
-	write_expected_super $new_head &&
+	add_submodule_commits &&
+	add_superproject_commits &&
 	(
 		cd downstream &&
 		git fetch >../actual.out 2>../actual.err
@@ -378,19 +387,7 @@ test_expect_success "Recursion picks up all submodules when necessary" '
 '
 
 test_expect_success "'--recurse-submodules=on-demand' doesn't recurse when no new commits are fetched in the superproject (and ignores config)" '
-	add_upstream_commit &&
-	(
-		cd submodule &&
-		(
-			cd subdir/deepsubmodule &&
-			git fetch &&
-			git checkout -q FETCH_HEAD
-		) &&
-		git add subdir/deepsubmodule &&
-		git commit -m "new deepsubmodule" &&
-		new_head=$(git rev-parse --short HEAD) &&
-		write_expected_sub $new_head
-	) &&
+	add_submodule_commits &&
 	(
 		cd downstream &&
 		git config fetch.recurseSubmodules true &&
@@ -402,10 +399,8 @@ test_expect_success "'--recurse-submodules=on-demand' doesn't recurse when no ne
 '
 
 test_expect_success "'--recurse-submodules=on-demand' recurses as deep as necessary (and ignores config)" '
-	git add submodule &&
-	git commit -m "new submodule" &&
-	new_head=$(git rev-parse --short HEAD) &&
-	write_expected_super $new_head &&
+	add_submodule_commits &&
+	add_superproject_commits &&
 	(
 		cd downstream &&
 		git config fetch.recurseSubmodules false &&
@@ -425,7 +420,7 @@ test_expect_success "'--recurse-submodules=on-demand' recurses as deep as necess
 '
 
 test_expect_success "'--recurse-submodules=on-demand' stops when no new submodule commits are found in the superproject (and ignores config)" '
-	add_upstream_commit &&
+	add_submodule_commits &&
 	echo a >> file &&
 	git add file &&
 	git commit -m "new file" &&
@@ -446,7 +441,7 @@ test_expect_success "'fetch.recurseSubmodules=on-demand' overrides global config
 		cd downstream &&
 		git fetch --recurse-submodules
 	) &&
-	add_upstream_commit &&
+	add_submodule_commits &&
 	git config --global fetch.recurseSubmodules false &&
 	git add submodule &&
 	git commit -m "new submodule" &&
@@ -472,7 +467,7 @@ test_expect_success "'submodule.<sub>.fetchRecurseSubmodules=on-demand' override
 		cd downstream &&
 		git fetch --recurse-submodules
 	) &&
-	add_upstream_commit &&
+	add_submodule_commits &&
 	git config fetch.recurseSubmodules false &&
 	git add submodule &&
 	git commit -m "new submodule" &&
@@ -522,7 +517,7 @@ test_expect_success "'fetch.recurseSubmodules=on-demand' works also without .git
 		cd downstream &&
 		git fetch --recurse-submodules
 	) &&
-	add_upstream_commit &&
+	add_submodule_commits &&
 	git add submodule &&
 	git rm .gitmodules &&
 	git commit -m "new submodule without .gitmodules" &&
Signature · 1 line
2.33.GIT
author
#131 lore raw

PATCH v5 4/10 submodule: make static functions read submodules from commits

In reply to Glen Choo

A future commit will teach "fetch --recurse-submodules" to fetch unpopulated submodules. To prepare for this, teach the necessary static functions how to read submodules from superproject commits using a "treeish_name" argument (instead of always reading from the index and filesystem) but do not actually change where submodules are read from. Submodules will be read from commits when we fetch unpopulated submodules.

---
 submodule.c | 29 +++++++++++++++++++----------
 1 file changed, 19 insertions(+), 10 deletions(-)
submodule.c +19 -10
index 5ace18a7d9..4f3300f2cb 100644
@@ -932,6 +932,7 @@ struct has_commit_data {
 	struct repository *repo;
 	int result;
 	const char *path;
+	const struct object_id *super_oid;
 };
 
 static int check_has_commit(const struct object_id *oid, void *data)
@@ -940,7 +941,7 @@ static int check_has_commit(const struct object_id *oid, void *data)
 	struct repository subrepo;
 	enum object_type type;
 
-	if (repo_submodule_init(&subrepo, cb->repo, cb->path, null_oid())) {
+	if (repo_submodule_init(&subrepo, cb->repo, cb->path, cb->super_oid)) {
 		cb->result = 0;
 		goto cleanup;
 	}
@@ -968,9 +969,15 @@ static int check_has_commit(const struct object_id *oid, void *data)
 
 static int submodule_has_commits(struct repository *r,
 				 const char *path,
+				 const struct object_id *super_oid,
 				 struct oid_array *commits)
 {
-	struct has_commit_data has_commit = { r, 1, path };
+	struct has_commit_data has_commit = {
+		.repo = r,
+		.result = 1,
+		.path = path,
+		.super_oid = super_oid
+	};
 
 	/*
 	 * Perform a cheap, but incorrect check for the existence of 'commits'.
@@ -1017,7 +1024,7 @@ static int submodule_needs_pushing(struct repository *r,
 				   const char *path,
 				   struct oid_array *commits)
 {
-	if (!submodule_has_commits(r, path, commits))
+	if (!submodule_has_commits(r, path, null_oid(), commits))
 		/*
 		 * NOTE: We do consider it safe to return "no" here. The
 		 * correct answer would be "We do not know" instead of
@@ -1277,7 +1284,7 @@ static void calculate_changed_submodule_paths(struct repository *r,
 		if (!path)
 			continue;
 
-		if (submodule_has_commits(r, path, commits)) {
+		if (submodule_has_commits(r, path, null_oid(), commits)) {
 			oid_array_clear(commits);
 			*name->string = '\0';
 		}
@@ -1402,12 +1409,13 @@ static const struct submodule *get_non_gitmodules_submodule(const char *path)
 }
 
 static struct fetch_task *fetch_task_create(struct repository *r,
-					    const char *path)
+					    const char *path,
+					    const struct object_id *treeish_name)
 {
 	struct fetch_task *task = xmalloc(sizeof(*task));
 	memset(task, 0, sizeof(*task));
 
-	task->sub = submodule_from_path(r, null_oid(), path);
+	task->sub = submodule_from_path(r, treeish_name, path);
 	if (!task->sub) {
 		/*
 		 * No entry in .gitmodules? Technically not a submodule,
@@ -1439,11 +1447,12 @@ static void fetch_task_release(struct fetch_task *p)
 }
 
 static struct repository *get_submodule_repo_for(struct repository *r,
-						 const char *path)
+						 const char *path,
+						 const struct object_id *treeish_name)
 {
 	struct repository *ret = xmalloc(sizeof(*ret));
 
-	if (repo_submodule_init(ret, r, path, null_oid())) {
+	if (repo_submodule_init(ret, r, path, treeish_name)) {
 		free(ret);
 		return NULL;
 	}
@@ -1464,7 +1473,7 @@ static int get_next_submodule(struct child_process *cp,
 		if (!S_ISGITLINK(ce->ce_mode))
 			continue;
 
-		task = fetch_task_create(spf->r, ce->name);
+		task = fetch_task_create(spf->r, ce->name, null_oid());
 		if (!task)
 			continue;
 
@@ -1487,7 +1496,7 @@ static int get_next_submodule(struct child_process *cp,
 			continue;
 		}
 
-		task->repo = get_submodule_repo_for(spf->r, task->sub->path);
+		task->repo = get_submodule_repo_for(spf->r, task->sub->path, null_oid());
 		if (task->repo) {
 			struct strbuf submodule_prefix = STRBUF_INIT;
 			child_process_init(cp);
Signature · 1 line
2.33.GIT
author
#132 lore raw

PATCH v5 5/10 submodule: inline submodule_commits() into caller

In reply to Glen Choo

When collecting the string_list of changed submodule names, the new submodules commits are stored in the string_list_item.util as an oid_array. A subsequent commit will replace the oid_array with a struct that has more information.

Prepare for this change by inlining submodule_commits() (which inserts into the string_list and initializes the string_list_item.util) into its only caller so that the code is easier to refactor later.

---
 submodule.c | 22 ++++++----------------
 1 file changed, 6 insertions(+), 16 deletions(-)
submodule.c +6 -16
index 4f3300f2cb..3bc189cf05 100644
@@ -782,19 +782,6 @@ const struct submodule *submodule_from_ce(const struct cache_entry *ce)
 	return submodule_from_path(the_repository, null_oid(), ce->name);
 }
 
-static struct oid_array *submodule_commits(struct string_list *submodules,
-					   const char *name)
-{
-	struct string_list_item *item;
-
-	item = string_list_insert(submodules, name);
-	if (item->util)
-		return (struct oid_array *) item->util;
-
-	/* NEEDSWORK: should we have oid_array_init()? */
-	item->util = xcalloc(1, sizeof(struct oid_array));
-	return (struct oid_array *) item->util;
-}
 
 struct collect_changed_submodules_cb_data {
 	struct repository *repo;
@@ -830,9 +817,9 @@ static void collect_changed_submodules_cb(struct diff_queue_struct *q,
 
 	for (i = 0; i < q->nr; i++) {
 		struct diff_filepair *p = q->queue[i];
-		struct oid_array *commits;
 		const struct submodule *submodule;
 		const char *name;
+		struct string_list_item *item;
 
 		if (!S_ISGITLINK(p->two->mode))
 			continue;
@@ -859,8 +846,11 @@ static void collect_changed_submodules_cb(struct diff_queue_struct *q,
 		if (!name)
 			continue;
 
-		commits = submodule_commits(changed, name);
-		oid_array_append(commits, &p->two->oid);
+		item = string_list_insert(changed, name);
+		if (!item->util)
+			/* NEEDSWORK: should we have oid_array_init()? */
+			item->util = xcalloc(1, sizeof(struct oid_array));
+		oid_array_append(item->util, &p->two->oid);
 	}
 }
 
Signature · 1 line
2.33.GIT
author
#133 lore raw

PATCH v5 6/10 submodule: store new submodule commits oid_array in a struct

In reply to Glen Choo

This commit prepares for a future commit that will teach git fetch --recurse-submodules how to fetch submodules that are present in <gitdir>/modules, but are not populated. To do this, we need to store more information about the changed submodule so that we can read the submodule configuration from the superproject commit instead of the filesystem.

Refactor the changed submodules string_list.util to hold a struct instead of an oid_array. This struct only holds the new_commits oid_array for now; more information will be added later.

---
 submodule.c | 52 ++++++++++++++++++++++++++++++++++------------------
 1 file changed, 34 insertions(+), 18 deletions(-)
submodule.c +34 -18
index 3bc189cf05..0b9c25f9d3 100644
@@ -806,6 +806,20 @@ static const char *default_name_or_path(const char *path_or_name)
 	return path_or_name;
 }
 
+/*
+ * Holds relevant information for a changed submodule. Used as the .util
+ * member of the changed submodule string_list_item.
+ */
+struct changed_submodule_data {
+	/* The submodule commits that have changed in the rev walk. */
+	struct oid_array new_commits;
+};
+
+static void changed_submodule_data_clear(struct changed_submodule_data *cs_data)
+{
+	oid_array_clear(&cs_data->new_commits);
+}
+
 static void collect_changed_submodules_cb(struct diff_queue_struct *q,
 					  struct diff_options *options,
 					  void *data)
@@ -820,6 +834,7 @@ static void collect_changed_submodules_cb(struct diff_queue_struct *q,
 		const struct submodule *submodule;
 		const char *name;
 		struct string_list_item *item;
+		struct changed_submodule_data *cs_data;
 
 		if (!S_ISGITLINK(p->two->mode))
 			continue;
@@ -848,9 +863,9 @@ static void collect_changed_submodules_cb(struct diff_queue_struct *q,
 
 		item = string_list_insert(changed, name);
 		if (!item->util)
-			/* NEEDSWORK: should we have oid_array_init()? */
-			item->util = xcalloc(1, sizeof(struct oid_array));
-		oid_array_append(item->util, &p->two->oid);
+			item->util = xcalloc(1, sizeof(struct changed_submodule_data));
+		cs_data = item->util;
+		oid_array_append(&cs_data->new_commits, &p->two->oid);
 	}
 }
 
@@ -897,11 +912,12 @@ static void collect_changed_submodules(struct repository *r,
 	reset_revision_walk();
 }
 
-static void free_submodules_oids(struct string_list *submodules)
+static void free_submodules_data(struct string_list *submodules)
 {
 	struct string_list_item *item;
 	for_each_string_list_item(item, submodules)
-		oid_array_clear((struct oid_array *) item->util);
+		changed_submodule_data_clear(item->util);
+
 	string_list_clear(submodules, 1);
 }
 
@@ -1074,7 +1090,7 @@ int find_unpushed_submodules(struct repository *r,
 	collect_changed_submodules(r, &submodules, &argv);
 
 	for_each_string_list_item(name, &submodules) {
-		struct oid_array *commits = name->util;
+		struct changed_submodule_data *cs_data = name->util;
 		const struct submodule *submodule;
 		const char *path = NULL;
 
@@ -1087,11 +1103,11 @@ int find_unpushed_submodules(struct repository *r,
 		if (!path)
 			continue;
 
-		if (submodule_needs_pushing(r, path, commits))
+		if (submodule_needs_pushing(r, path, &cs_data->new_commits))
 			string_list_insert(needs_pushing, path);
 	}
 
-	free_submodules_oids(&submodules);
+	free_submodules_data(&submodules);
 	strvec_clear(&argv);
 
 	return needs_pushing->nr;
@@ -1261,7 +1277,7 @@ static void calculate_changed_submodule_paths(struct repository *r,
 	collect_changed_submodules(r, changed_submodule_names, &argv);
 
 	for_each_string_list_item(name, changed_submodule_names) {
-		struct oid_array *commits = name->util;
+		struct changed_submodule_data *cs_data = name->util;
 		const struct submodule *submodule;
 		const char *path = NULL;
 
@@ -1274,8 +1290,8 @@ static void calculate_changed_submodule_paths(struct repository *r,
 		if (!path)
 			continue;
 
-		if (submodule_has_commits(r, path, null_oid(), commits)) {
-			oid_array_clear(commits);
+		if (submodule_has_commits(r, path, null_oid(), &cs_data->new_commits)) {
+			changed_submodule_data_clear(cs_data);
 			*name->string = '\0';
 		}
 	}
@@ -1312,7 +1328,7 @@ int submodule_touches_in_range(struct repository *r,
 
 	strvec_clear(&args);
 
-	free_submodules_oids(&subs);
+	free_submodules_data(&subs);
 	return ret;
 }
 
@@ -1596,7 +1612,7 @@ static int fetch_finish(int retvalue, struct strbuf *err,
 	struct fetch_task *task = task_cb;
 
 	struct string_list_item *it;
-	struct oid_array *commits;
+	struct changed_submodule_data *cs_data;
 
 	if (!task || !task->sub)
 		BUG("callback cookie bogus");
@@ -1624,14 +1640,14 @@ static int fetch_finish(int retvalue, struct strbuf *err,
 		/* Could be an unchanged submodule, not contained in the list */
 		goto out;
 
-	commits = it->util;
-	oid_array_filter(commits,
+	cs_data = it->util;
+	oid_array_filter(&cs_data->new_commits,
 			 commit_missing_in_sub,
 			 task->repo);
 
 	/* Are there commits we want, but do not exist? */
-	if (commits->nr) {
-		task->commits = commits;
+	if (cs_data->new_commits.nr) {
+		task->commits = &cs_data->new_commits;
 		ALLOC_GROW(spf->oid_fetch_tasks,
 			   spf->oid_fetch_tasks_nr + 1,
 			   spf->oid_fetch_tasks_alloc);
@@ -1689,7 +1705,7 @@ int fetch_populated_submodules(struct repository *r,
 
 	strvec_clear(&spf.args);
 out:
-	free_submodules_oids(&spf.changed_submodule_names);
+	free_submodules_data(&spf.changed_submodule_names);
 	return spf.result;
 }
 
Signature · 1 line
2.33.GIT
author
#134 lore raw

PATCH v5 7/10 submodule: extract get_fetch_task()

In reply to Glen Choo

get_next_submodule() configures the parallel submodule fetch by performing two functions:

* iterate the index to find submodules
* configure the child processes to fetch the submodules found in the previous step

Extract the index iterating code into an iterator function, get_fetch_task(), so that get_next_submodule() is agnostic of how to find submodules. This prepares for a subsequent commit will teach the fetch machinery to also iterate through the list of changed submodules (in addition to the index).

---
 submodule.c | 61 +++++++++++++++++++++++++++++++----------------------
 1 file changed, 36 insertions(+), 25 deletions(-)
submodule.c +36 -25
index 0b9c25f9d3..7a5316b6f7 100644
@@ -1389,6 +1389,7 @@ struct fetch_task {
 	struct repository *repo;
 	const struct submodule *sub;
 	unsigned free_sub : 1; /* Do we need to free the submodule? */
+	const char *default_argv; /* The default fetch mode. */
 
 	struct oid_array *commits; /* Ensure these commits are fetched */
 };
@@ -1466,14 +1467,11 @@ static struct repository *get_submodule_repo_for(struct repository *r,
 	return ret;
 }
 
-static int get_next_submodule(struct child_process *cp,
-			      struct strbuf *err, void *data, void **task_cb)
+static struct fetch_task *
+get_fetch_task(struct submodule_parallel_fetch *spf, struct strbuf *err)
 {
-	struct submodule_parallel_fetch *spf = data;
-
 	for (; spf->count < spf->r->index->cache_nr; spf->count++) {
 		const struct cache_entry *ce = spf->r->index->cache[spf->count];
-		const char *default_argv;
 		struct fetch_task *task;
 
 		if (!S_ISGITLINK(ce->ce_mode))
@@ -1493,10 +1491,10 @@ static int get_next_submodule(struct child_process *cp,
 					&spf->changed_submodule_names,
 					task->sub->name))
 				continue;
-			default_argv = "on-demand";
+			task->default_argv = "on-demand";
 			break;
 		case RECURSE_SUBMODULES_ON:
-			default_argv = "yes";
+			task->default_argv = "yes";
 			break;
 		case RECURSE_SUBMODULES_OFF:
 			continue;
@@ -1504,29 +1502,12 @@ static int get_next_submodule(struct child_process *cp,
 
 		task->repo = get_submodule_repo_for(spf->r, task->sub->path, null_oid());
 		if (task->repo) {
-			struct strbuf submodule_prefix = STRBUF_INIT;
-			child_process_init(cp);
-			cp->dir = task->repo->gitdir;
-			prepare_submodule_repo_env_in_gitdir(&cp->env_array);
-			cp->git_cmd = 1;
 			if (!spf->quiet)
 				strbuf_addf(err, _("Fetching submodule %s%s\n"),
 					    spf->prefix, ce->name);
-			strvec_init(&cp->args);
-			strvec_pushv(&cp->args, spf->args.v);
-			strvec_push(&cp->args, default_argv);
-			strvec_push(&cp->args, "--submodule-prefix");
-
-			strbuf_addf(&submodule_prefix, "%s%s/",
-						       spf->prefix,
-						       task->sub->path);
-			strvec_push(&cp->args, submodule_prefix.buf);
 
 			spf->count++;
-			*task_cb = task;
-
-			strbuf_release(&submodule_prefix);
-			return 1;
+			return task;
 		} else {
 			struct strbuf empty_submodule_path = STRBUF_INIT;
 
@@ -1550,6 +1531,36 @@ static int get_next_submodule(struct child_process *cp,
 			strbuf_release(&empty_submodule_path);
 		}
 	}
+	return NULL;
+}
+
+static int get_next_submodule(struct child_process *cp, struct strbuf *err,
+			      void *data, void **task_cb)
+{
+	struct submodule_parallel_fetch *spf = data;
+	struct fetch_task *task = get_fetch_task(spf, err);
+
+	if (task) {
+		struct strbuf submodule_prefix = STRBUF_INIT;
+
+		child_process_init(cp);
+		cp->dir = task->repo->gitdir;
+		prepare_submodule_repo_env_in_gitdir(&cp->env_array);
+		cp->git_cmd = 1;
+		strvec_init(&cp->args);
+		strvec_pushv(&cp->args, spf->args.v);
+		strvec_push(&cp->args, task->default_argv);
+		strvec_push(&cp->args, "--submodule-prefix");
+
+		strbuf_addf(&submodule_prefix, "%s%s/",
+						spf->prefix,
+						task->sub->path);
+		strvec_push(&cp->args, submodule_prefix.buf);
+		*task_cb = task;
+
+		strbuf_release(&submodule_prefix);
+		return 1;
+	}
 
 	if (spf->oid_fetch_tasks_nr) {
 		struct fetch_task *task =
Signature · 1 line
2.33.GIT
author
#135 lore raw

PATCH v5 8/10 submodule: move logic into fetch_task_create()

In reply to Glen Choo

get_fetch_task() gets a fetch task by iterating the index; a future commit will introduce a similar function, get_fetch_task_from_changed(), that gets a fetch task from the list of changed submodules. Both functions are similar in that they need to:

* create a fetch task
* initialize the submodule repo for the fetch task
* determine the default recursion mode

Move all of this logic into fetch_task_create() so that it is no longer split between fetch_task_create() and get_fetch_task(). This will make it easier to share code with get_fetch_task_from_changed().

---
 submodule.c | 99 ++++++++++++++++++++++++++++-------------------------
 1 file changed, 52 insertions(+), 47 deletions(-)
submodule.c +52 -47
index 7a5316b6f7..b36ef26752 100644
@@ -1415,32 +1415,6 @@ static const struct submodule *get_non_gitmodules_submodule(const char *path)
 	return (const struct submodule *) ret;
 }
 
-static struct fetch_task *fetch_task_create(struct repository *r,
-					    const char *path,
-					    const struct object_id *treeish_name)
-{
-	struct fetch_task *task = xmalloc(sizeof(*task));
-	memset(task, 0, sizeof(*task));
-
-	task->sub = submodule_from_path(r, treeish_name, path);
-	if (!task->sub) {
-		/*
-		 * No entry in .gitmodules? Technically not a submodule,
-		 * but historically we supported repositories that happen to be
-		 * in-place where a gitlink is. Keep supporting them.
-		 */
-		task->sub = get_non_gitmodules_submodule(path);
-		if (!task->sub) {
-			free(task);
-			return NULL;
-		}
-
-		task->free_sub = 1;
-	}
-
-	return task;
-}
-
 static void fetch_task_release(struct fetch_task *p)
 {
 	if (p->free_sub)
@@ -1467,6 +1441,57 @@ static struct repository *get_submodule_repo_for(struct repository *r,
 	return ret;
 }
 
+static struct fetch_task *fetch_task_create(struct submodule_parallel_fetch *spf,
+					    const char *path,
+					    const struct object_id *treeish_name)
+{
+	struct fetch_task *task = xmalloc(sizeof(*task));
+	memset(task, 0, sizeof(*task));
+
+	task->sub = submodule_from_path(spf->r, treeish_name, path);
+
+	if (!task->sub) {
+		/*
+		 * No entry in .gitmodules? Technically not a submodule,
+		 * but historically we supported repositories that happen to be
+		 * in-place where a gitlink is. Keep supporting them.
+		 */
+		task->sub = get_non_gitmodules_submodule(path);
+		if (!task->sub)
+			goto cleanup;
+
+		task->free_sub = 1;
+	}
+
+	switch (get_fetch_recurse_config(task->sub, spf))
+	{
+	default:
+	case RECURSE_SUBMODULES_DEFAULT:
+	case RECURSE_SUBMODULES_ON_DEMAND:
+		if (!task->sub ||
+			!string_list_lookup(
+				&spf->changed_submodule_names,
+				task->sub->name))
+			goto cleanup;
+		task->default_argv = "on-demand";
+		break;
+	case RECURSE_SUBMODULES_ON:
+		task->default_argv = "yes";
+		break;
+	case RECURSE_SUBMODULES_OFF:
+		goto cleanup;
+	}
+
+	task->repo = get_submodule_repo_for(spf->r, path, treeish_name);
+
+	return task;
+
+ cleanup:
+	fetch_task_release(task);
+	free(task);
+	return NULL;
+}
+
 static struct fetch_task *
 get_fetch_task(struct submodule_parallel_fetch *spf, struct strbuf *err)
 {
@@ -1477,30 +1502,10 @@ get_fetch_task(struct submodule_parallel_fetch *spf, struct strbuf *err)
 		if (!S_ISGITLINK(ce->ce_mode))
 			continue;
 
-		task = fetch_task_create(spf->r, ce->name, null_oid());
+		task = fetch_task_create(spf, ce->name, null_oid());
 		if (!task)
 			continue;
 
-		switch (get_fetch_recurse_config(task->sub, spf))
-		{
-		default:
-		case RECURSE_SUBMODULES_DEFAULT:
-		case RECURSE_SUBMODULES_ON_DEMAND:
-			if (!task->sub ||
-			    !string_list_lookup(
-					&spf->changed_submodule_names,
-					task->sub->name))
-				continue;
-			task->default_argv = "on-demand";
-			break;
-		case RECURSE_SUBMODULES_ON:
-			task->default_argv = "yes";
-			break;
-		case RECURSE_SUBMODULES_OFF:
-			continue;
-		}
-
-		task->repo = get_submodule_repo_for(spf->r, task->sub->path, null_oid());
 		if (task->repo) {
 			if (!spf->quiet)
 				strbuf_addf(err, _("Fetching submodule %s%s\n"),
Signature · 1 line
2.33.GIT
author
#136 lore raw

PATCH v5 9/10 fetch: fetch unpopulated, changed submodules

In reply to Glen Choo

"git fetch --recurse-submodules" only considers populated submodules (i.e. submodules that can be found by iterating the index), which makes "git fetch" behave differently based on which commit is checked out. As a result, even if the user has initialized all submodules correctly, they may not fetch the necessary submodule commits, and commands like "git checkout --recurse-submodules" might fail.

Teach "git fetch" to fetch cloned, changed submodules regardless of whether they are populated. This is in addition to the current behavior of fetching populated submodules (which is always attempted regardless of what was fetched in the superproject, or even if nothing was fetched in the superproject).

A submodule may be encountered multiple times (via the list of populated submodules or via the list of changed submodules). When this happens, "git fetch" only reads the 'populated copy' and ignores the 'changed copy'. Amend the verify_fetch_result() test helper so that we can assert on which 'copy' is being read.

---
 Documentation/fetch-options.txt |  26 ++--
 Documentation/git-fetch.txt     |  10 +-
 builtin/fetch.c                 |  14 +-
 submodule.c                     | 178 +++++++++++++++++++--
 submodule.h                     |  12 +-
 t/t5526-fetch-submodules.sh     | 268 +++++++++++++++++++++++++++++++-
 6 files changed, 462 insertions(+), 46 deletions(-)
Documentation/fetch-options.txt +17 -9
index f903683189..6cdd9d43c5 100644
@@ -186,15 +186,23 @@ endif::git-pull[]
 ifndef::git-pull[]
 --recurse-submodules[=yes|on-demand|no]::
 	This option controls if and under what conditions new commits of
-	populated submodules should be fetched too. It can be used as a
-	boolean option to completely disable recursion when set to 'no' or to
-	unconditionally recurse into all populated submodules when set to
-	'yes', which is the default when this option is used without any
-	value. Use 'on-demand' to only recurse into a populated submodule
-	when the superproject retrieves a commit that updates the submodule's
-	reference to a commit that isn't already in the local submodule
-	clone. By default, 'on-demand' is used, unless
-	`fetch.recurseSubmodules` is set (see linkgit:git-config[1]).
+	submodules should be fetched too. When recursing through submodules,
+	`git fetch` always attempts to fetch "changed" submodules, that is, a
+	submodule that has commits that are referenced by a newly fetched
+	superproject commit but are missing in the local submodule clone. A
+	changed submodule can be fetched as long as it is present locally e.g.
+	in `$GIT_DIR/modules/` (see linkgit:gitsubmodules[7]); if the upstream
+	adds a new submodule, that submodule cannot be fetched until it is
+	cloned e.g. by `git submodule update`.
++
+When set to 'on-demand', only changed submodules are fetched. When set
+to 'yes', all populated submodules are fetched and submodules that are
+both unpopulated and changed are fetched. When set to 'no', submodules
+are never fetched.
++
+When unspecified, this uses the value of `fetch.recurseSubmodules` if it
+is set (see linkgit:git-config[1]), defaulting to 'on-demand' if unset.
+When this option is used without any value, it defaults to 'yes'.
 endif::git-pull[]
 
 -j::
Documentation/git-fetch.txt +4 -6
index 550c16ca61..e9d364669a 100644
@@ -287,12 +287,10 @@ include::transfer-data-leaks.txt[]
 
 BUGS
 ----
-Using --recurse-submodules can only fetch new commits in already checked
-out submodules right now. When e.g. upstream added a new submodule in the
-just fetched commits of the superproject the submodule itself cannot be
-fetched, making it impossible to check out that submodule later without
-having to do a fetch again. This is expected to be fixed in a future Git
-version.
+Using --recurse-submodules can only fetch new commits in submodules that are
+present locally e.g. in `$GIT_DIR/modules/`. If the upstream adds a new
+submodule, that submodule cannot be fetched until it is cloned e.g. by `git
+submodule update`. This is expected to be fixed in a future Git version.
 
 SEE ALSO
 --------
builtin/fetch.c +7 -7
index 95832ba1df..97a89763c8 100644
@@ -2178,13 +2178,13 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
 			max_children = fetch_parallel_config;
 
 		add_options_to_argv(&options);
-		result = fetch_populated_submodules(the_repository,
-						    &options,
-						    submodule_prefix,
-						    recurse_submodules,
-						    recurse_submodules_default,
-						    verbosity < 0,
-						    max_children);
+		result = fetch_submodules(the_repository,
+					  &options,
+					  submodule_prefix,
+					  recurse_submodules,
+					  recurse_submodules_default,
+					  verbosity < 0,
+					  max_children);
 		strvec_clear(&options);
 	}
 
submodule.c +162 -16

289 lines not on this page · raw message

submodule.h +6 -6
index 784ceffc0e..61bebde319 100644
@@ -88,12 +88,12 @@ int should_update_submodules(void);
  */
 const struct submodule *submodule_from_ce(const struct cache_entry *ce);
 void check_for_new_submodule_commits(struct object_id *oid);
-int fetch_populated_submodules(struct repository *r,
-			       const struct strvec *options,
-			       const char *prefix,
-			       int command_line_option,
-			       int default_option,
-			       int quiet, int max_parallel_jobs);
+int fetch_submodules(struct repository *r,
+		     const struct strvec *options,
+		     const char *prefix,
+		     int command_line_option,
+		     int default_option,
+		     int quiet, int max_parallel_jobs);
 unsigned is_submodule_modified(const char *path, int ignore_untracked);
 int submodule_uses_gitfile(const char *path);
 
t/t5526-fetch-submodules.sh +266 -2

299 lines not on this page · raw message

Signature · 1 line
2.33.GIT
author
#137 lore raw

PATCH v5 10/10 submodule: fix latent check_has_commit() bug

In reply to Glen Choo

When check_has_commit() is called on a missing submodule, initialization of the struct repository fails, but it attempts to clear the struct anyway (which is a fatal error). This bug is masked by its only caller, submodule_has_commits(), first calling add_submodule_odb(). The latter fails if the submodule does not exist, making submodule_has_commits() exit early and not invoke check_has_commit().

Fix this bug, and because calling add_submodule_odb() is no longer necessary as of 13a2f620b2 (submodule: pass repo to check_has_commit(), 2021-10-08), remove that call too.

This is the last caller of add_submodule_odb(), so remove that function. (Submodule ODBs are still added as alternates via add_submodule_odb_by_path().)

---
 submodule.c | 35 ++---------------------------------
 submodule.h |  9 ++++-----
 2 files changed, 6 insertions(+), 38 deletions(-)
submodule.c +2 -33
index 1f5f39ce18..6e6b2d04e4 100644
@@ -167,26 +167,6 @@ void stage_updated_gitmodules(struct index_state *istate)
 
 static struct string_list added_submodule_odb_paths = STRING_LIST_INIT_NODUP;
 
-/* TODO: remove this function, use repo_submodule_init instead. */
-int add_submodule_odb(const char *path)
-{
-	struct strbuf objects_directory = STRBUF_INIT;
-	int ret = 0;
-
-	ret = strbuf_git_path_submodule(&objects_directory, path, "objects/");
-	if (ret)
-		goto done;
-	if (!is_directory(objects_directory.buf)) {
-		ret = -1;
-		goto done;
-	}
-	string_list_insert(&added_submodule_odb_paths,
-			   strbuf_detach(&objects_directory, NULL));
-done:
-	strbuf_release(&objects_directory);
-	return ret;
-}
-
 void add_submodule_odb_by_path(const char *path)
 {
 	string_list_insert(&added_submodule_odb_paths, xstrdup(path));
@@ -971,7 +951,8 @@ static int check_has_commit(const struct object_id *oid, void *data)
 
 	if (repo_submodule_init(&subrepo, cb->repo, cb->path, cb->super_oid)) {
 		cb->result = 0;
-		goto cleanup;
+		/* subrepo failed to init, so don't clean it up. */
+		return 0;
 	}
 
 	type = oid_object_info(&subrepo, oid, NULL);
@@ -1007,18 +988,6 @@ static int submodule_has_commits(struct repository *r,
 		.super_oid = super_oid
 	};
 
-	/*
-	 * Perform a cheap, but incorrect check for the existence of 'commits'.
-	 * This is done by adding the submodule's object store to the in-core
-	 * object store, and then querying for each commit's existence.  If we
-	 * do not have the commit object anywhere, there is no chance we have
-	 * it in the object store of the correct submodule and have it
-	 * reachable from a ref, so we can fail early without spawning rev-list
-	 * which is expensive.
-	 */
-	if (add_submodule_odb(path))
-		return 0;
-
 	oid_array_for_each_unique(commits, check_has_commit, &has_commit);
 
 	if (has_commit.result) {
submodule.h +4 -5
index 61bebde319..40c1445237 100644
@@ -103,12 +103,11 @@ int submodule_uses_gitfile(const char *path);
 int bad_to_remove_submodule(const char *path, unsigned flags);
 
 /*
- * Call add_submodule_odb() to add the submodule at the given path to a list.
- * When register_all_submodule_odb_as_alternates() is called, the object stores
- * of all submodules in that list will be added as alternates in
- * the_repository.
+ * Call add_submodule_odb_by_path() to add the submodule at the given
+ * path to a list. When register_all_submodule_odb_as_alternates() is
+ * called, the object stores of all submodules in that list will be
+ * added as alternates in the_repository.
  */
-int add_submodule_odb(const char *path);
 void add_submodule_odb_by_path(const char *path);
 int register_all_submodule_odb_as_alternates(void);
 
Signature · 1 line
2.33.GIT
#138 lore raw
In reply to Glen Choo
Quoted 5 lines · Glen Choo

It's true that we don't need <.super_oid, .path> in order to init the subrepo, but it turns out that recursive fetch reads some configuration values from .gitmodules (via submodule_from_path()), so we still need to store super_oid in order to read the correct .gitmodules file.

OK, but then do we know which .gitmodules file is the "correct" one, when there are more than one .super_oid? Or do we assume that .gitmodules does not change in the range of superproject commits we have fetched before deciding what commits need to be fetched in the submodules?

Quoted 6 lines

== Since v4
- Rename test helpers (s/check_/write_expected_)
- Test style fixes
- Update test comments
- Remove the manual test_cmp in the test that checks sub2 (but we still construct expect.err.super manually).

All of these changes looked sensible.

Will queue. Thanks.

author
#139 lore raw
Quoted 8 lines · Junio C Hamano
Quoted 5 lines · Glen Choo

It's true that we don't need <.super_oid, .path> in order to init the subrepo, but it turns out that recursive fetch reads some configuration values from .gitmodules (via submodule_from_path()), so we still need to store super_oid in order to read the correct .gitmodules file.

Quoted 5 lines

OK, but then do we know which .gitmodules file is the "correct" one, when there are more than one .super_oid? Or do we assume that .gitmodules does not change in the range of superproject commits we have fetched before deciding what commits need to be fetched in the submodules?

This uses a "first one wins approach", which obviously doesn't have correctness guarantees. But in practice, I don't think this is likely to cause problems:

- As far as I can tell, the only value we read from .gitmodules is 'submodule.<name>.fetchRecurseSubmodules', and this value gets overridden by two other values: the CLI option, and the config variable with the same name in .git/config.

During "git submodule init", we copy the config values from .gitmodules to .git/config. Since we can only fetch init-ed submodules anyway, it's quite unlikely that we will ever actually make use of the .gitmodules config.

- Even if we do use the .gitmodules config values, it's unlikely that the values in .gitmodules will change often, so it _probably_ won't matter which one we choose.

- This only matters when the submodule is not in the index. If the submodule _is_ in the index, we read .gitmodules from the filesystem i.e. these patches shouldn't change the behavior for submodules in the index.

#140 lore raw
Quoted 9 lines · Glen Choo

This uses a "first one wins approach", which obviously doesn't have correctness guarantees. But in practice, I don't think this is likely to cause problems:

- As far as I can tell, the only value we read from .gitmodules is 'submodule.<name>.fetchRecurseSubmodules', and this value gets overridden by two other values: the CLI option, and the config variable with the same name in .git/config.

Quoted 4 lines

During "git submodule init", we copy the config values from .gitmodules to .git/config. Since we can only fetch init-ed submodules anyway, it's quite unlikely that we will ever actually make use of the .gitmodules config.

These are reasonable.

Quoted 3 lines

- Even if we do use the .gitmodules config values, it's unlikely that the values in .gitmodules will change often, so it _probably_ won't matter which one we choose.

What bad things would we see if the value changes during the span of history of the superproject we fetched? How often we would see broken behaviour is immaterial and breakage being rare is a no excuse to import a new code with designed-in flaw. Unless the "rare" is "never", that is.

I would think using ANY values from .gitmodules without having the end-user agree with the settings and copying the settings to the .git/config is a BUG. So if it mattered from which superproject commit we took .gitmodules from, that would mean we already have such a bug and it is not a new problem.

That would be a reasonable argument for this topic. Together with the previous point, i.e. we do not copy values we see in the in-tree .gitmodules file to .git/config anyway, it would make a good enough assurance, I would think.

Quoted 4 lines

- This only matters when the submodule is not in the index. If the submodule _is_ in the index, we read .gitmodules from the filesystem i.e. these patches shouldn't change the behavior for submodules in the index.

How often we would see broken behaviour does not matter. If it is broken when the submodule is not in the index, we need to know.

But as you said, it does not sound likely that in-tree .gitmodules matters.

It leads to a possible #leftoverbit clean-up. Because we only fetch submodules that are initialized, the API functions we are using in this series has no reason to require us to feed _a_ commit in the superproject to them so that they can find .gitmodules in them.

Fixing the API can probably be left outside the scope of the topic, to be done soon after the dust from the topic settles, I think, to avoid distracting us from the topic.

Thanks.

author
#141 lore raw
Quoted 22 lines · Junio C Hamano
Quoted 9 lines · Glen Choo

This uses a "first one wins approach", which obviously doesn't have correctness guarantees. But in practice, I don't think this is likely to cause problems:

- As far as I can tell, the only value we read from .gitmodules is 'submodule.<name>.fetchRecurseSubmodules', and this value gets overridden by two other values: the CLI option, and the config variable with the same name in .git/config.

Quoted 4 lines

During "git submodule init", we copy the config values from .gitmodules to .git/config. Since we can only fetch init-ed submodules anyway, it's quite unlikely that we will ever actually make use of the .gitmodules config.

These are reasonable.

Quoted 3 lines

- Even if we do use the .gitmodules config values, it's unlikely that the values in .gitmodules will change often, so it _probably_ won't matter which one we choose.

Quoted 5 lines

What bad things would we see if the value changes during the span of history of the superproject we fetched? How often we would see broken behaviour is immaterial and breakage being rare is a no excuse to import a new code with designed-in flaw. Unless the "rare" is "never", that is.

Makes sense, I'll keep this mind.

Quoted 10 lines

I would think using ANY values from .gitmodules without having the end-user agree with the settings and copying the settings to the .git/config is a BUG. So if it mattered from which superproject commit we took .gitmodules from, that would mean we already have such a bug and it is not a new problem.

That would be a reasonable argument for this topic. Together with the previous point, i.e. we do not copy values we see in the in-tree .gitmodules file to .git/config anyway, it would make a good enough assurance, I would think.

To clarify, does this opinion of "don't use config values that aren't copied into .git/config" extend to in-tree .gitmodules? Prior to this series, we always read the in-tree .gitmodules to get the config - the user does not need to copy the settings to .git/config, but we don't pick a commit to read .gitmodules from.

If we still want to consider in-tree .gitmodules e.g. by merging .git/config and .gitmodules, then we still have the new problem of choosing the right .gitmodules.

If the answer is "no, we don't even consider in-tree .gitmodules" (unless we really have to, like cloning a new submodule), that seems pretty safe and predictable because we wouldn't have to look in two different places to figure out what the user wants. And more crucially, we'd never have to guess which .gitmodules to read - which will become more of an issue as we add more support for init-ed but unpopulated submodules.

Quoted 4 lines

It leads to a possible #leftoverbit clean-up. Because we only fetch submodules that are initialized, the API functions we are using in this series has no reason to require us to feed _a_ commit in the superproject to them so that they can find .gitmodules in them.

Hm, this is true; an initialized submodule should already have the 'expected' information in .git/config. And if we no longer have to fret about whether we're reading the correct .gitmodules, we can revisit the idea of "init a subrepo using only its name".

Quoted 5 lines

Fixing the API can probably be left outside the scope of the topic, to be done soon after the dust from the topic settles, I think, to avoid distracting us from the topic.

Thanks.

#142 lore raw
Quoted 5 lines · Glen Choo

To clarify, does this opinion of "don't use config values that aren't copied into .git/config" extend to in-tree .gitmodules? Prior to this series, we always read the in-tree .gitmodules to get the config - the user does not need to copy the settings to .git/config, but we don't pick a commit to read .gitmodules from.

I think we do, but I also think it was a huge mistake to allow repository data to directly affect the behaviour of local checkout.

Fixing that is most likely outside the scope of this series, though.

author
#143 lore raw
Quoted 10 lines · Junio C Hamano
Quoted 5 lines · Glen Choo

To clarify, does this opinion of "don't use config values that aren't copied into .git/config" extend to in-tree .gitmodules? Prior to this series, we always read the in-tree .gitmodules to get the config - the user does not need to copy the settings to .git/config, but we don't pick a commit to read .gitmodules from.

I think we do, but I also think it was a huge mistake to allow repository data to directly affect the behaviour of local checkout.

I'm inclined to agree.

Quoted 1 line

Fixing that is most likely outside the scope of this series, though.

Agree. Thanks!

author
#144 lore raw
Quoted 10 lines · Glen Choo
Quoted 8 lines · Junio C Hamano
Quoted 5 lines · Glen Choo

To clarify, does this opinion of "don't use config values that aren't copied into .git/config" extend to in-tree .gitmodules? Prior to this series, we always read the in-tree .gitmodules to get the config - the user does not need to copy the settings to .git/config, but we don't pick a commit to read .gitmodules from.

Quoted 8 lines
Quoted 2 lines

I think we do, but I also think it was a huge mistake to allow repository data to directly affect the behaviour of local checkout.

I'm inclined to agree.

Quoted 1 line

Fixing that is most likely outside the scope of this series, though.

Agree. Thanks!

I thought that this would have been the end of the discussion, but after reading <[email protected]>, I guess I had the wrong impression (oops).

If I am reading everything correctly, we both agree that it's not good to read _any_ config values from .gitmodules (even if it's in-tree), and that we should clean it up outside of this topic. So for this topic to be merged into 'next', is it enough to say that I will fix this behavior in a follow up topic?

#145 lore raw
Quoted 25 lines · Glen Choo
Quoted 10 lines · Glen Choo
Quoted 8 lines · Junio C Hamano
Quoted 5 lines · Glen Choo

To clarify, does this opinion of "don't use config values that aren't copied into .git/config" extend to in-tree .gitmodules? Prior to this series, we always read the in-tree .gitmodules to get the config - the user does not need to copy the settings to .git/config, but we don't pick a commit to read .gitmodules from.

Quoted 8 lines
Quoted 2 lines

I think we do, but I also think it was a huge mistake to allow repository data to directly affect the behaviour of local checkout.

I'm inclined to agree.

Quoted 1 line

Fixing that is most likely outside the scope of this series, though.

Agree. Thanks!

I thought that this would have been the end of the discussion, but after reading <[email protected]>, I guess I had the wrong impression (oops).

Quoted 5 lines

If I am reading everything correctly, we both agree that it's not good to read _any_ config values from .gitmodules (even if it's in-tree), and that we should clean it up outside of this topic. So for this topic to be merged into 'next', is it enough to say that I will fix this behavior in a follow up topic?

At least we should remember that is something to be fixed. It may not be you personally who addresses that issue, though ;-)

author
#146 lore raw
Quoted 27 lines · Junio C Hamano
Quoted 25 lines · Glen Choo
Quoted 10 lines · Glen Choo
Quoted 8 lines · Junio C Hamano
Quoted 5 lines · Glen Choo

To clarify, does this opinion of "don't use config values that aren't copied into .git/config" extend to in-tree .gitmodules? Prior to this series, we always read the in-tree .gitmodules to get the config - the user does not need to copy the settings to .git/config, but we don't pick a commit to read .gitmodules from.

Quoted 8 lines
Quoted 2 lines

I think we do, but I also think it was a huge mistake to allow repository data to directly affect the behaviour of local checkout.

I'm inclined to agree.

Quoted 1 line

Fixing that is most likely outside the scope of this series, though.

Agree. Thanks!

I thought that this would have been the end of the discussion, but after reading <[email protected]>, I guess I had the wrong impression (oops).

Quoted 8 lines
Quoted 5 lines

If I am reading everything correctly, we both agree that it's not good to read _any_ config values from .gitmodules (even if it's in-tree), and that we should clean it up outside of this topic. So for this topic to be merged into 'next', is it enough to say that I will fix this behavior in a follow up topic?

At least we should remember that is something to be fixed. It may not be you personally who addresses that issue, though ;-)

Perhaps squashing in a NEEDSWORK comment into [PATCH v5 09/10] will suffice? I can also resend this series if preferred.

----- >8 --------- >8 --------- >8 --------- >8 --------- >8 ----

submodule.c +15 -0
index 6e6b2d04e4..93c78a4dc3 100644
@@ -795,6 +795,21 @@ static const char *default_name_or_path(const char *path_or_name)
  * superproject commit that points to the submodule, but this is
  * arbitrary - we can choose any (super_oid, path) that matches the
  * submodule's name.
+ *
+ * NEEDSWORK: Storing an arbitrary commit is undesirable because we can't
+ * guarantee that we're reading the commit that the user would expect. A better
+ * scheme would be to just fetch a submodule by its name. This requires two
+ * steps:
+ * - Create a function that behaves like repo_submodule_init(), but accepts a
+ *   submodule name instead of treeish_name and path. This should be easy
+ *   because repo_submodule_init() internally uses the submodule's name.
+ *
+ * - Replace most instances of 'struct submodule' (which is the .gitmodules
+ *   config) with just the submodule name. This is OK because we expect
+ *   submodule settings to be stored in .git/config (via "git submodule init"),
+ *   not .gitmodules. This also lets us delete get_non_gitmodules_submodule(),
+ *   which constructs a bogus 'struct submodule' for the sake of giving a
+ *   placeholder name to a gitlink.
  */
 struct changed_submodule_data {
 	/*
#147 lore raw
Quoted 2 lines · Glen Choo

Perhaps squashing in a NEEDSWORK comment into [PATCH v5 09/10] will suffice? I can also resend this series if preferred.

It should work. Let me try it in the last integration cycle of today.

Quoted 28 lines

----- >8 --------- >8 --------- >8 --------- >8 --------- >8 ----

submodule.c +15 -0
index 6e6b2d04e4..93c78a4dc3 100644
@@ -795,6 +795,21 @@ static const char *default_name_or_path(const char *path_or_name)
  * superproject commit that points to the submodule, but this is
  * arbitrary - we can choose any (super_oid, path) that matches the
  * submodule's name.
+ *
+ * NEEDSWORK: Storing an arbitrary commit is undesirable because we can't
+ * guarantee that we're reading the commit that the user would expect. A better
+ * scheme would be to just fetch a submodule by its name. This requires two
+ * steps:
+ * - Create a function that behaves like repo_submodule_init(), but accepts a
+ *   submodule name instead of treeish_name and path. This should be easy
+ *   because repo_submodule_init() internally uses the submodule's name.
+ *
+ * - Replace most instances of 'struct submodule' (which is the .gitmodules
+ *   config) with just the submodule name. This is OK because we expect
+ *   submodule settings to be stored in .git/config (via "git submodule init"),
+ *   not .gitmodules. This also lets us delete get_non_gitmodules_submodule(),
+ *   which constructs a bogus 'struct submodule' for the sake of giving a
+ *   placeholder name to a gitlink.
  */
 struct changed_submodule_data {
 	/*
Reviewed-by on the v5 series
#148 lore raw
In reply to Glen Choo
Quoted 5 lines · Glen Choo

- <[email protected]> I've described the differences between the no-submodule-in-index test and the other-submodule-in-index test (their comments now refer to one another, so the contrast is more obvious), but didn't reorder them because I thought that made the test setup less intuitive to read.

Thanks - the comments make sense.

Quoted 8 lines

- <[email protected]> I added expect.err.sub2 to verify_test_result() but didn't change write_expected_super() to account for sub2. It turned out to be tricky to predict the output when 'super' fetches >1 branch because each fetched branch can affect the formatting. e.g.

    	   OLD_HEAD..super  super           -> origin/super
Quoted 7 lines

can become

    	   OLD_HEAD..super  super                   -> origin/super
    	   OLD_HEAD..super  some-other-branch       -> origin/some-other-branch

(I could work around this by replacing the whitespace with sed, but it seemed like too much overhead for a one-off test).

Overwriting just the super part works for me, thanks.

The only thing remaining from me is my comment about fetching OIDs from one submodule into another (of the same name but different URL) [1], but I looked into it myself and we can probably postpone handling this to another patch set.

In such a patch set, we would probably need to store the URLs that are reported by upstream .gitmodules somewhere. (I forgot that we don't use them in this patch.) And then, either implement an autosync function (like "git submodule sync", perhaps gated by a "--sync-submodules" argument so that users can include it when fetching new commits and exclude it when fetching historical commits) and/or use those URLs in a diagnostic message to be printed when the fetch fails.

As it is, the existing fetch-into-submodules-at-HEAD also suffers from the same flaw, so I'm OK postponing this to another patch set.

So,

[1] https://lore.kernel.org/git/[email protected]/