Skip to contentSkip to conversation

CI: run "make [test]" directly, use $GITHUB_ENV

PATCH Superseded Ævar Arnfjörð Bjarmason 9 participants 260 messages Feb 21, 2022 – May 25, 2022 Kbuild Page 2 of 3

lore
Share & tools
lore
#122 lore raw

PATCH v3 8/29 CI: have "static-analysis" run a "make ci-static-analysis" target

In reply to Ævar Arnfjörð Bjarmason

Make it easier to run the CI logic locally by creating "ci-static-analysis" target.

Now the "check-coccicheck" logic added in 0860a7641ba (travis-ci: fail if Coccinelle static analysis found something to transform, 2018-07-23) can be easily tested outside of CI, and likewise the sanity check added in 0e7696c64db (ci: disallow directional formatting, 2021-11-04).

These targets could be improved, the "hdr-check" target will re-do all of its work every time it's run, and could with a change similar to my c234e8a0ecf (Makefile: make the "sparse" target non-.PHONY, 2021-09-23). The same goes for the new "ci-check-directional-formatting" target.

But without those improvements being able to easily run a 1=1 equivalent of the checks we do in CI during a local build is already an improvement. Subsequent commits will expand on this ability.

This change will also make the CI check faster, since we can take advantage of parallelism. The "make coccicheck" takes a very long time, and at the tail end we might only have one job pegging 100% of one CPU core for a long time. Now any extra cores will be free to run the rest of the targets under "ci-static-analysis".

---
 .github/workflows/main.yml |  3 +--
 Makefile                   | 28 ++++++++++++++++++++++++++++
 ci/run-static-analysis.sh  | 30 ------------------------------
 shared.mak                 |  1 +
 4 files changed, 30 insertions(+), 32 deletions(-)
 delete mode 100755 ci/run-static-analysis.sh
.github/workflows/main.yml +1 -2
index 309b9141249..ad752010102 100644
@@ -317,8 +317,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/run-static-analysis.sh
-    - run: ci/check-directional-formatting.bash
+    - run: make ci-static-analysis
   sparse:
     needs: ci-config
     if: needs.ci-config.outputs.enabled == 'yes'
Makefile +28 -0
index f8bccfab5e9..8717ea7e745 100644
@@ -3031,6 +3031,20 @@ coccicheck: $(addsuffix .patch,$(filter-out %.pending.cocci,$(wildcard contrib/c
 # See contrib/coccinelle/README
 coccicheck-pending: $(addsuffix .patch,$(wildcard contrib/coccinelle/*.pending.cocci))
 
+.PHONY: check-coccicheck
+check-coccicheck: coccicheck
+	$(QUIET_CHECK)for cocci_patch in contrib/coccinelle/*.patch; do \
+		if test -s "$$cocci_patch"; then \
+			echo "Coccinelle suggests the following changes in '$$cocci_patch':"; \
+			cat "$$cocci_patch"; \
+			fail=UnfortunatelyYes; \
+		fi \
+	done; \
+	if test -n "$$fail"; then \
+		echo "error: Coccinelle suggested some changes"; \
+		exit 1; \
+	fi
+
 .PHONY: coccicheck coccicheck-pending
 
 ### Installation rules
@@ -3466,3 +3480,17 @@ $(FUZZ_PROGRAMS): all
 		$(XDIFF_OBJS) $(EXTLIBS) git.o [email protected] $(LIB_FUZZING_ENGINE) -o $@
 
 fuzz-all: $(FUZZ_PROGRAMS)
+
+### CI "check" targets
+#
+# These targets are run from the CI, see .github/workflows/main.yml,
+# but can also be run manually to run the same assertions locally.
+
+.PHONY: ci-check-directional-formatting
+ci-check-directional-formatting:
+	$(QUIET_CHECK)ci/check-directional-formatting.bash
+
+.PHONY: ci-static-analysis
+ci-static-analysis: ci-check-directional-formatting
+ci-static-analysis: check-coccicheck
+ci-static-analysis: hdr-check
ci/run-static-analysis.sh deleted +0 -30
deleted file mode 100755
index 5a87b1cac96..00000000000
@@ -1,30 +0,0 @@
-#!/bin/sh
-#
-# Perform various static code analysis checks
-#
-
-. ${0%/*}/lib.sh
-
-make coccicheck
-
-set +x
-
-fail=
-for cocci_patch in contrib/coccinelle/*.patch
-do
-	if test -s "$cocci_patch"
-	then
-		echo "$(tput setaf 1)Coccinelle suggests the following changes in '$cocci_patch':$(tput sgr0)"
-		cat "$cocci_patch"
-		fail=UnfortunatelyYes
-	fi
-done
-
-if test -n "$fail"
-then
-	echo "$(tput setaf 1)error: Coccinelle suggested some changes$(tput sgr0)"
-	exit 1
-fi
-
-make hdr-check ||
-exit 1
shared.mak +1 -0
index 50d4596f0d6..83e0aa16e7d 100644
@@ -68,6 +68,7 @@ ifndef V
 	QUIET_HDR      = @echo '   ' HDR $(<:hcc=h);
 	QUIET_RC       = @echo '   ' RC $@;
 	QUIET_SPATCH   = @echo '   ' SPATCH $<;
+	QUIET_CHECK    = @echo '   ' CHECK $@;
 
 ## Used in "Documentation/Makefile"
 	QUIET_ASCIIDOC	= @echo '   ' ASCIIDOC $@;
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#123 lore raw

PATCH v3 9/29 CI: have "static-analysis" run "check-builtins", not "documentation"

In reply to Ævar Arnfjörð Bjarmason

Move the "make check-builtins" check from the "documentation" job to the "static-analysis" job.

The "check-builtins" target added in c74390e4a1d (cherry is built-in, do not ship git-cherry.sh, 2006-11-05) is unrelated to the documentation, so it's odd that b98712b9aa9 (travis-ci: build documentation, 2016-05-04) added it to the "documentation" job.

Let's just move it to the "ci-static-analysis" target, and while we're at it improve the output with $(QUIET_CHECK).

---
 Makefile                 | 3 ++-
 ci/test-documentation.sh | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)
Makefile +2 -1
index 8717ea7e745..2e1ae6c6067 100644
@@ -3402,7 +3402,7 @@ check-docs::
 ### Make sure built-ins do not have dups and listed in git.c
 #
 check-builtins::
-	./check-builtins.sh
+	$(QUIET_CHECK)./check-builtins.sh
 
 ### Test suite coverage testing
 #
@@ -3492,5 +3492,6 @@ ci-check-directional-formatting:
 
 .PHONY: ci-static-analysis
 ci-static-analysis: ci-check-directional-formatting
+ci-static-analysis: check-builtins
 ci-static-analysis: check-coccicheck
 ci-static-analysis: hdr-check
ci/test-documentation.sh +0 -1
index 9e0652c30dd..41e2b126311 100755
@@ -14,7 +14,6 @@ filter_log () {
 	    "$1"
 }
 
-make check-builtins
 make check-docs
 
 # Build docs with AsciiDoc
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#124 lore raw

PATCH v3 10/29 CI: move p4 and git-lfs variables to ci/install-dependencies.sh

In reply to Ævar Arnfjörð Bjarmason

Move the declaration of variables that are only used by the "ubuntu-latest" block in "ci/install-dependencies.sh" there from "ci/lib.sh".

This makes the code easier to follow, and changes "ci/lib.sh" to a library that only exports CI variables for general use, we didn't need to export these $P4_PATH and $GIT_LFS_PATH variables.

---
 ci/install-dependencies.sh | 17 +++++++++++++++--
 ci/lib.sh                  | 12 ------------
 2 files changed, 15 insertions(+), 14 deletions(-)
ci/install-dependencies.sh +15 -2
index e7ea8799411..34b4400f38b 100755
@@ -5,14 +5,27 @@
 
 . ${0%/*}/lib.sh
 
-P4WHENCE=http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION
-LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION
 UBUNTU_COMMON_PKGS="make libssl-dev libcurl4-openssl-dev libexpat-dev
  tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl
  libemail-valid-perl libio-socket-ssl-perl libnet-smtp-ssl-perl"
 
 case "$runs_on_pool" in
 ubuntu-latest)
+	# The Linux build installs the defined dependency versions below.
+	# The OS X build installs much more recent versions, whichever
+	# were recorded in the Homebrew database upon creating the OS X
+	# image.
+	# Keep that in mind when you encounter a broken OS X build!
+	LINUX_P4_VERSION="16.2"
+	LINUX_GIT_LFS_VERSION="1.5.2"
+
+	P4_PATH="$HOME/custom/p4"
+	GIT_LFS_PATH="$HOME/custom/git-lfs"
+	export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH"
+
+	P4WHENCE=http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION
+	LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION
+
 	sudo apt-get -q update
 	sudo apt-get -q -y install language-pack-is libsvn-perl apache2 \
 		$UBUNTU_COMMON_PKGS $CC_PACKAGE
ci/lib.sh +0 -12
index 21c009757f4..5736a4d2e04 100755
@@ -67,18 +67,6 @@ ubuntu-latest)
 	fi
 
 	export GIT_TEST_HTTPD=true
-
-	# The Linux build installs the defined dependency versions below.
-	# The OS X build installs much more recent versions, whichever
-	# were recorded in the Homebrew database upon creating the OS X
-	# image.
-	# Keep that in mind when you encounter a broken OS X build!
-	export LINUX_P4_VERSION="16.2"
-	export LINUX_GIT_LFS_VERSION="1.5.2"
-
-	P4_PATH="$HOME/custom/p4"
-	GIT_LFS_PATH="$HOME/custom/git-lfs"
-	export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH"
 	;;
 macos-latest)
 	if [ "$jobname" = osx-gcc ]
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#125 lore raw

PATCH v3 11/29 CI: consistently use "export" in ci/lib.sh

In reply to Ævar Arnfjörð Bjarmason

Change the "ci/lib.sh" script co consistently use "export", for e.g. MAKEFLAGS we were exporting it, and then assigning to it, let's do it the other way around.

Right now this doesn't matter, since we in
e.g. "ci/install-dependencies.sh" source this file, and don't use something like "env(1)" to retrieve these variables. But in a subsequent commit we'll "export" these variables through a wrapper (to additionally write them to a GitHub CI-specific $GITHUB_ENV file). This change makes that subsequent change easier to read, as it won't need to do any control flow refactoring.

---
 ci/lib.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)
ci/lib.sh +4 -3
index 5736a4d2e04..5783eed1b20 100755
@@ -30,7 +30,7 @@ export TERM=${TERM:-dumb}
 NPROC=10
 
 # Clear MAKEFLAGS that may come from the outside world.
-export MAKEFLAGS=--jobs=$NPROC
+MAKEFLAGS=--jobs=$NPROC
 
 if test "$GITHUB_ACTIONS" = "true"
 then
@@ -38,9 +38,10 @@ then
 	CC="${CC:-gcc}"
 
 	export GIT_PROVE_OPTS="--timer --jobs $NPROC"
-	export GIT_TEST_OPTS="--verbose-log -x"
+	GIT_TEST_OPTS="--verbose-log -x"
 	test Windows != "$RUNNER_OS" ||
 	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
+	export GIT_TEST_OPTS
 else
 	echo "Could not identify CI type" >&2
 	env >&2
@@ -94,4 +95,4 @@ linux-leaks)
 	;;
 esac
 
-MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}"
+export MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}"
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#126 lore raw

PATCH v3 12/29 CI: export variables via a wrapper

In reply to Ævar Arnfjörð Bjarmason

Add and use a "setenv" wrapper to export the variables set in ci/lib.sh.

For now this merely adds debugging information about what variables are set, but in a subsequent commit we'll expand this to set variables in $GITHUB_ENV instead of the environment, and thus carry variable settings between steps in a job.

We'll also eventually be interested in what type of variable we're setting, and when it needs to be used, because we'll prominently expose what variables we set in what CI "step", so having only the ones we actually use will improve readability.

To that end call the "setenv" function with optional "--build", "--test" and "--all" parameters to indicate whether they're needed for building, testing or both.

We currently ignore the "varname" parameter, but will make use of it in a subsequent commit. Adding it here reduces the needless churn, as we won't need to change all these lines again for that subsequent change.

---
 ci/lib.sh                 | 51 ++++++++++++++++++++++++++++++---------
 ci/run-build-and-tests.sh | 34 +++++++++++++-------------
 2 files changed, 57 insertions(+), 28 deletions(-)
ci/lib.sh +40 -11
index 5783eed1b20..3727e6ec2f9 100755
@@ -13,6 +13,35 @@ then
 	exit 1
 fi
 
+# Helper functions
+setenv () {
+	while test $# != 0
+	do
+		case "$1" in
+		--build)
+			;;
+		--test)
+			;;
+		--all)
+			;;
+		-*)
+			echo "BUG: bad setenv() option '$1'" >&2
+			exit 1
+			;;
+		*)
+			break
+			;;
+		esac
+		shift
+	done
+
+	key=$1
+	val=$2
+	shift 2
+
+	eval "export $key=\"$val\""
+}
+
 check_unignored_build_artifacts ()
 {
 	! git ls-files --other --exclude-standard --error-unmatch \
@@ -24,7 +53,7 @@ check_unignored_build_artifacts ()
 }
 
 # GitHub Action doesn't set TERM, which is required by tput
-export TERM=${TERM:-dumb}
+setenv TERM ${TERM:-dumb}
 
 # How many jobs to run in parallel?
 NPROC=10
@@ -37,21 +66,21 @@ then
 	CI_TYPE=github-actions
 	CC="${CC:-gcc}"
 
-	export GIT_PROVE_OPTS="--timer --jobs $NPROC"
+	setenv --test GIT_PROVE_OPTS "--timer --jobs $NPROC"
 	GIT_TEST_OPTS="--verbose-log -x"
 	test Windows != "$RUNNER_OS" ||
 	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
-	export GIT_TEST_OPTS
+	setenv --test GIT_TEST_OPTS "$GIT_TEST_OPTS"
 else
 	echo "Could not identify CI type" >&2
 	env >&2
 	exit 1
 fi
 
-export DEVELOPER=1
-export DEFAULT_TEST_TARGET=prove
-export GIT_TEST_CLONE_2GB=true
-export SKIP_DASHED_BUILT_INS=YesPlease
+setenv --build DEVELOPER 1
+setenv --test DEFAULT_TEST_TARGET prove
+setenv --test GIT_TEST_CLONE_2GB true
+setenv --build SKIP_DASHED_BUILT_INS YesPlease
 
 case "$runs_on_pool" in
 ubuntu-latest)
@@ -67,7 +96,7 @@ ubuntu-latest)
 		MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python2"
 	fi
 
-	export GIT_TEST_HTTPD=true
+	setenv --test GIT_TEST_HTTPD true
 	;;
 macos-latest)
 	if [ "$jobname" = osx-gcc ]
@@ -90,9 +119,9 @@ linux-musl)
 	MAKEFLAGS="$MAKEFLAGS GIT_TEST_UTF8_LOCALE=C.UTF-8"
 	;;
 linux-leaks)
-	export SANITIZE=leak
-	export GIT_TEST_PASSING_SANITIZE_LEAK=true
+	setenv --build SANITIZE leak
+	setenv --test GIT_TEST_PASSING_SANITIZE_LEAK true
 	;;
 esac
 
-export MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}"
+setenv --all MAKEFLAGS "$MAKEFLAGS CC=${CC:-cc}"
ci/run-build-and-tests.sh +17 -17
index e5a21985b44..35d45a9373e 100755
@@ -9,33 +9,33 @@ export MAKE_TARGETS="all test"
 
 case "$jobname" in
 linux-gcc)
-	export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
+	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME main
 	;;
 linux-TEST-vars)
-	export GIT_TEST_SPLIT_INDEX=yes
-	export GIT_TEST_MERGE_ALGORITHM=recursive
-	export GIT_TEST_FULL_IN_PACK_ARRAY=true
-	export GIT_TEST_OE_SIZE=10
-	export GIT_TEST_OE_DELTA_SIZE=5
-	export GIT_TEST_COMMIT_GRAPH=1
-	export GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS=1
-	export GIT_TEST_MULTI_PACK_INDEX=1
-	export GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=1
-	export GIT_TEST_ADD_I_USE_BUILTIN=1
-	export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master
-	export GIT_TEST_WRITE_REV_INDEX=1
-	export GIT_TEST_CHECKOUT_WORKERS=2
+	setenv --test GIT_TEST_SPLIT_INDEX yes
+	setenv --test GIT_TEST_MERGE_ALGORITHM recursive
+	setenv --test GIT_TEST_FULL_IN_PACK_ARRAY true
+	setenv --test GIT_TEST_OE_SIZE 10
+	setenv --test GIT_TEST_OE_DELTA_SIZE 5
+	setenv --test GIT_TEST_COMMIT_GRAPH 1
+	setenv --test GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS 1
+	setenv --test GIT_TEST_MULTI_PACK_INDEX 1
+	setenv --test GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP 1
+	setenv --test GIT_TEST_ADD_I_USE_BUILTIN 1
+	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME master
+	setenv --test GIT_TEST_WRITE_REV_INDEX 1
+	setenv --test GIT_TEST_CHECKOUT_WORKERS 2
 	;;
 linux-clang)
-	export GIT_TEST_DEFAULT_HASH=sha1
+	setenv --test GIT_TEST_DEFAULT_HASH sha1
 	;;
 linux-sha256)
-	export GIT_TEST_DEFAULT_HASH=sha256
+	setenv --test GIT_TEST_DEFAULT_HASH sha256
 	;;
 pedantic)
 	# Don't run the tests; we only care about whether Git can be
 	# built.
-	export DEVOPTS=pedantic
+	setenv --build DEVOPTS pedantic
 	export MAKE_TARGETS=all
 	;;
 esac
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#127 lore raw

PATCH v3 13/29 CI: remove "run-build-and-tests.sh", run "make [test]" directly

In reply to Ævar Arnfjörð Bjarmason

Remove the already thin "ci/run-build-and-tests.sh" wrapper and instead make the CI run "make" or "make test" directly. By doing this we'll be able to easily see at a glance whether our failure was in the compilation or testing, whether that's via human eyes or improve machine readability.

We also need to run our new "ci/check-unignored-build-artifacts.sh" on success() in the CI now, just like we already had a step conditional on failure() running ci/print-test-failures.sh.

The reason we used a "ci/run-build-and-tests.sh" wrapper in the first place had to do with Travis CI-specific constraints that no longer apply to us, as the Travis CI support has been removed.

Instead we can configure the CI in an earlier step by running "ci/lib.sh", which under GitHub CI will write the environment variables we need to the "$GITHUB_ENV" file.

We'll then have access to them in subsequent steps, and crucially those variables will be prominently visible at the start of each step via an expandable drop-down in the UI.drop-do.

I.e. this changes the CI run from a top-down flow like (pseudocode):

- job:
- step1:
- use ci/lib.sh to set env vars
- run a script like ci/run-build-and-tests.sh
- step2:
- if: failure()
- use ci/lib.sh to set env vars
- run ci/print-test-failures.sh

To:

- job:
- step1:
- set variables in $GITHUB_ENV using ci/lib.sh
- step2:
- make
- step3:
- make test
- step4:
- if: failure()
- run ci/print-test-failures.sh
- step5:
- if: success()
- run ci/check-unignored-build-artifacts.sh

There is a proposal[2] to get some of the benefits of this approach by not re-arranging our variable setup in this way, but to instead use the GitHub CI grouping syntax to focus on the relevant parts of "make" or "make test" when we have failures.

Doing it this way makes for better looking GitHub CI UI, and lays much better ground work for our CI going forward. Because:

* The CI logic will be more portable to a future CI system, since a common feature of them is to run various commands in sequence, but a future system won't necessarily support the GitHub-specifics syntax of "grouping" output within a "step".

Even if those systems don't support a "$GITHUB_ENV" emulating will be much easier than to deal with some CI-specific grouping syntax.

* At the start of every step the GitHub CI presents an expandable list of environment variables from "$GITHUB_ENV". We'll now see exactly what variables affected that step (although we currently overshoot that a bit, and always define all variables).

* CI failures will be easier to reproduce locally, as this makes the relevant ci/* scripts something that sets up our environment, but leaves "make" and "make test" working as they do locally.

To reproduce a run the user only needs to set the variables discussed in the drop-down above, either manually or by running "ci/lib.sh".

* The output will be less verbose. The "ci/lib.sh" script uses "set -x", and before this e.g. "ci/static-analysis.sh" would start with 40 lines of trace output, culminating in using "export" to export the relevant environment variables.

Now that verbosity is in the earlier "ci/lib.sh" step, and not in any subsequent one. The "make" targets then start out with the relevant output non-trace output right away.

* If we do want to use the grouping syntax within a "step" it'll now be easier to do so. It doesn't support nesting, so we'd have to make a choice between using it for e.g. "make" v.s. "make test", or individual test failures. See "sadly" in [3].

1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/
3. https://lore.kernel.org/git/9333ba781b8240f704e739b00d274f8c3d887e39.1643050574.git.gitgitgadget@gmail.com/

---
 .github/workflows/main.yml            | 14 ++++++--
 ci/check-unignored-build-artifacts.sh |  8 +++++
 ci/install-dependencies.sh            |  4 +++
 ci/lib.sh                             | 36 +++++++++++++++++++-
 ci/make-test-artifacts.sh             |  2 --
 ci/run-build-and-tests.sh             | 47 ---------------------------
 ci/run-test-slice.sh                  |  2 --
 ci/test-documentation.sh              |  1 -
 8 files changed, 58 insertions(+), 56 deletions(-)
 create mode 100755 ci/check-unignored-build-artifacts.sh
 delete mode 100755 ci/run-build-and-tests.sh
.github/workflows/main.yml +11 -3
index ad752010102..c83f6a21ca3 100644
@@ -268,7 +268,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/run-build-and-tests.sh
+    - run: ci/lib.sh
+    - run: make
+    - run: make test
+      if: success()
     - run: ci/print-test-failures.sh
       if: failure()
     - name: Upload failed tests' directories
@@ -292,6 +295,7 @@ jobs:
           image: daald/ubuntu32:xenial
         - jobname: pedantic
           image: fedora
+          skip-tests: no
     env:
       jobname: ${{matrix.vector.jobname}}
     runs-on: ubuntu-latest
@@ -299,9 +303,12 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - run: ci/install-docker-dependencies.sh
-    - run: ci/run-build-and-tests.sh
+    - run: ci/lib.sh
+    - run: make
+    - run: make test
+      if: success() && matrix.vector.skip-tests != 'no'
     - run: ci/print-test-failures.sh
-      if: failure()
+      if: failure() && matrix.vector.skip-tests != 'no'
     - name: Upload failed tests' directories
       if: failure() && env.FAILED_TEST_ARTIFACTS != ''
       uses: actions/upload-artifact@v1
@@ -317,6 +324,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
+    - run: ci/lib.sh
     - run: make ci-static-analysis
   sparse:
     needs: ci-config
ci/check-unignored-build-artifacts.sh new file +8 -0
new file mode 100755
index 00000000000..56d04b0db9a
@@ -0,0 +1,8 @@
+#!/bin/sh
+#
+# Check whether the build created anything not in our .gitignore
+#
+
+. ${0%/*}/lib.sh
+
+check_unignored_build_artifacts
ci/install-dependencies.sh +4 -0
index 34b4400f38b..01006379fd0 100755
@@ -22,6 +22,10 @@ ubuntu-latest)
 	P4_PATH="$HOME/custom/p4"
 	GIT_LFS_PATH="$HOME/custom/git-lfs"
 	export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH"
+	if test -n "$GITHUB_PATH"
+	then
+		echo "$PATH" >>"$GITHUB_PATH"
+	fi
 
 	P4WHENCE=http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION
 	LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION
ci/lib.sh +35 -1
index 3727e6ec2f9..cc9a368be42 100755
@@ -1,3 +1,5 @@
+#!/bin/sh
+
 # Library of functions shared by all CI scripts
 
 # Set 'exit on error' for all CI scripts to let the caller know that
@@ -39,7 +41,10 @@ setenv () {
 	val=$2
 	shift 2
 
-	eval "export $key=\"$val\""
+	if test -n "$GITHUB_ENV"
+	then
+		echo "$key=$val" >>"$GITHUB_ENV"
+	fi
 }
 
 check_unignored_build_artifacts ()
@@ -109,6 +114,35 @@ macos-latest)
 esac
 
 case "$jobname" in
+linux-gcc)
+	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME main
+	;;
+linux-TEST-vars)
+	setenv --test GIT_TEST_SPLIT_INDEX yes
+	setenv --test GIT_TEST_MERGE_ALGORITHM recursive
+	setenv --test GIT_TEST_FULL_IN_PACK_ARRAY true
+	setenv --test GIT_TEST_OE_SIZE 10
+	setenv --test GIT_TEST_OE_DELTA_SIZE 5
+	setenv --test GIT_TEST_COMMIT_GRAPH 1
+	setenv --test GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS 1
+	setenv --test GIT_TEST_MULTI_PACK_INDEX 1
+	setenv --test GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP 1
+	setenv --test GIT_TEST_ADD_I_USE_BUILTIN 1
+	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME master
+	setenv --test GIT_TEST_WRITE_REV_INDEX 1
+	setenv --test GIT_TEST_CHECKOUT_WORKERS 2
+	;;
+linux-clang)
+	setenv --test GIT_TEST_DEFAULT_HASH sha1
+	;;
+linux-sha256)
+	setenv --test GIT_TEST_DEFAULT_HASH sha256
+	;;
+pedantic)
+	# Don't run the tests; we only care about whether Git can be
+	# built.
+	setenv --build DEVOPTS pedantic
+	;;
 linux32)
 	CC=gcc
 	;;
ci/make-test-artifacts.sh +0 -2
index 646967481f6..45298562982 100755
@@ -8,5 +8,3 @@ mkdir -p "$1" # in case ci/lib.sh decides to quit early
 . ${0%/*}/lib.sh
 
 make artifacts-tar ARTIFACTS_DIRECTORY="$1"
-
-check_unignored_build_artifacts
ci/run-build-and-tests.sh deleted +0 -47

47 lines not on this page · raw message

ci/run-test-slice.sh +0 -2
index b9a682b4bcd..70326961454 100755
@@ -8,5 +8,3 @@
 make --quiet -C t T="$(cd t &&
 	./helper/test-tool path-utils slice-tests "$1" "$2" t[0-9]*.sh |
 	tr '\n' ' ')"
-
-check_unignored_build_artifacts
ci/test-documentation.sh +0 -1
index 41e2b126311..6b5cce03bd7 100755
@@ -39,4 +39,3 @@ test -s Documentation/git.html
 grep '<meta name="generator" content="Asciidoctor ' Documentation/git.html
 
 rm -f stdout.log stderr.log stderr.raw
-check_unignored_build_artifacts
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#128 lore raw

Re: [PATCH v3 13/29] CI: remove "run-build-and-tests.sh", run "make [test]" directly

Quoted 78 lines · Ævar Arnfjörð Bjarmason

Remove the already thin "ci/run-build-and-tests.sh" wrapper and instead make the CI run "make" or "make test" directly. By doing this we'll be able to easily see at a glance whether our failure was in the compilation or testing, whether that's via human eyes or improve machine readability.

We also need to run our new "ci/check-unignored-build-artifacts.sh" on success() in the CI now, just like we already had a step conditional on failure() running ci/print-test-failures.sh.

The reason we used a "ci/run-build-and-tests.sh" wrapper in the first place had to do with Travis CI-specific constraints that no longer apply to us, as the Travis CI support has been removed.

Instead we can configure the CI in an earlier step by running "ci/lib.sh", which under GitHub CI will write the environment variables we need to the "$GITHUB_ENV" file.

We'll then have access to them in subsequent steps, and crucially those variables will be prominently visible at the start of each step via an expandable drop-down in the UI.drop-do.

I.e. this changes the CI run from a top-down flow like (pseudocode):

- job:
- step1:
- use ci/lib.sh to set env vars
- run a script like ci/run-build-and-tests.sh
- step2:
- if: failure()
- use ci/lib.sh to set env vars
- run ci/print-test-failures.sh

To:

- job:
- step1:
- set variables in $GITHUB_ENV using ci/lib.sh
- step2:
- make
- step3:
- make test
- step4:
- if: failure()
- run ci/print-test-failures.sh
- step5:
- if: success()
- run ci/check-unignored-build-artifacts.sh

There is a proposal[2] to get some of the benefits of this approach by not re-arranging our variable setup in this way, but to instead use the GitHub CI grouping syntax to focus on the relevant parts of "make" or "make test" when we have failures.

Doing it this way makes for better looking GitHub CI UI, and lays much better ground work for our CI going forward. Because:

* The CI logic will be more portable to a future CI system, since a common feature of them is to run various commands in sequence, but a future system won't necessarily support the GitHub-specifics syntax of "grouping" output within a "step".

Even if those systems don't support a "$GITHUB_ENV" emulating will be much easier than to deal with some CI-specific grouping syntax.

* At the start of every step the GitHub CI presents an expandable list of environment variables from "$GITHUB_ENV". We'll now see exactly what variables affected that step (although we currently overshoot that a bit, and always define all variables).

* CI failures will be easier to reproduce locally, as this makes the relevant ci/* scripts something that sets up our environment, but leaves "make" and "make test" working as they do locally.

To reproduce a run the user only needs to set the variables discussed in the drop-down above, either manually or by running "ci/lib.sh".

Quoted 8 lines

* The output will be less verbose. The "ci/lib.sh" script uses "set -x", and before this e.g. "ci/static-analysis.sh" would start with 40 lines of trace output, culminating in using "export" to export the relevant environment variables.

Now that verbosity is in the earlier "ci/lib.sh" step, and not in any subsequent one. The "make" targets then start out with the relevant output non-trace output right away.

s/output non-trace output/non-trace output/

Quoted 10 lines

* If we do want to use the grouping syntax within a "step" it'll now be easier to do so. It doesn't support nesting, so we'd have to make a choice between using it for e.g. "make" v.s. "make test", or individual test failures. See "sadly" in [3].

1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/
3. https://lore.kernel.org/git/9333ba781b8240f704e739b00d274f8c3d887e39.1643050574.git.gitgitgadget@gmail.com/

#129 lore raw

PATCH v3 14/29 ci/lib.sh: use "test" instead of "["

In reply to Ævar Arnfjörð Bjarmason

Change the ci/lib.sh script to use "if test" rather than "if [". Until the preceding commit this script relied on the non-POSIX "export X=Y" syntax, but now that it's POSIX portable let's use our usual style in it, and not the bash-specific style that was used in it historically.

---
 ci/lib.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
ci/lib.sh +2 -2
index cc9a368be42..ffc0e524aaa 100755
@@ -94,7 +94,7 @@ ubuntu-latest)
 		break
 	fi
 
-	if [ "$jobname" = linux-gcc ]
+	if test "$jobname" = linux-gcc
 	then
 		MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python3"
 	else
@@ -104,7 +104,7 @@ ubuntu-latest)
 	setenv --test GIT_TEST_HTTPD true
 	;;
 macos-latest)
-	if [ "$jobname" = osx-gcc ]
+	if test "$jobname" = osx-gcc
 	then
 		MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
 	else
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#130 lore raw

PATCH v3 15/29 CI: check ignored unignored build artifacts in "win[+VS] build" too

In reply to Ævar Arnfjörð Bjarmason

Use the newly created "ci/check-unignored-build-artifacts.sh" script in the "win build" and "win+VS build" steps too. These steps weren't doing this sanity check of untracked files against .gitignore before, but as the check passes for them let's keep doing that extra sanity check.

---
 .github/workflows/main.yml | 6 ++++++
 1 file changed, 6 insertions(+)
.github/workflows/main.yml +6 -0
index c83f6a21ca3..05c5771e7f6 100644
@@ -93,6 +93,9 @@ jobs:
         HOME: ${{runner.workspace}}
         NO_PERL: 1
       run: . /etc/profile && ci/make-test-artifacts.sh artifacts
+    - run: ci/check-unignored-build-artifacts.sh
+      if: success()
+      shell: bash
     - name: zip up tracked files
       run: git archive -o artifacts/tracked.tar.gz HEAD
     - name: upload tracked files and build artifacts
@@ -180,6 +183,9 @@ jobs:
       run: |
         mkdir -p artifacts &&
         eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts NO_GETTEXT=YesPlease 2>&1 | grep ^tar)"
+    - run: ci/check-unignored-build-artifacts.sh
+      if: success()
+      shell: bash
     - name: zip up tracked files
       run: git archive -o artifacts/tracked.tar.gz HEAD
     - name: upload tracked files and build artifacts
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#131 lore raw

PATCH v3 16/29 CI: invoke "make artifacts-tar" directly in windows-build

In reply to Ævar Arnfjörð Bjarmason

Change the windows-build job to invoke the "make artifacts-tar" step directly, instead of calling a "ci/make-test-artifacts.sh" script.

The script was needed because "ci/lib.sh" would set up various environment variables for us, but now we can instead use the "ci/lib.sh" in its script mode.

The "mkdir -p" added in b819f1d2cec (ci: parallelize testing on Windows, 2019-01-29) isn't needed, the same commit added that "mkdir -p" to the "artifacts-tar" rule itself, so we can have "make" create the directory for us.

This also has the benefit of making the "build" step less chatty, since it won't start with the verbose "set -x" output, that's now contained in the "ci/lib.sh" step.

---
 .github/workflows/main.yml |  5 ++++-
 ci/make-test-artifacts.sh  | 10 ----------
 2 files changed, 4 insertions(+), 11 deletions(-)
 delete mode 100755 ci/make-test-artifacts.sh
.github/workflows/main.yml +4 -1
index 05c5771e7f6..2c23a19eac2 100644
@@ -87,12 +87,15 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
+    - run: ci/lib.sh
+      shell: bash
     - name: build
       shell: bash
       env:
         HOME: ${{runner.workspace}}
         NO_PERL: 1
-      run: . /etc/profile && ci/make-test-artifacts.sh artifacts
+      run: . /etc/profile && make artifacts-tar ARTIFACTS_DIRECTORY=artifacts
+      if: success()
     - run: ci/check-unignored-build-artifacts.sh
       if: success()
       shell: bash
ci/make-test-artifacts.sh deleted +0 -10
deleted file mode 100755
index 45298562982..00000000000
@@ -1,10 +0,0 @@
-#!/bin/sh
-#
-# Build Git and store artifacts for testing
-#
-
-mkdir -p "$1" # in case ci/lib.sh decides to quit early
-
-. ${0%/*}/lib.sh
-
-make artifacts-tar ARTIFACTS_DIRECTORY="$1"
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#132 lore raw

PATCH v3 17/29 CI: split up and reduce "ci/test-documentation.sh"

In reply to Ævar Arnfjörð Bjarmason

Change the "ci/test-documentation.sh" script to run the bash-specific parts in as one command in the CI job itself, and to run the two "make doc" commands at the top-level.

It'll now be obvious from the title of the step if if we failed in the asciidoc or asciidoctor step.

Since the "check_unignored_build_artifacts()" function is now only used in "ci/check-unignored-build-artifacts.sh" move that function there.

The recipe for the job in ".github/workflows/main.yml" is now a bit verbose because it's effectively the same job twice, with a "make clean" in-between. It would be better for the verbosity to run it via a matrix as done in the alternate approach in [1] does, but then we'd sacrifice overall CPU time for the brevity. It might still be worth doing, but let's go for this simpler approach for now.

1. https://lore.kernel.org/git/[email protected]/

---
 .github/workflows/main.yml            | 16 +++++++++++++-
 ci/check-unignored-build-artifacts.sh | 10 +++++++++
 ci/lib.sh                             | 10 ---------
 ci/test-documentation.sh              | 31 +++++++++------------------
 4 files changed, 35 insertions(+), 32 deletions(-)
.github/workflows/main.yml +15 -1
index 2c23a19eac2..92b914f16fd 100644
@@ -365,4 +365,18 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/test-documentation.sh
+    - run: ci/lib.sh
+    - run: make check-docs
+    - run: "make doc > >(tee stdout.log) 2> >(tee stderr.raw >&2)"
+      shell: bash
+    - run: ci/test-documentation.sh AsciiDoc
+      if: success()
+    - run: ci/check-unignored-build-artifacts.sh
+      if: success()
+    - run: make clean
+    - run: "make USE_ASCIIDOCTOR=1 doc > >(tee stdout.log) 2> >(tee stderr.raw >&2)"
+      shell: bash
+    - run: ci/test-documentation.sh Asciidoctor
+      if: success()
+    - run: ci/check-unignored-build-artifacts.sh
+      if: success()
ci/check-unignored-build-artifacts.sh +10 -0
index 56d04b0db9a..0bc04f32804 100755
@@ -5,4 +5,14 @@
 
 . ${0%/*}/lib.sh
 
+check_unignored_build_artifacts ()
+{
+	! git ls-files --other --exclude-standard --error-unmatch \
+		-- ':/*' 2>/dev/null ||
+	{
+		echo "$(tput setaf 1)error: found unignored build artifacts$(tput sgr0)"
+		false
+	}
+}
+
 check_unignored_build_artifacts
ci/lib.sh +0 -10
index ffc0e524aaa..35a972fa3f5 100755
@@ -47,16 +47,6 @@ setenv () {
 	fi
 }
 
-check_unignored_build_artifacts ()
-{
-	! git ls-files --other --exclude-standard --error-unmatch \
-		-- ':/*' 2>/dev/null ||
-	{
-		echo "$(tput setaf 1)error: found unignored build artifacts$(tput sgr0)"
-		false
-	}
-}
-
 # GitHub Action doesn't set TERM, which is required by tput
 setenv TERM ${TERM:-dumb}
 
ci/test-documentation.sh +10 -21
index 6b5cce03bd7..b8a6a6f664e 100755
@@ -1,10 +1,12 @@
-#!/usr/bin/env bash
+#!/bin/sh
 #
-# Perform sanity checks on documentation and build it.
+# Perform sanity checks on "make doc" output and built documentation
 #
 
 . ${0%/*}/lib.sh
 
+generator=$1
+
 filter_log () {
 	sed -e '/^GIT_VERSION = /d' \
 	    -e "/constant Gem::ConfigMap is deprecated/d" \
@@ -14,28 +16,15 @@ filter_log () {
 	    "$1"
 }
 
-make check-docs
-
-# Build docs with AsciiDoc
-make doc > >(tee stdout.log) 2> >(tee stderr.raw >&2)
-cat stderr.raw
-filter_log stderr.raw >stderr.log
-test ! -s stderr.log
-test -s Documentation/git.html
-test -s Documentation/git.xml
-test -s Documentation/git.1
-grep '<meta name="generator" content="AsciiDoc ' Documentation/git.html
-
-rm -f stdout.log stderr.log stderr.raw
-check_unignored_build_artifacts
-
-# Build docs with AsciiDoctor
-make clean
-make USE_ASCIIDOCTOR=1 doc > >(tee stdout.log) 2> >(tee stderr.raw >&2)
 cat stderr.raw
 filter_log stderr.raw >stderr.log
 test ! -s stderr.log
 test -s Documentation/git.html
-grep '<meta name="generator" content="Asciidoctor ' Documentation/git.html
+if test "$generator" = "Asciidoctor"
+then
+	test -s Documentation/git.xml
+	test -s Documentation/git.1
+fi
+grep "<meta name=\"generator\" content=\"$generator " Documentation/git.html
 
 rm -f stdout.log stderr.log stderr.raw
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#133 lore raw

Re: [PATCH v3 17/29] CI: split up and reduce "ci/test-documentation.sh"

Quoted 6 lines · Ævar Arnfjörð Bjarmason

Change the "ci/test-documentation.sh" script to run the bash-specific parts in as one command in the CI job itself, and to run the two "make doc" commands at the top-level.

It'll now be obvious from the title of the step if if we failed in the asciidoc or asciidoctor step.

s/if if/if/

Quoted 14 lines

Since the "check_unignored_build_artifacts()" function is now only used in "ci/check-unignored-build-artifacts.sh" move that function there.

The recipe for the job in ".github/workflows/main.yml" is now a bit verbose because it's effectively the same job twice, with a "make clean" in-between. It would be better for the verbosity to run it via a matrix as done in the alternate approach in [1] does, but then we'd sacrifice overall CPU time for the brevity. It might still be worth doing, but let's go for this simpler approach for now.

1. https://lore.kernel.org/git/[email protected]/

#134 lore raw

PATCH v3 18/29 CI: combine ci/install{,-docker}-dependencies.sh

In reply to Ævar Arnfjörð Bjarmason

Combine the "ci/install-dependencies.sh" and "ci/install-docker-dependencies.sh" into one script. The "case" statement in the latter only cared about "$jobname", and can be folded into the same "case" statement in the former.

The reason they split up is historical, and because the "ci/install-dependencies.sh" used "ci/lib.sh", which requires "bash". At least one of the docker containers doesn't have "bash". To make the existing code POSIX-compatible we need to replace pushd/popd with a sub-shell, but no other changes were needed.

This also fixes issue in "ci/install-docker-dependencies.sh" where we'd hide errors due to not using "set -e". Now that we include "ci/lib.sh" we won't have that potential issue.

---
 .github/workflows/main.yml        |  2 +-
 ci/install-dependencies.sh        | 27 ++++++++++++++++++++++-----
 ci/install-docker-dependencies.sh | 22 ----------------------
 3 files changed, 23 insertions(+), 28 deletions(-)
 delete mode 100755 ci/install-docker-dependencies.sh
.github/workflows/main.yml +1 -1
index 92b914f16fd..3fd3e4f873c 100644
@@ -311,7 +311,7 @@ jobs:
     container: ${{matrix.vector.image}}
     steps:
     - uses: actions/checkout@v1
-    - run: ci/install-docker-dependencies.sh
+    - run: ci/install-dependencies.sh
     - run: ci/lib.sh
     - run: make
     - run: make test
ci/install-dependencies.sh +22 -5
index 01006379fd0..9388289d0ed 100755
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh
 #
 # Install dependencies required to build and test Git on Linux and macOS
 #
@@ -34,18 +34,20 @@ ubuntu-latest)
 	sudo apt-get -q -y install language-pack-is libsvn-perl apache2 \
 		$UBUNTU_COMMON_PKGS $CC_PACKAGE
 	mkdir --parents "$P4_PATH"
-	pushd "$P4_PATH"
+	(
+		cd "$P4_PATH"
 		wget --quiet "$P4WHENCE/bin.linux26x86_64/p4d"
 		wget --quiet "$P4WHENCE/bin.linux26x86_64/p4"
 		chmod u+x p4d
 		chmod u+x p4
-	popd
+	)
 	mkdir --parents "$GIT_LFS_PATH"
-	pushd "$GIT_LFS_PATH"
+	(
+		cd "$GIT_LFS_PATH"
 		wget --quiet "$LFSWHENCE/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz"
 		tar --extract --gunzip --file "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz"
 		cp git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs .
-	popd
+	)
 	;;
 macos-latest)
 	export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1
@@ -90,6 +92,21 @@ linux-gcc-default)
 	sudo apt-get -q update
 	sudo apt-get -q -y install $UBUNTU_COMMON_PKGS
 	;;
+linux32)
+	linux32 --32bit i386 sh -c '
+		apt update >/dev/null &&
+		apt install -y build-essential libcurl4-openssl-dev \
+			libssl-dev libexpat-dev gettext python >/dev/null
+	'
+	;;
+linux-musl)
+	apk add --update build-base curl-dev openssl-dev expat-dev gettext \
+		pcre2-dev python3 musl-libintl perl-utils ncurses >/dev/null
+	;;
+pedantic)
+	dnf -yq update >/dev/null &&
+	dnf -yq install make gcc findutils diffutils perl python3 gettext zlib-devel expat-devel openssl-devel curl-devel pcre2-devel >/dev/null
+	;;
 esac
 
 if type p4d >/dev/null && type p4 >/dev/null
ci/install-docker-dependencies.sh deleted +0 -22
deleted file mode 100755
index 78b7e326da6..00000000000
@@ -1,22 +0,0 @@
-#!/bin/sh
-#
-# Install dependencies required to build and test Git inside container
-#
-
-case "$jobname" in
-linux32)
-	linux32 --32bit i386 sh -c '
-		apt update >/dev/null &&
-		apt install -y build-essential libcurl4-openssl-dev \
-			libssl-dev libexpat-dev gettext python >/dev/null
-	'
-	;;
-linux-musl)
-	apk add --update build-base curl-dev openssl-dev expat-dev gettext \
-		pcre2-dev python3 musl-libintl perl-utils ncurses >/dev/null
-	;;
-pedantic)
-	dnf -yq update >/dev/null &&
-	dnf -yq install make gcc findutils diffutils perl python3 gettext zlib-devel expat-devel openssl-devel curl-devel pcre2-devel >/dev/null
-	;;
-esac
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#135 lore raw

PATCH v3 19/29 CI: move "env" definitions into ci/lib.sh

In reply to Ævar Arnfjörð Bjarmason

Have the GitHub CI jobs use the "ci/lib".sh as a source of truth for environment variables that affect their builds and tests. This brings them in line with the rest of the jobs, and make it easier to see the entire CI configuration per jobname at a glance.

To do this we need to add a ci/lib.sh "step" to the remaining jobs that didn't have it added in preceding commits.

The Makefile parameters are luckily all accepted via the environment, so we can export these instead of adding them to MAKEFLAGS.

Let's also use the documented GIT_CONIFG_* mechanism for setting config instead of the internal GIT_CONFIG_PARAMETERS variable. This adjusts code added in 889cacb6897 (ci: configure GitHub Actions for CI/PR, 2020-04-11), we could probably use the GIT_AUTHOR_NAME etc, but let's keep setting this via config, just with the documented mechanism.

By setting "ARTIFACTS_DIRECTORY=artifacts" once we don't need to repeat it in various places, let's get it from the environment instead.

Finally, the "DEVELOPER: 1" was only needed by the "sparse" job, which wasn't using "ci/lib.sh" (and which sets DEVELOPER=1). Let's instead have the "sparse" job use the "ci/lib.sh" and remove DEVELOPER=1 from ".github/workflows/main.yml".

This substantially speeds up the "sparse" job, since it'll now pick up the "--jobs" setting in MAKEFLAGS that we use everywhere else. Before it ran in around 4m30s, now in around 2m30s.

---
 .github/workflows/main.yml | 29 +++++++++++------------------
 ci/lib.sh                  | 21 +++++++++++++++++++++
 2 files changed, 32 insertions(+), 18 deletions(-)
.github/workflows/main.yml +11 -18
index 3fd3e4f873c..fe37e0f1b36 100644
@@ -2,9 +2,6 @@ name: CI
 
 on: [push, pull_request]
 
-env:
-  DEVELOPER: 1
-
 jobs:
   ci-config:
     name: config
@@ -93,8 +90,7 @@ jobs:
       shell: bash
       env:
         HOME: ${{runner.workspace}}
-        NO_PERL: 1
-      run: . /etc/profile && make artifacts-tar ARTIFACTS_DIRECTORY=artifacts
+      run: . /etc/profile && make artifacts-tar
       if: success()
     - run: ci/check-unignored-build-artifacts.sh
       if: success()
@@ -105,7 +101,7 @@ jobs:
       uses: actions/upload-artifact@v2
       with:
         name: windows-artifacts
-        path: artifacts
+        path: ${{env.ARTIFACTS_DIRECTORY}}
   windows-test:
     name: win test
     env:
@@ -126,6 +122,8 @@ jobs:
       shell: bash
       run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
+    - run: ci/lib.sh
+      shell: bash
     - name: test
       shell: bash
       run: . /etc/profile && ci/run-test-slice.sh ${{matrix.nr}} 10
@@ -145,9 +143,6 @@ jobs:
       jobname: vs-build
     needs: ci-config
     if: needs.ci-config.outputs.enabled == 'yes'
-    env:
-      NO_PERL: 1
-      GIT_CONFIG_PARAMETERS: "'user.name=CI' 'user.email=ci@git'"
     runs-on: windows-latest
     steps:
     - uses: actions/checkout@v2
@@ -171,6 +166,8 @@ jobs:
     - name: copy dlls to root
       shell: cmd
       run: compat\vcbuild\vcpkg_copy_dlls.bat release
+    - run: ci/lib.sh
+      shell: bash
     - name: generate Visual Studio solution
       shell: bash
       run: |
@@ -181,21 +178,16 @@ jobs:
     - name: bundle artifact tar
       shell: bash
       env:
-        MSVC: 1
         VCPKG_ROOT: ${{github.workspace}}\compat\vcbuild\vcpkg
       run: |
-        mkdir -p artifacts &&
-        eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts NO_GETTEXT=YesPlease 2>&1 | grep ^tar)"
-    - run: ci/check-unignored-build-artifacts.sh
-      if: success()
-      shell: bash
+        eval "$(make -n artifacts-tar 2>&1 | grep -e ^mkdir -e ^tar)"
     - name: zip up tracked files
       run: git archive -o artifacts/tracked.tar.gz HEAD
     - name: upload tracked files and build artifacts
       uses: actions/upload-artifact@v2
       with:
         name: vs-artifacts
-        path: artifacts
+        path: ${{env.ARTIFACTS_DIRECTORY}}
   vs-test:
     name: win+VS test
     env:
@@ -216,10 +208,10 @@ jobs:
     - name: extract tracked files and build artifacts
       shell: bash
       run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz
+    - run: ci/lib.sh
+      shell: bash
     - name: test
       shell: bash
-      env:
-        NO_SVN_TESTS: 1
       run: . /etc/profile && ci/run-test-slice.sh ${{matrix.nr}} 10
     - name: ci/print-test-failures.sh
       if: failure()
@@ -354,6 +346,7 @@ jobs:
     - uses: actions/checkout@v2
     - name: Install other dependencies
       run: ci/install-dependencies.sh
+    - run: ci/lib.sh
     - run: make sparse
   documentation:
     name: documentation
ci/lib.sh +21 -0
index 35a972fa3f5..8c574b21a67 100755
@@ -104,6 +104,27 @@ macos-latest)
 esac
 
 case "$jobname" in
+windows-build)
+	setenv --build NO_PERL NoThanks
+	setenv --build ARTIFACTS_DIRECTORY artifacts
+	;;
+vs-build)
+	setenv --build NO_PERL NoThanks
+	setenv --build NO_GETTEXT NoThanks
+	setenv --build ARTIFACTS_DIRECTORY artifacts
+	setenv --build INCLUDE_DLLS_IN_ARTIFACTS YesPlease
+	setenv --build MSVC YesPlease
+
+	setenv --build GIT_CONFIG_COUNT 2
+	setenv --build GIT_CONFIG_KEY_0 user.name
+	setenv --build GIT_CONFIG_VALUE_0 CI
+	setenv --build GIT_CONFIG_KEY_1 user.emailname
+	setenv --build GIT_CONFIG_VALUE_1 ci@git
+	setenv --build GIT_CONFIG_VALUE_1 ci@git
+	;;
+vs-test)
+	setenv --test NO_SVN_TESTS YesPlease
+	;;
 linux-gcc)
 	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME main
 	;;
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#136 lore raw

PATCH v3 20/29 ci/run-test-slice.sh: replace shelling out with "echo"

In reply to Ævar Arnfjörð Bjarmason

Amend this code added in b819f1d2cec (ci: parallelize testing on Windows, 2019-01-29) to stop shelling out to "tr", all we're trying to do here is to turn '\n'-delimited output into a ' '-delimited list. To do that we can just use "echo".

None of these files will contain the odd characters using "echo" might choke on in the general case (i.e. '\'-escapes and the like).

This has the added benefit of not adding a redundant ' ' to the end of the T-parameter, which didn't matter in practice, but as we'll see in a subsequent commit makes for slightly nicer output, as we'll show this variable using a GitHub CI-native feature.

---
 ci/run-test-slice.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
ci/run-test-slice.sh +3 -3
index 70326961454..1de6a18ca47 100755
@@ -5,6 +5,6 @@
 
 . ${0%/*}/lib.sh
 
-make --quiet -C t T="$(cd t &&
-	./helper/test-tool path-utils slice-tests "$1" "$2" t[0-9]*.sh |
-	tr '\n' ' ')"
+tests=$(echo $(cd t && ./helper/test-tool path-utils slice-tests "$1" "$2" \
+	t[0-9]*.sh))
+make --quiet -C t T="$tests"
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#137 lore raw

PATCH v3 21/29 CI: pre-select test slice in Windows & VS tests

In reply to Ævar Arnfjörð Bjarmason

In preceding commits the tests have been changed to do their setup via $GITHUB_ENV in one step, and to then have subsequent steps that re-use that environment.

Let's change the "test slice" tests added in b819f1d2cec (ci:
parallelize testing on Windows, 2019-01-29) to do the same. These tests select 10% of the tests to run in 10 "test slices". Now we'll select those in a step that immediately precedes the testing step, and then simply invoke "make -C t -e".

This has the advantage that the tests to be run are now listed in the standard "Run" drop-down at the start of the "test" step.

Since the "T" variable in "t/Makefile" doesn't normally accept overrides from the environment we need to invoke "make" with the "-e" option (a.k.a. "--environment-overrides"). We could also make $(T) in t/Makefile be a "?=" assigned variable, but this way works, and is arguably clearer as it's more obvious that we're injecting a special list of tests that override the normal behavior of that Makefile.

Note that we cannot run the top-level "make test" here, because of how the Windows CI builds git, i.e. either via CMake or some option that would cause "make test" to recompile git itself. Instead we run "make -C t [...]".

---
 .github/workflows/main.yml                     | 10 ++++++++--
 ci/{run-test-slice.sh => select-test-slice.sh} |  4 ++--
 2 files changed, 10 insertions(+), 4 deletions(-)
 rename ci/{run-test-slice.sh => select-test-slice.sh} (57%)
.github/workflows/main.yml +8 -2
index fe37e0f1b36..1e13718b989 100644
@@ -124,9 +124,12 @@ jobs:
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
     - run: ci/lib.sh
       shell: bash
+    - name: select tests
+      run: . /etc/profile && ci/select-test-slice.sh ${{matrix.nr}} 10
+      shell: bash
     - name: test
       shell: bash
-      run: . /etc/profile && ci/run-test-slice.sh ${{matrix.nr}} 10
+      run: . /etc/profile && make -C t -e
     - name: ci/print-test-failures.sh
       if: failure()
       shell: bash
@@ -210,9 +213,12 @@ jobs:
       run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz
     - run: ci/lib.sh
       shell: bash
+    - name: select tests
+      run: . /etc/profile && ci/select-test-slice.sh ${{matrix.nr}} 10
+      shell: bash
     - name: test
       shell: bash
-      run: . /etc/profile && ci/run-test-slice.sh ${{matrix.nr}} 10
+      run: . /etc/profile && make -C t -e
     - name: ci/print-test-failures.sh
       if: failure()
       shell: bash
ci/select-test-slice.sh renamed from ci/run-test-slice.sh +2 -2
similarity index 57%
rename from ci/run-test-slice.sh
rename to ci/select-test-slice.sh
index 1de6a18ca47..ec602f8a053 100755
@@ -1,10 +1,10 @@
 #!/bin/sh
 #
-# Test Git in parallel
+# Select a portion of the tests for testing Git in parallel
 #
 
 . ${0%/*}/lib.sh
 
 tests=$(echo $(cd t && ./helper/test-tool path-utils slice-tests "$1" "$2" \
 	t[0-9]*.sh))
-make --quiet -C t T="$tests"
+echo T="$tests" >>$GITHUB_ENV
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#138 lore raw

PATCH v3 22/29 CI: only invoke ci/lib.sh as "steps" in main.yml

In reply to Ævar Arnfjörð Bjarmason

Change the scripts in ci/ to stop using ci/lib.sh as a library, now that the only thing it did for them was to "set -ex" and possibly set TERM=dumb.

Let's create a ci/lib-tput.sh for those that need to use "tput" instead, and have these scripts invoke "set -ex" themselves.

This makes their invocation a lot less verbose, since they'll be relying on an earlier step in the CI job to have set the variables in $GITHUB_ENV, and won't be spewing their own trace output to set those variables again.

Let's also create a ci/lib-ci-type.sh, and have ci/lib.sh and ci/print-test-failures.sh share the logic to discover the CI type. We could have set the CI_TYPE in the environment with "setenv", but let's avoid that verbosity for this purely internal variable.

The "ci/lib.sh" is now no longer a "Library of functions shared by all CI scripts", so let's remove that commentary, and the misleading comment about "set -ex" being for "installing dependencies", we're now no longer using it in "ci/install-dependencies.sh" (but it does its own "set -ex").

---
 ci/check-unignored-build-artifacts.sh |  4 +++-
 ci/install-dependencies.sh            |  2 +-
 ci/lib-ci-type.sh                     |  9 +++++++++
 ci/lib-tput.sh                        |  5 +++++
 ci/lib.sh                             | 27 ++++++++++-----------------
 ci/print-test-failures.sh             |  6 +++---
 ci/select-test-slice.sh               |  2 +-
 ci/test-documentation.sh              |  2 +-
 8 files changed, 33 insertions(+), 24 deletions(-)
 create mode 100644 ci/lib-ci-type.sh
 create mode 100644 ci/lib-tput.sh
ci/check-unignored-build-artifacts.sh +3 -1
index 0bc04f32804..c27d6a97f45 100755
@@ -3,7 +3,9 @@
 # Check whether the build created anything not in our .gitignore
 #
 
-. ${0%/*}/lib.sh
+set -ex
+
+. ${0%/*}/lib-tput.sh
 
 check_unignored_build_artifacts ()
 {
ci/install-dependencies.sh +1 -1
index 9388289d0ed..ee9af62fc92 100755
@@ -3,7 +3,7 @@
 # Install dependencies required to build and test Git on Linux and macOS
 #
 
-. ${0%/*}/lib.sh
+set -ex
 
 UBUNTU_COMMON_PKGS="make libssl-dev libcurl4-openssl-dev libexpat-dev
  tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl
ci/lib-ci-type.sh new file +9 -0
new file mode 100644
index 00000000000..6f01fd9e5d9
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+if test "$GITHUB_ACTIONS" = "true"
+then
+	CI_TYPE=github-actions
+else
+	echo "Could not identify CI type" >&2
+	exit 1
+fi
ci/lib-tput.sh new file +5 -0
new file mode 100644
index 00000000000..b62ef13c44c
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# GitHub Action doesn't set TERM, which is required by tput
+TERM=${TERM:-dumb}
+export TERM
ci/lib.sh +10 -17
index 8c574b21a67..8f7e178f3ff 100755
@@ -1,13 +1,9 @@
 #!/bin/sh
-
-# Library of functions shared by all CI scripts
-
-# Set 'exit on error' for all CI scripts to let the caller know that
-# something went wrong.
-# Set tracing executed commands, primarily setting environment variables
-# and installing dependencies.
 set -ex
 
+# Helper libraries
+. ${0%/*}/lib-ci-type.sh
+
 # Starting assertions
 if test -z "$jobname"
 then
@@ -47,18 +43,14 @@ setenv () {
 	fi
 }
 
-# GitHub Action doesn't set TERM, which is required by tput
-setenv TERM ${TERM:-dumb}
-
 # How many jobs to run in parallel?
 NPROC=10
 
 # Clear MAKEFLAGS that may come from the outside world.
 MAKEFLAGS=--jobs=$NPROC
 
-if test "$GITHUB_ACTIONS" = "true"
-then
-	CI_TYPE=github-actions
+case "$CI_TYPE" in
+github-actions)
 	CC="${CC:-gcc}"
 
 	setenv --test GIT_PROVE_OPTS "--timer --jobs $NPROC"
@@ -66,11 +58,12 @@ then
 	test Windows != "$RUNNER_OS" ||
 	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
 	setenv --test GIT_TEST_OPTS "$GIT_TEST_OPTS"
-else
-	echo "Could not identify CI type" >&2
-	env >&2
+	;;
+*)
+	echo "Unhandled CI type: $CI_TYPE" >&2
 	exit 1
-fi
+	;;
+esac
 
 setenv --build DEVELOPER 1
 setenv --test DEFAULT_TEST_TARGET prove
ci/print-test-failures.sh +3 -3
index a878cca67e4..2405f65650a 100755
@@ -3,10 +3,10 @@
 # Print output of failing tests
 #
 
-. ${0%/*}/lib.sh
+set -e
 
-# Tracing executed commands would produce too much noise in the loop below.
-set +x
+. ${0%/*}/lib-ci-type.sh
+. ${0%/*}/lib-tput.sh
 
 cd t/
 
ci/select-test-slice.sh +1 -1
index ec602f8a053..a0332095bb3 100755
@@ -3,7 +3,7 @@
 # Select a portion of the tests for testing Git in parallel
 #
 
-. ${0%/*}/lib.sh
+set -ex
 
 tests=$(echo $(cd t && ./helper/test-tool path-utils slice-tests "$1" "$2" \
 	t[0-9]*.sh))
ci/test-documentation.sh +1 -1
index b8a6a6f664e..64ff212cdaa 100755
@@ -3,7 +3,7 @@
 # Perform sanity checks on "make doc" output and built documentation
 #
 
-. ${0%/*}/lib.sh
+set -ex
 
 generator=$1
 
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#139 lore raw

PATCH v3 23/29 CI: narrow down variable definitions in --build and --test

In reply to Ævar Arnfjörð Bjarmason

In a preceding step the "setvar" function was made to take a "--build", "--test" or "--all" argument to indicate where the variables it sets were used.

Let's make use of that by having the relevant parts of ".github/workflows/main.yml" invoke "ci/lib.sh" with those options.

By doing this the set of variables shown in build-only steps will be fewer, which makes diagnosing anything going on there easier, as we won't have to look at a deluge of e.g. GIT_TEST_* variables.

Since we'll now always run "--build" for "make" and "--test" for "make test" we can stop setting the "fat" MAKEFLAGS entirely on the "test" steps, i.e. the one with PYTHON_PATH, CC etc. These will all be carried over from the earlier --build step.

That won't be true in the case of the "windows-test" and "vs-test" jobs, since they run separately from the corresponding "build" step. So we'll need to make sure that we have the --jobs=N argument for those.

This doesn't matter that much, as we'll still have --jobs=N in GIT_PROVE_OPTS. So the only thing we'll use it for is parallelism in the t/Makefile before we get to running "prove". Still, it's good to be consistent for good measure, and to run the t/Makefile itself in parallel.

---
 .github/workflows/main.yml | 20 +++++++++++---------
 ci/lib.sh                  | 30 +++++++++++++++++++++++-------
 2 files changed, 34 insertions(+), 16 deletions(-)
.github/workflows/main.yml +11 -9
index 1e13718b989..0787cadc76b 100644
@@ -84,7 +84,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
       shell: bash
     - name: build
       shell: bash
@@ -122,7 +122,7 @@ jobs:
       shell: bash
       run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
-    - run: ci/lib.sh
+    - run: ci/lib.sh --test
       shell: bash
     - name: select tests
       run: . /etc/profile && ci/select-test-slice.sh ${{matrix.nr}} 10
@@ -169,7 +169,7 @@ jobs:
     - name: copy dlls to root
       shell: cmd
       run: compat\vcbuild\vcpkg_copy_dlls.bat release
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
       shell: bash
     - name: generate Visual Studio solution
       shell: bash
@@ -211,7 +211,7 @@ jobs:
     - name: extract tracked files and build artifacts
       shell: bash
       run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz
-    - run: ci/lib.sh
+    - run: ci/lib.sh --test
       shell: bash
     - name: select tests
       run: . /etc/profile && ci/select-test-slice.sh ${{matrix.nr}} 10
@@ -275,8 +275,9 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
     - run: make
+    - run: ci/lib.sh --test
     - run: make test
       if: success()
     - run: ci/print-test-failures.sh
@@ -310,8 +311,9 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - run: ci/install-dependencies.sh
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
     - run: make
+    - run: ci/lib.sh --test
     - run: make test
       if: success() && matrix.vector.skip-tests != 'no'
     - run: ci/print-test-failures.sh
@@ -331,7 +333,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
     - run: make ci-static-analysis
   sparse:
     needs: ci-config
@@ -352,7 +354,7 @@ jobs:
     - uses: actions/checkout@v2
     - name: Install other dependencies
       run: ci/install-dependencies.sh
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
     - run: make sparse
   documentation:
     name: documentation
@@ -364,7 +366,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
     - run: make check-docs
     - run: "make doc > >(tee stdout.log) 2> >(tee stderr.raw >&2)"
       shell: bash
ci/lib.sh +23 -7
index 8f7e178f3ff..50bc766254b 100755
@@ -5,6 +5,13 @@ set -ex
 . ${0%/*}/lib-ci-type.sh
 
 # Starting assertions
+mode=$1
+if test -z "$mode"
+then
+	echo "need a $0 mode, e.g. --build or --test" >&2
+	exit 1
+fi
+
 if test -z "$jobname"
 then
 	echo "must set a CI jobname in the environment" >&2
@@ -16,11 +23,12 @@ setenv () {
 	while test $# != 0
 	do
 		case "$1" in
-		--build)
-			;;
-		--test)
-			;;
-		--all)
+		--build | --test)
+			if test "$1" != "$mode"
+			then
+				return 0
+			fi
+
 			;;
 		-*)
 			echo "BUG: bad setenv() option '$1'" >&2
@@ -46,8 +54,12 @@ setenv () {
 # How many jobs to run in parallel?
 NPROC=10
 
+# For "--test" we carry the MAKEFLAGS over from earlier steps, except
+# in stand-alone jobs which will use $COMMON_MAKEFLAGS.
+COMMON_MAKEFLAGS=--jobs=$NPROC
+
 # Clear MAKEFLAGS that may come from the outside world.
-MAKEFLAGS=--jobs=$NPROC
+MAKEFLAGS=$COMMON_MAKEFLAGS
 
 case "$CI_TYPE" in
 github-actions)
@@ -101,6 +113,9 @@ windows-build)
 	setenv --build NO_PERL NoThanks
 	setenv --build ARTIFACTS_DIRECTORY artifacts
 	;;
+windows-test)
+	setenv --test MAKEFLAGS "$COMMON_MAKEFLAGS"
+	;;
 vs-build)
 	setenv --build NO_PERL NoThanks
 	setenv --build NO_GETTEXT NoThanks
@@ -117,6 +132,7 @@ vs-build)
 	;;
 vs-test)
 	setenv --test NO_SVN_TESTS YesPlease
+	setenv --test MAKEFLAGS "$COMMON_MAKEFLAGS"
 	;;
 linux-gcc)
 	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME main
@@ -162,4 +178,4 @@ linux-leaks)
 	;;
 esac
 
-setenv --all MAKEFLAGS "$MAKEFLAGS CC=${CC:-cc}"
+setenv --build MAKEFLAGS "$MAKEFLAGS CC=${CC:-cc}"
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#140 lore raw

PATCH v3 24/29 CI: add more variables to MAKEFLAGS, except under vs-build

In reply to Ævar Arnfjörð Bjarmason

It's clearer that "DEVELOPER" is a flag that affects the Makefile itself in particular if it's put into "MAKEFLAGS than" if it generically sits in the environment. Let's move both it and "SKIP_DASHED_BUILT_INS" to "MAKEFLAGS".

We can't do this under vs-build, since that invokes cmake. Let's have only that job set these in the environment.

---
 ci/lib.sh | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)
ci/lib.sh +10 -2
index 50bc766254b..8fb0bfd43e1 100755
@@ -61,6 +61,13 @@ COMMON_MAKEFLAGS=--jobs=$NPROC
 # Clear MAKEFLAGS that may come from the outside world.
 MAKEFLAGS=$COMMON_MAKEFLAGS
 
+# Use common options for "make" (cmake in "vs-build" below uses the
+# intermediate variables directly)
+DEVELOPER=1
+MAKEFLAGS="$MAKEFLAGS DEVELOPER=$DEVELOPER"
+SKIP_DASHED_BUILT_INS=YesPlease
+MAKEFLAGS="$MAKEFLAGS SKIP_DASHED_BUILT_INS=$SKIP_DASHED_BUILT_INS"
+
 case "$CI_TYPE" in
 github-actions)
 	CC="${CC:-gcc}"
@@ -77,10 +84,8 @@ github-actions)
 	;;
 esac
 
-setenv --build DEVELOPER 1
 setenv --test DEFAULT_TEST_TARGET prove
 setenv --test GIT_TEST_CLONE_2GB true
-setenv --build SKIP_DASHED_BUILT_INS YesPlease
 
 case "$runs_on_pool" in
 ubuntu-latest)
@@ -117,6 +122,9 @@ windows-test)
 	setenv --test MAKEFLAGS "$COMMON_MAKEFLAGS"
 	;;
 vs-build)
+	setenv --build DEVELOPER $DEVELOPER
+	setenv --build SKIP_DASHED_BUILT_INS $SKIP_DASHED_BUILT_INS
+
 	setenv --build NO_PERL NoThanks
 	setenv --build NO_GETTEXT NoThanks
 	setenv --build ARTIFACTS_DIRECTORY artifacts
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#141 lore raw

PATCH v3 25/29 CI: set CC in MAKEFLAGS directly, don't add it to the environment

In reply to Ævar Arnfjörð Bjarmason

Rather than pass a "$CC" and "$CC_PACKAGE" in the environment to be picked up in ci/lib.sh let's instead have ci/lib.sh itself add it directly to MAKEFLAGS.

Setting CC=gcc by default made for confusing trace output, and since a preceding change to carry it and others over across "steps" in the GitHub CI it's been even more misleading. E.g. the "win+VS build" job confusingly has CC=gcc set, even though it builds with MSVC.

Let's instead reply on the Makefile default of CC=cc, and only override it for those jobs where it's needed. This does mean that we'll need to set it for the "pedantic" job, which previously relied on the default CC=gcc in case "clang" become the default on that platform.

This partially reverts my 707d2f2fe86 (CI: use "$runs_on_pool", not "$jobname" to select packages & config, 2021-11-23), i.e. we're now aiming to only set those variables specific jobs need.

---
 .github/workflows/main.yml | 13 -------------
 ci/lib.sh                  | 27 ++++++++++++++++++++++++---
 2 files changed, 24 insertions(+), 16 deletions(-)
.github/workflows/main.yml +0 -13
index 0787cadc76b..6d25ec4ae3b 100644
@@ -238,37 +238,24 @@ jobs:
       matrix:
         vector:
           - jobname: linux-clang
-            cc: clang
             pool: ubuntu-latest
           - jobname: linux-sha256
-            cc: clang
             os: ubuntu
             pool: ubuntu-latest
           - jobname: linux-gcc
-            cc: gcc
-            cc_package: gcc-8
             pool: ubuntu-latest
           - jobname: linux-TEST-vars
-            cc: gcc
             os: ubuntu
-            cc_package: gcc-8
             pool: ubuntu-latest
           - jobname: osx-clang
-            cc: clang
             pool: macos-latest
           - jobname: osx-gcc
-            cc: gcc
-            cc_package: gcc-9
             pool: macos-latest
           - jobname: linux-gcc-default
-            cc: gcc
             pool: ubuntu-latest
           - jobname: linux-leaks
-            cc: gcc
             pool: ubuntu-latest
     env:
-      CC: ${{matrix.vector.cc}}
-      CC_PACKAGE: ${{matrix.vector.cc_package}}
       jobname: ${{matrix.vector.jobname}}
       runs_on_pool: ${{matrix.vector.pool}}
     runs-on: ${{matrix.vector.pool}}
ci/lib.sh +24 -3
index 8fb0bfd43e1..c73b107d9c7 100755
@@ -51,6 +51,10 @@ setenv () {
 	fi
 }
 
+# Clear variables that may come from the outside world.
+CC=
+CC_PACKAGE=
+
 # How many jobs to run in parallel?
 NPROC=10
 
@@ -70,8 +74,6 @@ MAKEFLAGS="$MAKEFLAGS SKIP_DASHED_BUILT_INS=$SKIP_DASHED_BUILT_INS"
 
 case "$CI_TYPE" in
 github-actions)
-	CC="${CC:-gcc}"
-
 	setenv --test GIT_PROVE_OPTS "--timer --jobs $NPROC"
 	GIT_TEST_OPTS="--verbose-log -x"
 	test Windows != "$RUNNER_OS" ||
@@ -143,9 +145,16 @@ vs-test)
 	setenv --test MAKEFLAGS "$COMMON_MAKEFLAGS"
 	;;
 linux-gcc)
+	CC=gcc
+	CC_PACKAGE=gcc-8
 	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME main
 	;;
+linux-gcc-default)
+	CC=gcc
+	;;
 linux-TEST-vars)
+	CC=gcc
+	CC_PACKAGE=gcc-8
 	setenv --test GIT_TEST_SPLIT_INDEX yes
 	setenv --test GIT_TEST_MERGE_ALGORITHM recursive
 	setenv --test GIT_TEST_FULL_IN_PACK_ARRAY true
@@ -160,13 +169,23 @@ linux-TEST-vars)
 	setenv --test GIT_TEST_WRITE_REV_INDEX 1
 	setenv --test GIT_TEST_CHECKOUT_WORKERS 2
 	;;
+osx-gcc)
+	CC=gcc
+	CC_PACKAGE=gcc-9
+	;;
+osx-clang)
+	CC=clang
+	;;
 linux-clang)
+	CC=clang
 	setenv --test GIT_TEST_DEFAULT_HASH sha1
 	;;
 linux-sha256)
+	CC=clang
 	setenv --test GIT_TEST_DEFAULT_HASH sha256
 	;;
 pedantic)
+	CC=gcc
 	# Don't run the tests; we only care about whether Git can be
 	# built.
 	setenv --build DEVOPTS pedantic
@@ -181,9 +200,11 @@ linux-musl)
 	MAKEFLAGS="$MAKEFLAGS GIT_TEST_UTF8_LOCALE=C.UTF-8"
 	;;
 linux-leaks)
+	CC=gcc
 	setenv --build SANITIZE leak
 	setenv --test GIT_TEST_PASSING_SANITIZE_LEAK true
 	;;
 esac
 
-setenv --build MAKEFLAGS "$MAKEFLAGS CC=${CC:-cc}"
+MAKEFLAGS="$MAKEFLAGS${CC:+ CC=$CC}"
+setenv --build MAKEFLAGS "$MAKEFLAGS"
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#142 lore raw

Re: [PATCH v3 25/29] CI: set CC in MAKEFLAGS directly, don't add it to the environment

Quoted 9 lines · Ævar Arnfjörð Bjarmason

Rather than pass a "$CC" and "$CC_PACKAGE" in the environment to be picked up in ci/lib.sh let's instead have ci/lib.sh itself add it directly to MAKEFLAGS.

Setting CC=gcc by default made for confusing trace output, and since a preceding change to carry it and others over across "steps" in the GitHub CI it's been even more misleading. E.g. the "win+VS build" job confusingly has CC=gcc set, even though it builds with MSVC.

Quoted 5 lines

Let's instead reply on the Makefile default of CC=cc, and only override it for those jobs where it's needed. This does mean that we'll need to set it for the "pedantic" job, which previously relied on the default CC=gcc in case "clang" become the default on that platform.

s/reply/rely/

Quoted 5 lines

This partially reverts my 707d2f2fe86 (CI: use "$runs_on_pool", not "$jobname" to select packages & config, 2021-11-23), i.e. we're now aiming to only set those variables specific jobs need.

#143 lore raw

Re: [PATCH v3 25/29] CI: set CC in MAKEFLAGS directly, don't add it to the environment

In reply to Ævar Arnfjörð Bjarmason

ci/lib.sh linux-TEST-vars) · line 174

Quoted 7 lines · Ævar Arnfjörð Bjarmason
ci/lib.sh +3 -0
@@ -160,13 +169,23 @@ linux-TEST-vars)
        setenv --test GIT_TEST_WRITE_REV_INDEX 1
        setenv --test GIT_TEST_CHECKOUT_WORKERS 2
        ;;
+osx-gcc)
+       CC=gcc
+       CC_PACKAGE=gcc-9

not sure when this was broken since there were too many refactorings around this code, but this is definitely wrong.

macos' gcc is really clang, so if we really want to build with gcc instead (and there are 9, 10 and 11 versions installed) need to use instead (for version 9, which was what was used originally and what CC_PACKAGE was installing if needed)

CC=gcc-9

Right now both macos jobs are using clang, regardless of what the nicely named label says.

Carlo

#144 lore raw

Re: [PATCH v3 25/29] CI: set CC in MAKEFLAGS directly, don't add it to the environment

Quoted 18 lines · Carlo Arenas

ci/lib.sh linux-TEST-vars) · line 174

Quoted 7 lines · Ævar Arnfjörð Bjarmason
ci/lib.sh +3 -0
@@ -160,13 +169,23 @@ linux-TEST-vars)
        setenv --test GIT_TEST_WRITE_REV_INDEX 1
        setenv --test GIT_TEST_CHECKOUT_WORKERS 2
        ;;
+osx-gcc)
+       CC=gcc
+       CC_PACKAGE=gcc-9

not sure when this was broken since there were too many refactorings around this code, but this is definitely wrong.

macos' gcc is really clang, so if we really want to build with gcc instead (and there are 9, 10 and 11 versions installed) need to use instead (for version 9, which was what was used originally and what CC_PACKAGE was installing if needed)

Quoted 4 lines

CC=gcc-9

Right now both macos jobs are using clang, regardless of what the nicely named label says.

I didn't know gcc on OSX was clang, that does seem broken.

But unless I'm missing something that's already been the case on "master" for a while, i.e. this is the master run showing that we'll invoke "gcc":
https://github.com/git/git/runs/6031562726?check_suite_focus=true#step:3:6

And "seen", with this change, which shows that we'll do the same:
https://github.com/git/git/runs/6031564900?check_suite_focus=true#step:5:7

If I understand you correctly both are effectively a NOOP and we don't use that "gcc-9", but we should.

It looks like it was broken in my 707d2f2fe86 (CI: use "$runs_on_pool", not "$jobname" to select packages & config, 2021-11-23).

I'll fix that as along with any small follow-ups after this series, sound good?

#145 lore raw

Re: [PATCH v3 25/29] CI: set CC in MAKEFLAGS directly, don't add it to the environment

Quoted 2 lines · Ævar Arnfjörð Bjarmason

I'll fix that as along with any small follow-ups after this series, sound good?

Without any small follow-ups would be much much more preferrrable, and leave the follow-up to a separate series.

A series doing too many things at once can mix bad apples slip in by overwhelming the reviewer bandwidth, and I am afraid that it may be what happened in the original "bunch of Makefile updates" that the bug originated from. I am guessing that Carlo meant the same when he said "since there were too many refactorings around this code".

Thanks.

#146 lore raw

Re: [PATCH v3 25/29] CI: set CC in MAKEFLAGS directly, don't add it to the environment

In reply to Ævar Arnfjörð Bjarmason
Quoted 5 lines · Ævar Arnfjörð Bjarmason

But unless I'm missing something that's already been the case on "master" for a while, i.e. this is the master run showing that we'll invoke "gcc":
https://github.com/git/git/runs/6031562726?check_suite_focus=true#step:3:6

That would be unfortunate, considering the code in next (and all other not yet affected) branches still has the definition added in 176441bfb58 (ci: build Git with GCC 9 in the 'osx-gcc' build job, 2019-11-27) and which survived your travis-ci removal series.

Note that the value of CC is only relevant when the code is built, and GitHub actions provides its own for all tasks and at least this "mock" run (which adds $CC --version before make) with next seems to indicate the right compiler is still being used there and hopefully also in all other unaffected runs from all stable branches:

https://github.com/carenas/git/runs/6041742809

Eitherway, making sure that we really use gcc in the jobs that are tagged as such will make these extra runs (and their additional CPU time) less wasteful and should be corrected ASAP.

Carlo

#147 lore raw

Re: [PATCH v3 25/29] CI: set CC in MAKEFLAGS directly, don't add it to the environment

Quoted 9 lines · Carlo Arenas
Quoted 5 lines · Ævar Arnfjörð Bjarmason

But unless I'm missing something that's already been the case on "master" for a while, i.e. this is the master run showing that we'll invoke "gcc":
https://github.com/git/git/runs/6031562726?check_suite_focus=true#step:3:6

Quoted 4 lines

That would be unfortunate, considering the code in next (and all other not yet affected) branches still have the definition added in 176441bfb58 (ci: build Git with GCC 9 in the 'osx-gcc' build job, 2019-11-27) and which survived your travis-ci removal series.

Nevermind, my test was flawed, this[1] updated run does show this code was indeed refactored away and the bug also introduced to next and beyond.

Carlo

[1] https://github.com/carenas/git/runs/6042506530

#148 lore raw

PATCH v3 26/29 CI: set SANITIZE=leak in MAKEFLAGS directly

In reply to Ævar Arnfjörð Bjarmason

As in the preceding commit's change to do likewise for CC, let's not set SANITIZE in the environment. When I added the "linux-leaks" job in in 956d2e4639b (tests: add a test mode for SANITIZE=leak, run it in CI, 2021-09-23) I missed that I could do that.

---
 ci/lib.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
ci/lib.sh +1 -1
index c73b107d9c7..b439c1a4f82 100755
@@ -201,7 +201,7 @@ linux-musl)
 	;;
 linux-leaks)
 	CC=gcc
-	setenv --build SANITIZE leak
+	MAKEFLAGS="$MAKEFLAGS SANITIZE=leak"
 	setenv --test GIT_TEST_PASSING_SANITIZE_LEAK true
 	;;
 esac
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#149 lore raw

PATCH v3 27/29 CI: set PYTHON_PATH setting for osx-{clang,gcc} into "$jobname" case

In reply to Ævar Arnfjörð Bjarmason

Move the setting of "PYTHON_PATH=[...]" to the "$jobname" case statement. This partially backs out of my 707d2f2fe86 (CI: use "$runs_on_pool", not "$jobname" to select packages & config, 2021-11-23), now that we have a "osx-{clang,gcc}" anyway for setting "$CC" we might as well do away with this part of the "$runs_on_pool" case.

---
 ci/lib.sh | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)
ci/lib.sh +2 -8
index b439c1a4f82..d62cd757ef4 100755
@@ -105,14 +105,6 @@ ubuntu-latest)
 
 	setenv --test GIT_TEST_HTTPD true
 	;;
-macos-latest)
-	if test "$jobname" = osx-gcc
-	then
-		MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
-	else
-		MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python2)"
-	fi
-	;;
 esac
 
 case "$jobname" in
@@ -170,10 +162,12 @@ linux-TEST-vars)
 	setenv --test GIT_TEST_CHECKOUT_WORKERS 2
 	;;
 osx-gcc)
+	MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
 	CC=gcc
 	CC_PACKAGE=gcc-9
 	;;
 osx-clang)
+	MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python2)"
 	CC=clang
 	;;
 linux-clang)
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#150 lore raw

PATCH v3 28/29 CI: don't use "set -x" in "ci/lib.sh" output

In reply to Ævar Arnfjörð Bjarmason

Remove the "set -x" from the "ci/lib.sh" output. Before preceding commits the logic in that file was much more complex, and likely to fail in some scenarios.

Now we only task "ci/lib.sh" with setting various variables for subsequent steps in our jobs, so we can start emitting more tailored debugging output, which makes what it's doing easier to read.

This change also changes the output of the "ci/print-test-failures.sh" script, since it's the only other user of "ci/lib-ci-type.sh". In that case it's also helpful to know what "$CI_TYPE" we're using, as that script doesn't "set -x" and will act differently depending on the $CI_TYPE.

---
 ci/lib.sh | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)
ci/lib.sh +21 -2
index d62cd757ef4..94f83069884 100755
@@ -1,5 +1,5 @@
 #!/bin/sh
-set -ex
+set -e
 
 # Helper libraries
 . ${0%/*}/lib-ci-type.sh
@@ -18,15 +18,26 @@ then
 	exit 1
 fi
 
+# Show our configuration
+echo "CONFIG: CI_TYPE=$CI_TYPE" >&2
+echo "CONFIG: jobname=$jobname" >&2
+echo "CONFIG: runs_on_pool=$runs_on_pool" >&2
+echo "CONFIG: GITHUB_ENV=$GITHUB_ENV" >&2
+if test -n "$GITHUB_ENV"
+then
+	echo "CONFIG: GITHUB_ENV=$GITHUB_ENV" >&2
+fi
+
 # Helper functions
 setenv () {
+	local skip=
 	while test $# != 0
 	do
 		case "$1" in
 		--build | --test)
 			if test "$1" != "$mode"
 			then
-				return 0
+				skip=t
 			fi
 
 			;;
@@ -45,10 +56,18 @@ setenv () {
 	val=$2
 	shift 2
 
+	if test -n "$skip"
+	then
+		echo "SKIP '$key=$val'" >&2
+		return 0
+	fi
+
 	if test -n "$GITHUB_ENV"
 	then
 		echo "$key=$val" >>"$GITHUB_ENV"
 	fi
+
+	echo "SET: '$key=$val'" >&2
 }
 
 # Clear variables that may come from the outside world.
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#151 lore raw

PATCH v3 29/29 CI: make it easy to use ci/*.sh outside of CI

In reply to Ævar Arnfjörð Bjarmason

In preceding commits the ci/.sh scripts have lost most of their CI-specific assumptions. Let's go even further and explicitly support running ci/lib.sh outside of CI.

This was possible before by faking up enough CI-specific variables, but as shown in the new "help" output being added here using the ci/lib.sh to provide "CI-like" has now become trivial.

The ci/print-test-failures.sh scripts can now be used outside of CI as well, the only GitHub CI-specific part is now guarded by a check that we'll pass if outside of GitHub CI.

There's also a special-case here to not clobber $MAKEFLAGS in the environment if we're outside of CI, in case the user has e.g. "jN" or other flags to "make" that they'd prefer configured already.

---
 ci/lib-ci-type.sh         |  3 --
 ci/lib.sh                 | 61 ++++++++++++++++++++++++++++++++++++---
 ci/print-test-failures.sh | 11 ++-----
 3 files changed, 60 insertions(+), 15 deletions(-)
ci/lib-ci-type.sh +0 -3
index 6f01fd9e5d9..09acab7aaec 100644
@@ -3,7 +3,4 @@
 if test "$GITHUB_ACTIONS" = "true"
 then
 	CI_TYPE=github-actions
-else
-	echo "Could not identify CI type" >&2
-	exit 1
 fi
ci/lib.sh +57 -4

98 lines not on this page · raw message

ci/print-test-failures.sh +3 -8
index 2405f65650a..ba8428ad484 100755
@@ -38,19 +38,14 @@ do
 		test_name="${TEST_EXIT%.exit}"
 		test_name="${test_name##*/}"
 		trash_dir="trash directory.$test_name"
-		case "$CI_TYPE" in
-		github-actions)
+		if test "$CI_TYPE" = "github-actions"
+		then
 			mkdir -p failed-test-artifacts
 			echo "FAILED_TEST_ARTIFACTS=t/failed-test-artifacts" >>$GITHUB_ENV
 			cp "${TEST_EXIT%.exit}.out" failed-test-artifacts/
 			tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir"
 			continue
-			;;
-		*)
-			echo "Unhandled CI type: $CI_TYPE" >&2
-			exit 1
-			;;
-		esac
+		fi
 		trash_tgz_b64="trash.$test_name.base64"
 		if [ -d "$trash_dir" ]
 		then
Signature · 1 line
2.36.0.rc2.843.g193535c2aa7
#152 lore raw

Re: [PATCH v3 29/29] CI: make it easy to use ci/*.sh outside of CI

Quoted 45 lines · Ævar Arnfjörð Bjarmason

In preceding commits the ci/.sh scripts have lost most of their CI-specific assumptions. Let's go even further and explicitly support running ci/lib.sh outside of CI.

This was possible before by faking up enough CI-specific variables, but as shown in the new "help" output being added here using the ci/lib.sh to provide "CI-like" has now become trivial.

The ci/print-test-failures.sh scripts can now be used outside of CI as well, the only GitHub CI-specific part is now guarded by a check that we'll pass if outside of GitHub CI.

There's also a special-case here to not clobber $MAKEFLAGS in the environment if we're outside of CI, in case the user has e.g. "jN" or other flags to "make" that they'd prefer configured already.

---

ci/lib.sh +24 -0
@@ -1,6 +1,30 @@
+#  Usage
+CI_TYPE_HELP_COMMANDS='
+	# run "make all test" like the "linux-leaks" job
+	(eval $(jobname=linux-leaks ci/lib.sh --all) && make test)
+
+	# run "make all test" like the "linux-musl" job
+	(eval $(jobname=linux-musl ci/lib.sh --all) && make test)
+
+	# run "make test" like the "linux-TEST-vars" job (uses various GIT_TEST_* modes)
+	make && (eval $(jobname=linux-TEST-vars ci/lib.sh --test) && make test)
+
+	# run "make test" like the "linux-sha256" job
+	make && (eval $(jobname=linux-sha256 ci/lib.sh --test) && make test)
+'
+
+CI_TYPE_HELP="
+running $0 outside of CI? You can use ci/lib.sh to set up your
+environment like a given CI job. E.g.:
+$CI_TYPE_HELP_COMMANDS
+
+note that some of these (e.g. the linux-musl one) may not work as
+expected due to the CI job configuring a platform that may not match
+yours."
+

ci/lib.sh mode=$1 · line 41

Quoted 9 lines
ci/lib.sh +4 -0
@@ -9,6 +33,10 @@ mode=$1
  if test -z "$mode"
  then
  	echo "need a $0 mode, e.g. --build or --test" >&2
+	if test -z "$CI_TYPE"
+	then
+		echo "$CI_TYPE_HELP" >&2
+	fi
  	exit 1
  fi

It would never occur to me to try running a script named ci/lib.sh in the first place, and I'm not sure I would even think to poke around in the ci directory with the idea of being able to run CI tasks locally. Would it make sense to aid discovery by mentioning this new feature in the "GitHub CI" section of SubmittingPatches as a follow up to this series? (If so, perhaps the "GitHub CI" section should be renamed to "CI".)

ci/lib.sh CC_PACKAGE= · line 112

Quoted 6 lines
ci/lib.sh +3 -0
@@ -76,10 +108,29 @@ CC_PACKAGE=
  # How many jobs to run in parallel?
  NPROC=10
+case "$CI_TYPE" in
+'')
+	if command -v nproc >/dev/null

You need to redirect stderr too in order to avoid a scary error message on platforms lacking nproc:

if command -v nproc >/dev/null 2>&1

ci/lib.sh CC_PACKAGE= · line 118

Quoted 5 lines
ci/lib.sh +5 -0
+	then
+		NPROC=$(nproc)
+	else
+		NPROC=1
+	fi

Neither macOS nor BSD has an nproc command, however, they do have sysctl:

elif command -v sysctl >/dev/null 2>&1
then # macOS & BSD
NPROC=$(sysctl -n hw.ncpu 2>/dev/null)

and Windows provides a suitable environment variable:

elif test -n "$NUMBER_OF_PROCESSORS"
then # Windows
NPROC="$NUMBER_OF_PROCESSORS"

Quoted 7 lines
ci/lib.sh +7 -0
+	if test -n "$MAKEFLAGS"
+	then
+		COMMON_MAKEFLAGS="$MAKEFLAGS"
+	else
+		COMMON_MAKEFLAGS=--jobs=$NPROC
+	fi
+	;;
#153 lore raw

Re: [PATCH v3 29/29] CI: make it easy to use ci/*.sh outside of CI

Quoted 57 lines · Eric Sunshine
Quoted 20 lines · Ævar Arnfjörð Bjarmason

In preceding commits the ci/.sh scripts have lost most of their CI-specific assumptions. Let's go even further and explicitly support running ci/lib.sh outside of CI.

This was possible before by faking up enough CI-specific variables, but as shown in the new "help" output being added here using the ci/lib.sh to provide "CI-like" has now become trivial.

The ci/print-test-failures.sh scripts can now be used outside of CI as well, the only GitHub CI-specific part is now guarded by a check that we'll pass if outside of GitHub CI.

There's also a special-case here to not clobber $MAKEFLAGS in the environment if we're outside of CI, in case the user has e.g. "jN" or other flags to "make" that they'd prefer configured already.

---

ci/lib.sh line 9

Quoted 9 lines
ci/lib.sh +9 -0
@@ -1,6 +1,30 @@
+#  Usage
+CI_TYPE_HELP_COMMANDS='
+	# run "make all test" like the "linux-leaks" job
+	(eval $(jobname=linux-leaks ci/lib.sh --all) && make test)
+
+	# run "make all test" like the "linux-musl" job
+	(eval $(jobname=linux-musl ci/lib.sh --all) && make test)
+
+	# run "make test" like the "linux-TEST-vars" job (uses various

GIT_TEST_* modes)

Quoted 16 lines
ci/lib.sh +15 -0
+	make && (eval $(jobname=linux-TEST-vars ci/lib.sh --test) && make test)
+
+	# run "make test" like the "linux-sha256" job
+	make && (eval $(jobname=linux-sha256 ci/lib.sh --test) && make test)
+'
+
+CI_TYPE_HELP="
+running $0 outside of CI? You can use ci/lib.sh to set up your
+environment like a given CI job. E.g.:
+$CI_TYPE_HELP_COMMANDS
+
+note that some of these (e.g. the linux-musl one) may not work as
+expected due to the CI job configuring a platform that may not match
+yours."
+

ci/lib.sh mode=$1 · line 41

Quoted 9 lines
ci/lib.sh +4 -0
@@ -9,6 +33,10 @@ mode=$1
  if test -z "$mode"
  then
  	echo "need a $0 mode, e.g. --build or --test" >&2
+	if test -z "$CI_TYPE"
+	then
+		echo "$CI_TYPE_HELP" >&2
+	fi
  	exit 1
  fi
Quoted 7 lines

It would never occur to me to try running a script named ci/lib.sh in the first place, and I'm not sure I would even think to poke around in the ci directory with the idea of being able to run CI tasks locally. Would it make sense to aid discovery by mentioning this new feature in the "GitHub CI" section of SubmittingPatches as a follow up to this series? (If so, perhaps the "GitHub CI" section should be renamed to "CI".)

Yes, I'd like to do that, but as a follow-up, specifically to have failing tests in CI suggest a way to run them locally.

It would be better as ci/config.sh or something, but it was ci/lib.sh already, and this way minimized the diff/conflicts.

Quoted 11 lines

ci/lib.sh CC_PACKAGE= · line 112

Quoted 6 lines
ci/lib.sh +3 -0
@@ -76,10 +108,29 @@ CC_PACKAGE=
  # How many jobs to run in parallel?
  NPROC=10
+case "$CI_TYPE" in
+'')
+	if command -v nproc >/dev/null

You need to redirect stderr too in order to avoid a scary error message on platforms lacking nproc:

if command -v nproc >/dev/null 2>&1

I figured the scary error would be useful, it's only if you run it ad-hoc, so it would be a nice hint as to why we fallback on the default nproc.

Quoted 13 lines

ci/lib.sh CC_PACKAGE= · line 118

Quoted 5 lines
ci/lib.sh +5 -0
+	then
+		NPROC=$(nproc)
+	else
+		NPROC=1
+	fi

Neither macOS nor BSD has an nproc command, however, they do have sysctl:

elif command -v sysctl >/dev/null 2>&1
then # macOS & BSD
NPROC=$(sysctl -n hw.ncpu 2>/dev/null)

Quoted 5 lines

and Windows provides a suitable environment variable:

elif test -n "$NUMBER_OF_PROCESSORS"
then # Windows
NPROC="$NUMBER_OF_PROCESSORS"

*nod*

I'd rather not finish this part up though and instead just have this call some moral equivalent of t/helper/test-online-cpus.c.

Most of our CI is linux, so just nproc for now felt like a good initial addition which we could expand on later.

Quoted 7 lines
Quoted 7 lines
ci/lib.sh +7 -0
+	if test -n "$MAKEFLAGS"
+	then
+		COMMON_MAKEFLAGS="$MAKEFLAGS"
+	else
+		COMMON_MAKEFLAGS=--jobs=$NPROC
+	fi
+	;;
#154 lore raw

Re: [PATCH v3 00/29] CI: run "make" in CI "steps", improve UX

In reply to Ævar Arnfjörð Bjarmason
Quoted 4 lines · Ævar Arnfjörð Bjarmason

* The rather minor in code but rather big in UX terms is that I've added 29/29, which explicitly documents and shows examples of using the CI tooling in a way where you can locally run a CI target as if though it were a given $jobname, e.g.:

Yay.

Quoted 2 lines

Likewise the ci/print-test-failures.sh script can now be run locally without any extra mock-ups, e.g.:

Double yay.

Quoted 8 lines

@@ ci/lib.sh
+# Starting assertions
+if test -z "$jobname"
+then
-+ echo "must set a CI jobname" >&2
++ echo "must set a CI jobname in the environment" >&2
+ exit 1
+fi

OK. In the truely-automated-machine-initiated run, this should not matter, but once we start allowing end-user to interact with the guts of the CI machinery, being as helpful as possible matters.

#155 lore raw

v4 cover CI: run "make" in CI "steps", improve UX

In reply to Ævar Arnfjörð Bjarmason

This series changes CI "step" targets that are shellscripts that do N things to instead be single command invocations at the "step" level, driven by the CI recipe itself.

To do that we need to pass state that we previously re-setup for every "step" via $GITHUB_ENV, whose state we then helpfully show (this is just a standard GitHub CI feature) in a drop-down at the start of every "step".

I.e. at the tip of this series you can reliably look at that $GITHUB_ENV view to see what the full and relevant environment was for that "make", "make test" or whatever.

Changes since v3:

* Typo/grammar fixes pointed out by Eric Sunshine

* The ci/print-test-failures.sh scripts still had some dead Travis code (noticed while re-rolling). Removed it, and squashed into the travis removal commit.

* Split up the part of "CI: combine ci/install{,-docker}-dependencies.sh" that was replacing bash constructs with POSIX ones into a preceding commit doing only that.

* There was a regression in "CI: set CC in MAKEFLAGS directly, don't add it to the environment" where we didn't properly install the right CI package. The whole CC_PACKAGE is now contained within "ci/install-dependencies.sh".

* In addition to the regression Carlo noted already on "master" where the osx-gcc job is using clang. That's now fixed in a new "CI: have osx-gcc use gcc, not clang".

* Extend the last commit to have an "nproc" on BSD/OSX/Windows too, this is only for the case of ad-hoc running ci/lib.sh locally.

Ævar Arnfjörð Bjarmason (31):
CI: run "set -ex" early in ci/lib.sh
CI: make "$jobname" explicit, remove fallback
CI: remove more dead Travis CI support
CI: remove dead "tree skipping" code
CI: remove unused Azure ci/* code
CI/lib.sh: stop adding leading whitespace to $MAKEFLAGS
CI: don't have "git grep" invoke a pager in tree content check
CI: have "static-analysis" run a "make ci-static-analysis" target
CI: have "static-analysis" run "check-builtins", not "documentation"
CI: move p4 and git-lfs variables to ci/install-dependencies.sh
CI: consistently use "export" in ci/lib.sh
CI: export variables via a wrapper
CI: remove "run-build-and-tests.sh", run "make [test]" directly ci/lib.sh: use "test" instead of "["
CI: check ignored unignored build artifacts in "win[+VS] build" too
CI: invoke "make artifacts-tar" directly in windows-build
CI: split up and reduce "ci/test-documentation.sh"
CI: make ci/install-dependencies.sh POSIX-compatible
CI: combine ci/install{,-docker}-dependencies.sh
CI: move "env" definitions into ci/lib.sh ci/run-test-slice.sh: replace shelling out with "echo"
CI: pre-select test slice in Windows & VS tests
CI: only invoke ci/lib.sh as "steps" in main.yml
CI: narrow down variable definitions in --build and --test
CI: add more variables to MAKEFLAGS, except under vs-build
CI: set CC in MAKEFLAGS directly, don't add it to the environment
CI: set SANITIZE=leak in MAKEFLAGS directly
CI: set PYTHON_PATH setting for osx-{clang,gcc} into "$jobname" case
CI: don't use "set -x" in "ci/lib.sh" output
CI: have osx-gcc use gcc, not clang
CI: make it easy to use ci/*.sh outside of CI

 .github/workflows/main.yml            | 100 ++++---
 Makefile                              |  31 ++-
 ci/check-directional-formatting.bash  |   2 +-
 ci/check-unignored-build-artifacts.sh |  20 ++
 ci/install-dependencies.sh            |  74 ++++-
 ci/install-docker-dependencies.sh     |  22 --
 ci/lib-ci-type.sh                     |   6 +
 ci/lib-online_cpus.sh                 |  26 ++
 ci/lib-tput.sh                        |   5 +
 ci/lib.sh                             | 379 +++++++++++++++-----------
 ci/make-test-artifacts.sh             |  12 -
 ci/mount-fileshare.sh                 |  25 --
 ci/print-test-failures.sh             |  53 +---
 ci/run-build-and-tests.sh             |  54 ----
 ci/run-docker-build.sh                |  66 -----
 ci/run-docker.sh                      |  47 ----
 ci/run-static-analysis.sh             |  32 ---
 ci/run-test-slice.sh                  |  17 --
 ci/select-test-slice.sh               |  10 +
 ci/test-documentation.sh              |  37 +--
 ci/util/extract-trash-dirs.sh         |  50 ----
 shared.mak                            |   1 +
 22 files changed, 453 insertions(+), 616 deletions(-)
 create mode 100755 ci/check-unignored-build-artifacts.sh
 delete mode 100755 ci/install-docker-dependencies.sh
 create mode 100644 ci/lib-ci-type.sh
 create mode 100644 ci/lib-online_cpus.sh
 create mode 100644 ci/lib-tput.sh
 delete mode 100755 ci/make-test-artifacts.sh
 delete mode 100755 ci/mount-fileshare.sh
 delete mode 100755 ci/run-build-and-tests.sh
 delete mode 100755 ci/run-docker-build.sh
 delete mode 100755 ci/run-docker.sh
 delete mode 100755 ci/run-static-analysis.sh
 delete mode 100755 ci/run-test-slice.sh
 create mode 100755 ci/select-test-slice.sh
 delete mode 100755 ci/util/extract-trash-dirs.sh

Range-diff against v3:

 1:  3cb4749c5ea =  1:  ed27ca0b7e7 CI: run "set -ex" early in ci/lib.sh
 2:  7de95ff437e =  2:  737fe1e2c4c CI: make "$jobname" explicit, remove fallback
 3:  820c4e551d9 !  3:  d875069fc28 CI: remove more dead Travis CI support

@@ Commit message
would do the wrong thing, because the different test slices would each try to clobber the same "t/.prove" file.

- If a subsequent run then used the -"-state=failed,slow,save" it would
+ If a subsequent run then used the "--state=failed,slow,save" it would
defeat the purpose of "ci/run-test-slice.sh", since all slices would then run all tests. I.e. behavior of prove's "--state" options is to select tests to run from the provided "--state" file, in addition to
@@ Commit message
".github/workflows/main.yml" calls the same entry points as the main "regular" job.

+ For "ci/print-test-failures.sh" the creation of the tarball added in
+ aea8879a6ac (travis-ci: include the trash directories of failed tests
+ in the trace log, 2018-08-01) has been dead since my 4a6e4b96026,
+ which removed the fall-through case from the "case/esac" statement.
+

## ci/lib.sh ##
@@ ci/lib.sh: then
echo "Could not identify CI type" >&2

## ci/print-test-failures.sh ##
-@@ ci/print-test-failures.sh: do
- fi
- combined_trash_size=$new_combined_trash_size
+@@ ci/print-test-failures.sh: then
+ exit
+ fi

+-case "$jobname" in
+-osx-clang|osx-gcc)
+- # base64 in OSX doesn't wrap its output at 76 columns by
+- # default, but prints a single, very long line.
+- base64_opts="-b 76"
+- ;;
+-esac
+-
+-combined_trash_size=0
+ for TEST_EXIT in test-results/*.exit
+ do
+ if [ "$(cat "$TEST_EXIT")" != "0" ]
+@@ ci/print-test-failures.sh: do
+ azure-pipelines)
+ mkdir -p failed-test-artifacts
+ mv "$trash_dir" failed-test-artifacts
+- continue
+ ;;
+ github-actions)
+ mkdir -p failed-test-artifacts
+ echo "FAILED_TEST_ARTIFACTS=t/failed-test-artifacts" >>$GITHUB_ENV
+ cp "${TEST_EXIT%.exit}.out" failed-test-artifacts/
+ tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir"
+- continue
+ ;;
+ *)
+ echo "Unhandled CI type: $CI_TYPE" >&2
+ exit 1
+ ;;
+ esac
+- trash_tgz_b64="trash.$test_name.base64"
+- if [ -d "$trash_dir" ]
+- then
+- tar czp "$trash_dir" |base64 $base64_opts >"$trash_tgz_b64"
+-
+- trash_size=$(wc -c <"$trash_tgz_b64")
+- if [ $trash_size -gt 1048576 ]
+- then
+- # larger than 1MB
+- echo "$(tput setaf 1)Didn't include the trash directory of '$test_name' in the trace log, it's too big$(tput sgr0)"
+- continue
+- fi
+-
+- new_combined_trash_size=$(($combined_trash_size + $trash_size))
+- if [ $new_combined_trash_size -gt 1048576 ]
+- then
+- echo "$(tput setaf 1)Didn't include the trash directory of '$test_name' in the trace log, there is plenty of trash in there already.$(tput sgr0)"
+- continue
+- fi
+- combined_trash_size=$new_combined_trash_size
+-
- # DO NOT modify these two 'echo'-ed strings below
- # without updating 'ci/util/extract-trash-dirs.sh'
- # as well.
- echo "$(tput setaf 1)Start of trash directory of '$test_name':$(tput sgr0)"
- cat "$trash_tgz_b64"
- echo "$(tput setaf 1)End of trash directory of '$test_name'$(tput sgr0)"
+- echo "$(tput setaf 1)Start of trash directory of '$test_name':$(tput sgr0)"
+- cat "$trash_tgz_b64"
+- echo "$(tput setaf 1)End of trash directory of '$test_name'$(tput sgr0)"
+- fi
+ fi
+ done

## ci/run-build-and-tests.sh ##
@@

 4:  914e81118b8 =  4:  2590459db33 CI: remove dead "tree skipping" code
 5:  b2e456f9a9b !  5:  e311ea8b9b3 CI: remove unused Azure ci/* code
    @@ ci/print-test-failures.sh: do
     -		azure-pipelines)
     -			mkdir -p failed-test-artifacts
     -			mv "$trash_dir" failed-test-artifacts
    --			continue
     -			;;
      		github-actions)
      			mkdir -p failed-test-artifacts
 6:  30968e36bdd =  6:  151f28e3818 CI/lib.sh: stop adding leading whitespace to $MAKEFLAGS
 7:  cc81b9fe37e =  7:  6c2a8ee9c4e CI: don't have "git grep" invoke a pager in tree content check
 8:  2387b0c5842 =  8:  64a0a1fc8ce CI: have "static-analysis" run a "make ci-static-analysis" target
 9:  54a4d79bf8d =  9:  0f780bf9e3c CI: have "static-analysis" run "check-builtins", not "documentation"
10:  9a31b7d5011 = 10:  3cdeae9c141 CI: move p4 and git-lfs variables to ci/install-dependencies.sh
11:  8ab4e81e1ca = 11:  1d3d357dcef CI: consistently use "export" in ci/lib.sh
12:  9fe00566767 = 12:  5ef0756d095 CI: export variables via a wrapper
13:  13feda050c0 ! 13:  e0ce614eb6d CI: remove "run-build-and-tests.sh", run "make [test]" directly

@@ Commit message

Now that verbosity is in the earlier "ci/lib.sh" step, and not in any subsequent one. The "make" targets then start out with the
- relevant output non-trace output right away.
+ relevant non-trace output right away.

* If we do want to use the grouping syntax within a "step" it'll now be easier to do so. It doesn't support nesting, so we'd have to

14:  d3909b8e896 = 14:  a8e70124929 ci/lib.sh: use "test" instead of "["
15:  d63a689eae0 = 15:  bb335d1c0ca CI: check ignored unignored build artifacts in "win[+VS] build" too
16:  ac1ffe9c642 = 16:  5458e6dab0b CI: invoke "make artifacts-tar" directly in windows-build
17:  8837bfa5433 ! 17:  5e5a5b71700 CI: split up and reduce "ci/test-documentation.sh"

@@ Commit message
parts in as one command in the CI job itself, and to run the two "make doc" commands at the top-level.

- It'll now be obvious from the title of the step if if we failed in the
+ It'll now be obvious from the title of the step if we failed in the
asciidoc or asciidoctor step.

Since the "check_unignored_build_artifacts()" function is now only

 -:  ----------- > 18:  eec7a11376e CI: make ci/install-dependencies.sh POSIX-compatible
18:  aa491990c1a ! 19:  a9c98582de1 CI: combine ci/install{,-docker}-dependencies.sh

@@ Commit message
statement in the latter only cared about "$jobname", and can be folded into the same "case" statement in the former.

- The reason they split up is historical, and because the
- "ci/install-dependencies.sh" used "ci/lib.sh", which requires
- "bash". At least one of the docker containers doesn't have "bash". To
- make the existing code POSIX-compatible we need to replace pushd/popd
- with a sub-shell, but no other changes were needed.
+ The reason they split up is historical, until a preceding commit
+ "ci/lib.sh" required "bash", which might not have been available in
+ "docker".

This also fixes issue in "ci/install-docker-dependencies.sh" where we'd hide errors due to not using "set -e". Now that we include
@@ .github/workflows/main.yml: jobs:
- run: make test

## ci/install-dependencies.sh ##
-@@
--#!/usr/bin/env bash
-+#!/bin/sh
- #
- # Install dependencies required to build and test Git on Linux and macOS
- #
-@@ ci/install-dependencies.sh: ubuntu-latest)
- sudo apt-get -q -y install language-pack-is libsvn-perl apache2 \
- $UBUNTU_COMMON_PKGS $CC_PACKAGE
- mkdir --parents "$P4_PATH"
-- pushd "$P4_PATH"
-+ (
-+ cd "$P4_PATH"
- wget --quiet "$P4WHENCE/bin.linux26x86_64/p4d"
- wget --quiet "$P4WHENCE/bin.linux26x86_64/p4"
- chmod u+x p4d
- chmod u+x p4
-- popd
-+ )
- mkdir --parents "$GIT_LFS_PATH"
-- pushd "$GIT_LFS_PATH"
-+ (
-+ cd "$GIT_LFS_PATH"
- wget --quiet "$LFSWHENCE/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz"
- tar --extract --gunzip --file "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz"
- cp git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs .
-- popd
-+ )
- ;;
- macos-latest)
- export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1
@@ ci/install-dependencies.sh: linux-gcc-default)
sudo apt-get -q update
sudo apt-get -q -y install $UBUNTU_COMMON_PKGS

19:  39b13298425 = 20:  96e7eb68906 CI: move "env" definitions into ci/lib.sh
20:  0a92c478d28 = 21:  fdbe8554f5d ci/run-test-slice.sh: replace shelling out with "echo"
21:  16ac7f5c210 = 22:  f257ab59971 CI: pre-select test slice in Windows & VS tests
22:  939ef573b75 = 23:  1d95088da29 CI: only invoke ci/lib.sh as "steps" in main.yml
23:  8e79c3e39e0 = 24:  a7434b215db CI: narrow down variable definitions in --build and --test
24:  d1d5c1e2f65 = 25:  78f4609a9c1 CI: add more variables to MAKEFLAGS, except under vs-build
25:  1268792233f ! 26:  26a34f1d4b9 CI: set CC in MAKEFLAGS directly, don't add it to the environment
    @@ Metadata
      ## Commit message ##
         CI: set CC in MAKEFLAGS directly, don't add it to the environment
     
    -    Rather than pass a "$CC" and "$CC_PACKAGE" in the environment to be
    -    picked up in ci/lib.sh let's instead have ci/lib.sh itself add it
    -    directly to MAKEFLAGS.
    +    Rather than pass a "$CC" in the environment to be picked up in
    +    ci/lib.sh let's instead have ci/lib.sh itself add it directly to
    +    MAKEFLAGS. For "$CC_PACKAGE" its setting and use can stay within
    +    ci/install-dependencies.sh.
     
         Setting CC=gcc by default made for confusing trace output, and since a
         preceding change to carry it and others over across "steps" in the
         GitHub CI it's been even more misleading.  E.g. the "win+VS build" job
         confusingly has CC=gcc set, even though it builds with MSVC.
     
    -    Let's instead reply on the Makefile default of CC=cc, and only
    -    override it for those jobs where it's needed. This does mean that
    -    we'll need to set it for the "pedantic" job, which previously relied
    -    on the default CC=gcc in case "clang" become the default on that
    -    platform.
    +    Let's instead rely on the Makefile default of CC=cc, and only override
    +    it for those jobs where it's needed. This does mean that we'll need to
    +    set it for the "pedantic" job, which previously relied on the default
    +    CC=gcc in case "clang" become the default on that platform.
     
         This partially reverts my 707d2f2fe86 (CI: use "$runs_on_pool", not
         "$jobname" to select packages & config, 2021-11-23), i.e. we're now
    @@ .github/workflows/main.yml: jobs:
            runs_on_pool: ${{matrix.vector.pool}}
          runs-on: ${{matrix.vector.pool}}
     
    + ## ci/install-dependencies.sh ##
    +@@ ci/install-dependencies.sh: UBUNTU_COMMON_PKGS="make libssl-dev libcurl4-openssl-dev libexpat-dev
    +  tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl
    +  libemail-valid-perl libio-socket-ssl-perl libnet-smtp-ssl-perl"
    + 
    ++CC_PACKAGE=
    ++case "$jobname" in
    ++linux-gcc | linux-TEST-vars)
    ++	CC_PACKAGE=gcc-8
    ++	;;
    ++osx-gcc)
    ++	CC_PACKAGE=gcc-9
    ++	;;
    ++esac
    ++
    + case "$runs_on_pool" in
    + ubuntu-latest)
    + 	# The Linux build installs the defined dependency versions below.
    +
      ## ci/lib.sh ##
     @@ ci/lib.sh: setenv () {
      	fi
    @@ ci/lib.sh: setenv () {
      
     +# Clear variables that may come from the outside world.
     +CC=
    -+CC_PACKAGE=
     +
      # How many jobs to run in parallel?
      NPROC=10
    @@ ci/lib.sh: vs-test)
      	;;
      linux-gcc)
     +	CC=gcc
    -+	CC_PACKAGE=gcc-8
      	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME main
      	;;
     +linux-gcc-default)
    @@ ci/lib.sh: vs-test)
     +	;;
      linux-TEST-vars)
     +	CC=gcc
    -+	CC_PACKAGE=gcc-8
      	setenv --test GIT_TEST_SPLIT_INDEX yes
      	setenv --test GIT_TEST_MERGE_ALGORITHM recursive
      	setenv --test GIT_TEST_FULL_IN_PACK_ARRAY true
    @@ ci/lib.sh: linux-TEST-vars)
      	;;
     +osx-gcc)
     +	CC=gcc
    -+	CC_PACKAGE=gcc-9
     +	;;
     +osx-clang)
     +	CC=clang
26:  83138dacd3e = 27:  514de8d16b0 CI: set SANITIZE=leak in MAKEFLAGS directly
27:  0508777c68e ! 28:  6501059c594 CI: set PYTHON_PATH setting for osx-{clang,gcc} into "$jobname" case
    @@ ci/lib.sh: linux-TEST-vars)
      osx-gcc)
     +	MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
      	CC=gcc
    - 	CC_PACKAGE=gcc-9
      	;;
      osx-clang)
     +	MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python2)"
28:  c0ce0fd3a80 = 29:  1bdcf1399d9 CI: don't use "set -x" in "ci/lib.sh" output
 -:  ----------- > 30:  94abb826627 CI: have osx-gcc use gcc, not clang
29:  2e3c02fa0df ! 31:  4fc67e668da CI: make it easy to use ci/*.sh outside of CI
    @@ Commit message
         environment if we're outside of CI, in case the user has e.g. "jN" or
         other flags to "make" that they'd prefer configured already.
     
    +    Using "ci/lib.sh" as a stand-alone script is much more useful if it
    +    doesn't hardcode NPROC=10, let's provide a poor shellscript
    +    replacement for the online_cpus() we have in thread-utils.c to cover
    +    the most common OS's. It was suggested to use "2>&1" to invoke
    +    "command -v", but per my reading of [2] and my own testing that
    +    doesn't seem to be needed. Perhaps it's only needed for "which(1)"?
    +
    +    1. https://lore.kernel.org/git/[email protected]/
    +    2. https://pubs.opengroup.org/onlinepubs/009604499/utilities/command.html
    +
    +    Helped-by: Eric Sunshine <[email protected]>
         Signed-off-by: Ævar Arnfjörð Bjarmason <[email protected]>
     
      ## ci/lib-ci-type.sh ##
    @@ ci/lib-ci-type.sh
     -	exit 1
      fi
     
    + ## ci/lib-online_cpus.sh (new) ##
    +@@
    ++#!/bin/sh
    ++
    ++# TODO: Ideally we'd compile t/helper/test-online-cpus.c early, but
    ++# that presents a chicken & egg problem. But if we move it to a
    ++# stand-oline command...
    ++online_cpus() {
    ++	NPROC=
    ++
    ++	if command -v nproc >/dev/null
    ++	then
    ++		# GNU coreutils
    ++		NPROC=$(nproc)
    ++	elif command -v sysctl >/dev/null
    ++	then
    ++		# BSD & Mac OS X
    ++		NPROC=$(sysctl -n hw.ncpu)
    ++	elif test -n "$NUMBER_OF_PROCESSORS"
    ++	then
    ++		# Windows
    ++		NPROC="$NUMBER_OF_PROCESSORS"
    ++	else
    ++		NPROC=1
    ++	fi
    ++
    ++	echo $NPROC
    ++}
    +
      ## ci/lib.sh ##
     @@
      #!/bin/sh
    @@ ci/lib.sh: setenv () {
      	fi
      
      	echo "SET: '$key=$val'" >&2
    -@@ ci/lib.sh: CC_PACKAGE=
    +@@ ci/lib.sh: CC=
      
      # How many jobs to run in parallel?
      NPROC=10
     +case "$CI_TYPE" in
     +'')
    -+	if command -v nproc >/dev/null
    -+	then
    -+		NPROC=$(nproc)
    -+	else
    -+		NPROC=1
    -+	fi
    ++	. ${0%/*}/lib-online_cpus.sh
    ++	NPROC=$(online_cpus)
     +
     +	if test -n "$MAKEFLAGS"
     +	then
    @@ ci/lib.sh: github-actions)
     
      ## ci/print-test-failures.sh ##
     @@ ci/print-test-failures.sh: do
    - 		test_name="${TEST_EXIT%.exit}"
    - 		test_name="${test_name##*/}"
    - 		trash_dir="trash directory.$test_name"
    --		case "$CI_TYPE" in
    --		github-actions)
    -+		if test "$CI_TYPE" = "github-actions"
    -+		then
    - 			mkdir -p failed-test-artifacts
    - 			echo "FAILED_TEST_ARTIFACTS=t/failed-test-artifacts" >>$GITHUB_ENV
      			cp "${TEST_EXIT%.exit}.out" failed-test-artifacts/
      			tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir"
    - 			continue
    --			;;
    + 			;;
     -		*)
     -			echo "Unhandled CI type: $CI_TYPE" >&2
     -			exit 1
     -			;;
    --		esac
    -+		fi
    - 		trash_tgz_b64="trash.$test_name.base64"
    - 		if [ -d "$trash_dir" ]
    - 		then
    + 		esac
    + 	fi
    + done
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#156 lore raw

PATCH v4 1/31 CI: run "set -ex" early in ci/lib.sh

Change ci/lib.sh to run "set -ex" before it does anything else, before this we were omitting a few "export" commands in our trace output that happened before we enabled tracing.

---
 ci/lib.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
ci/lib.sh +6 -6
index cbc2f8f1caa..57141d38a85 100755
@@ -1,5 +1,11 @@
 # Library of functions shared by all CI scripts
 
+# Set 'exit on error' for all CI scripts to let the caller know that
+# something went wrong.
+# Set tracing executed commands, primarily setting environment variables
+# and installing dependencies.
+set -ex
+
 skip_branch_tip_with_tag () {
 	# Sometimes, a branch is pushed at the same time the tag that points
 	# at the same commit as the tip of the branch is pushed, and building
@@ -85,12 +91,6 @@ export TERM=${TERM:-dumb}
 # Clear MAKEFLAGS that may come from the outside world.
 export MAKEFLAGS=
 
-# Set 'exit on error' for all CI scripts to let the caller know that
-# something went wrong.
-# Set tracing executed commands, primarily setting environment variables
-# and installing dependencies.
-set -ex
-
 if test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI"
 then
 	CI_TYPE=azure-pipelines
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#157 lore raw

PATCH v4 2/31 CI: make "$jobname" explicit, remove fallback

In reply to Ævar Arnfjörð Bjarmason

Remove the $jobname fallback last adjusted in b011fabd6e0 (ci/lib.sh:
encapsulate Travis-specific things, 2019-01-27).

This makes all of our CI jobs have an explicit $jobname, and removes the special-case of the Windows jobs implicitly setting one (that they don't currently use). In subsequent commits we'll use this new $jobname explicitly.

This is a follow-up to my 25715419bf4 (CI: don't run "make test" twice in one job, 2021-11-23), since then all the jobs except the Windows jobs have had an explicit $jobname.

In the case of the Windows jobs we were using a jobname of windows-gcc for both, due to picking "gcc" as a default fallback for "$CC" in ci/lib.sh.

Since we'll need this to run ci/lib.sh it makes sense to move the check to the top of the file. Subsequent commits will add more such assertions to it.

---
 .github/workflows/main.yml |  8 ++++++++
 ci/lib.sh                  | 12 +++++++-----
 2 files changed, 15 insertions(+), 5 deletions(-)
.github/workflows/main.yml +8 -0
index c35200defb9..309b9141249 100644
@@ -79,6 +79,8 @@ jobs:
 
   windows-build:
     name: win build
+    env:
+      jobname: windows-build
     needs: ci-config
     if: needs.ci-config.outputs.enabled == 'yes'
     runs-on: windows-latest
@@ -100,6 +102,8 @@ jobs:
         path: artifacts
   windows-test:
     name: win test
+    env:
+      jobname: windows-test
     runs-on: windows-latest
     needs: [windows-build]
     strategy:
@@ -131,6 +135,8 @@ jobs:
         path: ${{env.FAILED_TEST_ARTIFACTS}}
   vs-build:
     name: win+VS build
+    env:
+      jobname: vs-build
     needs: ci-config
     if: needs.ci-config.outputs.enabled == 'yes'
     env:
@@ -183,6 +189,8 @@ jobs:
         path: artifacts
   vs-test:
     name: win+VS test
+    env:
+      jobname: vs-test
     runs-on: windows-latest
     needs: vs-build
     strategy:
ci/lib.sh +7 -5
index 57141d38a85..5fda166ad47 100755
@@ -6,6 +6,13 @@
 # and installing dependencies.
 set -ex
 
+# Starting assertions
+if test -z "$jobname"
+then
+	echo "must set a CI jobname in the environment" >&2
+	exit 1
+fi
+
 skip_branch_tip_with_tag () {
 	# Sometimes, a branch is pushed at the same time the tag that points
 	# at the same commit as the tip of the branch is pushed, and building
@@ -146,11 +153,6 @@ test -n "${DONT_SKIP_TAGS-}" ||
 skip_branch_tip_with_tag
 skip_good_tree
 
-if test -z "$jobname"
-then
-	jobname="$CI_OS_NAME-$CC"
-fi
-
 export DEVELOPER=1
 export DEFAULT_TEST_TARGET=prove
 export GIT_TEST_CLONE_2GB=true
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#158 lore raw

PATCH v4 3/31 CI: remove more dead Travis CI support

In reply to Ævar Arnfjörð Bjarmason

Remove code that I missed in 4a6e4b96026 (CI: remove Travis CI support, 2021-11-23). This code was only called from or used by the now-removed .travis.yml, or needed by the Travis CI environment.

For the symlinking in ci/run-build-and-tests.sh: Back when 3c93b829205 (travis-ci: build Git during the 'script' phase, 2018-01-08) and 4b060a4d973 (ci: use a junction on Windows instead of a symlink, 2019-01-27) adjusted this "ln" command, the Windows build would use ci/run-build-and-tests.sh.

As seen in 889cacb6897 (ci: configure GitHub Actions for CI/PR, 2020-04-11) the current windows build uses a different entry point under the GitHub CI, which doesn't use this .prove caching. Namely "ci/run-test-slice.sh".

We can be certain that it's never used in "ci/run-test-slice.sh" because to have a ".prove" file we'd need to use "--state=save", which we only do in the dead Azure codepath in ci/lib.sh. If it were used it would do the wrong thing, because the different test slices would each try to clobber the same "t/.prove" file.

If a subsequent run then used the "--state=failed,slow,save" it would defeat the purpose of "ci/run-test-slice.sh", since all slices would then run all tests. I.e. behavior of prove's "--state" options is to select tests to run from the provided "--state" file, in addition to those specified on the command-line.

For ci/run-docker{,-build}.sh: It was likewise last referenced in the .travis.yml removed in my 4a6e4b96026. The current "dockerized" run in ".github/workflows/main.yml" calls the same entry points as the main "regular" job.

For "ci/print-test-failures.sh" the creation of the tarball added in aea8879a6ac (travis-ci: include the trash directories of failed tests in the trace log, 2018-08-01) has been dead since my 4a6e4b96026, which removed the fall-through case from the "case/esac" statement.

---
 ci/lib.sh                     | 10 ++----
 ci/print-test-failures.sh     | 39 ---------------------
 ci/run-build-and-tests.sh     |  5 ---
 ci/run-docker-build.sh        | 66 -----------------------------------
 ci/run-docker.sh              | 47 -------------------------
 ci/run-test-slice.sh          |  5 ---
 ci/util/extract-trash-dirs.sh | 50 --------------------------
 7 files changed, 2 insertions(+), 220 deletions(-)
 delete mode 100755 ci/run-docker-build.sh
 delete mode 100755 ci/run-docker.sh
 delete mode 100755 ci/util/extract-trash-dirs.sh
ci/lib.sh +2 -8
index 5fda166ad47..510b20726ab 100755
@@ -106,9 +106,6 @@ then
 	CI_COMMIT="$BUILD_SOURCEVERSION"
 	CI_JOB_ID="$BUILD_BUILDID"
 	CI_JOB_NUMBER="$BUILD_BUILDNUMBER"
-	CI_OS_NAME="$(echo "$AGENT_OS" | tr A-Z a-z)"
-	test darwin != "$CI_OS_NAME" || CI_OS_NAME=osx
-	CI_REPO_SLUG="$(expr "$BUILD_REPOSITORY_URI" : '.*/\([^/]*/[^/]*\)$')"
 	CC="${CC:-gcc}"
 
 	# use a subdirectory of the cache dir (because the file share is shared
@@ -118,16 +115,13 @@ then
 	export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save"
 	export GIT_TEST_OPTS="--verbose-log -x --write-junit-xml"
 	MAKEFLAGS="$MAKEFLAGS --jobs=10"
-	test windows_nt != "$CI_OS_NAME" ||
+	test Windows_NT != "$AGENT_OS" ||
 	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
 elif test true = "$GITHUB_ACTIONS"
 then
 	CI_TYPE=github-actions
 	CI_BRANCH="$GITHUB_REF"
 	CI_COMMIT="$GITHUB_SHA"
-	CI_OS_NAME="$(echo "$RUNNER_OS" | tr A-Z a-z)"
-	test macos != "$CI_OS_NAME" || CI_OS_NAME=osx
-	CI_REPO_SLUG="$GITHUB_REPOSITORY"
 	CI_JOB_ID="$GITHUB_RUN_ID"
 	CC="${CC:-gcc}"
 	DONT_SKIP_TAGS=t
@@ -137,7 +131,7 @@ then
 	export GIT_PROVE_OPTS="--timer --jobs 10"
 	export GIT_TEST_OPTS="--verbose-log -x"
 	MAKEFLAGS="$MAKEFLAGS --jobs=10"
-	test windows != "$CI_OS_NAME" ||
+	test Windows != "$RUNNER_OS" ||
 	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
 else
 	echo "Could not identify CI type" >&2
ci/print-test-failures.sh +0 -39
index 57277eefcd0..740ebb7c529 100755
@@ -16,15 +16,6 @@ then
 	exit
 fi
 
-case "$jobname" in
-osx-clang|osx-gcc)
-	# base64 in OSX doesn't wrap its output at 76 columns by
-	# default, but prints a single, very long line.
-	base64_opts="-b 76"
-	;;
-esac
-
-combined_trash_size=0
 for TEST_EXIT in test-results/*.exit
 do
 	if [ "$(cat "$TEST_EXIT")" != "0" ]
@@ -42,47 +33,17 @@ do
 		azure-pipelines)
 			mkdir -p failed-test-artifacts
 			mv "$trash_dir" failed-test-artifacts
-			continue
 			;;
 		github-actions)
 			mkdir -p failed-test-artifacts
 			echo "FAILED_TEST_ARTIFACTS=t/failed-test-artifacts" >>$GITHUB_ENV
 			cp "${TEST_EXIT%.exit}.out" failed-test-artifacts/
 			tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir"
-			continue
 			;;
 		*)
 			echo "Unhandled CI type: $CI_TYPE" >&2
 			exit 1
 			;;
 		esac
-		trash_tgz_b64="trash.$test_name.base64"
-		if [ -d "$trash_dir" ]
-		then
-			tar czp "$trash_dir" |base64 $base64_opts >"$trash_tgz_b64"
-
-			trash_size=$(wc -c <"$trash_tgz_b64")
-			if [ $trash_size -gt 1048576 ]
-			then
-				# larger than 1MB
-				echo "$(tput setaf 1)Didn't include the trash directory of '$test_name' in the trace log, it's too big$(tput sgr0)"
-				continue
-			fi
-
-			new_combined_trash_size=$(($combined_trash_size + $trash_size))
-			if [ $new_combined_trash_size -gt 1048576 ]
-			then
-				echo "$(tput setaf 1)Didn't include the trash directory of '$test_name' in the trace log, there is plenty of trash in there already.$(tput sgr0)"
-				continue
-			fi
-			combined_trash_size=$new_combined_trash_size
-
-			# DO NOT modify these two 'echo'-ed strings below
-			# without updating 'ci/util/extract-trash-dirs.sh'
-			# as well.
-			echo "$(tput setaf 1)Start of trash directory of '$test_name':$(tput sgr0)"
-			cat "$trash_tgz_b64"
-			echo "$(tput setaf 1)End of trash directory of '$test_name'$(tput sgr0)"
-		fi
 	fi
 done
ci/run-build-and-tests.sh +0 -5
index 280dda7d285..0a1ec8c2bae 100755
@@ -5,11 +5,6 @@
 
 . ${0%/*}/lib.sh
 
-case "$CI_OS_NAME" in
-windows*) cmd //c mklink //j t\\.prove "$(cygpath -aw "$cache_dir/.prove")";;
-*) ln -s "$cache_dir/.prove" t/.prove;;
-esac
-
 export MAKE_TARGETS="all test"
 
 case "$jobname" in
ci/run-docker-build.sh deleted +0 -66

66 lines not on this page · raw message

ci/run-docker.sh deleted +0 -47

47 lines not on this page · raw message

ci/run-test-slice.sh +0 -5
index f8c2c3106a2..b9a682b4bcd 100755
@@ -5,11 +5,6 @@
 
 . ${0%/*}/lib.sh
 
-case "$CI_OS_NAME" in
-windows*) cmd //c mklink //j t\\.prove "$(cygpath -aw "$cache_dir/.prove")";;
-*) ln -s "$cache_dir/.prove" t/.prove;;
-esac
-
 make --quiet -C t T="$(cd t &&
 	./helper/test-tool path-utils slice-tests "$1" "$2" t[0-9]*.sh |
 	tr '\n' ' ')"
ci/util/extract-trash-dirs.sh deleted +0 -50

50 lines not on this page · raw message

Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#159 lore raw

PATCH v4 4/31 CI: remove dead "tree skipping" code

In reply to Ævar Arnfjörð Bjarmason

Remove the code related to skipping CI runs if the tree was successfully tested before. With the GitHub CI this is done with the "ci-config" step, see e76eec35540 (ci: allow per-branch config for GitHub Actions, 2020-05-07) and 7d78d5fc1a9 (ci: skip GitHub workflow runs for already-tested commits/trees, 2020-10-08).

This code hasn't been used since 4a6e4b96026 (CI: remove Travis CI support, 2021-11-23), and before that for the retired Azure pipeline support removed in 6081d3898fe (ci: retire the Azure Pipelines definition, 2020-04-11).

This change is needed because this and subsequent commits are turning "ci/lib.sh" into a dumber library that'll only be tasked with setting variables for CI jobs, or for specific steps within those jobs. Now we don't need to worry about it potentially skipping the run on its own.

This change also removes a subtle potential logic error introduced in 0e7696c64db (ci: disallow directional formatting, 2021-11-04). The "ci/check-directional-formatting.bash" script would have been made to run after the "save_good_tree" in invoked in "ci/run-static-analysis.sh". So if e.g. the "make coccicheck" failed we'd still mark the tree as good.

I.e. the addition of "ci/check-directional-formatting.bash" didn't take into account that the various users of the "save_good_tree" function in ci/*.sh made a hard assumption that they're the only ci/*.sh script that's being run (and they use "set -e").

This is entirely academic since we weren't actually running this code, but is something to be careful of if anyone ever needs to resurrect parts of this.

Finally, the cache_dir="$HOME/none" added in a3f2eec862b (ci/lib:
allow running in GitHub Actions, 2020-04-08) for the GitHub CI can be removed. It appears to have been added to appease the subsequent 'mkdir -p "$cache_dir"', which wasn't needed in GitHub CI.

1. https://lore.kernel.org/git/[email protected]/

---
 ci/lib.sh                 | 91 ---------------------------------------
 ci/run-build-and-tests.sh |  2 -
 ci/run-static-analysis.sh |  2 -
 ci/test-documentation.sh  |  2 -
 4 files changed, 97 deletions(-)
ci/lib.sh +0 -91
index 510b20726ab..d12eaf338f9 100755
@@ -13,75 +13,6 @@ then
 	exit 1
 fi
 
-skip_branch_tip_with_tag () {
-	# Sometimes, a branch is pushed at the same time the tag that points
-	# at the same commit as the tip of the branch is pushed, and building
-	# both at the same time is a waste.
-	#
-	# When the build is triggered by a push to a tag, $CI_BRANCH will
-	# have that tagname, e.g. v2.14.0.  Let's see if $CI_BRANCH is
-	# exactly at a tag, and if so, if it is different from $CI_BRANCH.
-	# That way, we can tell if we are building the tip of a branch that
-	# is tagged and we can skip the build because we won't be skipping a
-	# build of a tag.
-
-	if TAG=$(git describe --exact-match "$CI_BRANCH" 2>/dev/null) &&
-		test "$TAG" != "$CI_BRANCH"
-	then
-		echo "$(tput setaf 2)Tip of $CI_BRANCH is exactly at $TAG$(tput sgr0)"
-		exit 0
-	fi
-}
-
-# Save some info about the current commit's tree, so we can skip the build
-# job if we encounter the same tree again and can provide a useful info
-# message.
-save_good_tree () {
-	echo "$(git rev-parse $CI_COMMIT^{tree}) $CI_COMMIT $CI_JOB_NUMBER $CI_JOB_ID" >>"$good_trees_file"
-	# limit the file size
-	tail -1000 "$good_trees_file" >"$good_trees_file".tmp
-	mv "$good_trees_file".tmp "$good_trees_file"
-}
-
-# Skip the build job if the same tree has already been built and tested
-# successfully before (e.g. because the branch got rebased, changing only
-# the commit messages).
-skip_good_tree () {
-	if test true = "$GITHUB_ACTIONS"
-	then
-		return
-	fi
-
-	if ! good_tree_info="$(grep "^$(git rev-parse $CI_COMMIT^{tree}) " "$good_trees_file")"
-	then
-		# Haven't seen this tree yet, or no cached good trees file yet.
-		# Continue the build job.
-		return
-	fi
-
-	echo "$good_tree_info" | {
-		read tree prev_good_commit prev_good_job_number prev_good_job_id
-
-		if test "$CI_JOB_ID" = "$prev_good_job_id"
-		then
-			cat <<-EOF
-			$(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0)
-			This commit has already been built and tested successfully by this build job.
-			To force a re-build delete the branch's cache and then hit 'Restart job'.
-			EOF
-		else
-			cat <<-EOF
-			$(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0)
-			This commit's tree has already been built and tested successfully in build job $prev_good_job_number for commit $prev_good_commit.
-			The log of that build job is available at $SYSTEM_TASKDEFINITIONSURI$SYSTEM_TEAMPROJECT/_build/results?buildId=$prev_good_job_id
-			To force a re-build delete the branch's cache and then hit 'Restart job'.
-			EOF
-		fi
-	}
-
-	exit 0
-}
-
 check_unignored_build_artifacts ()
 {
 	! git ls-files --other --exclude-standard --error-unmatch \
@@ -102,16 +33,8 @@ if test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI"
 then
 	CI_TYPE=azure-pipelines
 	# We are running in Azure Pipelines
-	CI_BRANCH="$BUILD_SOURCEBRANCH"
-	CI_COMMIT="$BUILD_SOURCEVERSION"
-	CI_JOB_ID="$BUILD_BUILDID"
-	CI_JOB_NUMBER="$BUILD_BUILDNUMBER"
 	CC="${CC:-gcc}"
 
-	# use a subdirectory of the cache dir (because the file share is shared
-	# among *all* phases)
-	cache_dir="$HOME/test-cache/$SYSTEM_PHASENAME"
-
 	export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save"
 	export GIT_TEST_OPTS="--verbose-log -x --write-junit-xml"
 	MAKEFLAGS="$MAKEFLAGS --jobs=10"
@@ -120,13 +43,7 @@ then
 elif test true = "$GITHUB_ACTIONS"
 then
 	CI_TYPE=github-actions
-	CI_BRANCH="$GITHUB_REF"
-	CI_COMMIT="$GITHUB_SHA"
-	CI_JOB_ID="$GITHUB_RUN_ID"
 	CC="${CC:-gcc}"
-	DONT_SKIP_TAGS=t
-
-	cache_dir="$HOME/none"
 
 	export GIT_PROVE_OPTS="--timer --jobs 10"
 	export GIT_TEST_OPTS="--verbose-log -x"
@@ -139,14 +56,6 @@ else
 	exit 1
 fi
 
-good_trees_file="$cache_dir/good-trees"
-
-mkdir -p "$cache_dir"
-
-test -n "${DONT_SKIP_TAGS-}" ||
-skip_branch_tip_with_tag
-skip_good_tree
-
 export DEVELOPER=1
 export DEFAULT_TEST_TARGET=prove
 export GIT_TEST_CLONE_2GB=true
ci/run-build-and-tests.sh +0 -2
index 0a1ec8c2bae..e5a21985b44 100755
@@ -45,5 +45,3 @@ esac
 # start running tests.
 make $MAKE_TARGETS
 check_unignored_build_artifacts
-
-save_good_tree
ci/run-static-analysis.sh +0 -2
index 65bcebda41a..5a87b1cac96 100755
@@ -28,5 +28,3 @@ fi
 
 make hdr-check ||
 exit 1
-
-save_good_tree
ci/test-documentation.sh +0 -2
index de41888430a..9e0652c30dd 100755
@@ -41,5 +41,3 @@ grep '<meta name="generator" content="Asciidoctor ' Documentation/git.html
 
 rm -f stdout.log stderr.log stderr.raw
 check_unignored_build_artifacts
-
-save_good_tree
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#160 lore raw

PATCH v4 5/31 CI: remove unused Azure ci/* code

In reply to Ævar Arnfjörð Bjarmason

Remove Azure-specific code that's been unused since 6081d3898fe (ci:
retire the Azure Pipelines definition, 2020-04-11). As noted in a larger removal of all of the Azure-supporting code in [1] (although that missed some of this) there is more of it in-tree, but let's focus on only the ci/* code for now.

This is needed because in subsequent commits this unused code would either need to be changed to accommodate changes in ci/*, or be removed.

As we'll see in those subsequent commits the end-state we're heading towards will actually make it easier to add new CI types in the future, even though the only one we're left with now is the GitHub CI. I.e. the "ci/*" framework will be made to do less, not more. We'll be offloading more of what it does to our generic build and test system.

While I'm at it (since the line needs to be touched anyway) change an odd 'if test true == "$GITHUB_ACTIONS"' to the more usual style used in other places of 'if test "$GITHUB_ACTIONS" = "true"'.

1. https://lore.kernel.org/git/[email protected]/

---
 ci/install-dependencies.sh |  3 ---
 ci/lib.sh                  | 13 +------------
 ci/mount-fileshare.sh      | 25 -------------------------
 ci/print-test-failures.sh  |  4 ----
 4 files changed, 1 insertion(+), 44 deletions(-)
 delete mode 100755 ci/mount-fileshare.sh
ci/install-dependencies.sh +0 -3
index dbcebad2fb2..e7ea8799411 100755
@@ -34,8 +34,6 @@ macos-latest)
 	export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1
 	# Uncomment this if you want to run perf tests:
 	# brew install gnu-time
-	test -z "$BREW_INSTALL_PACKAGES" ||
-	brew install $BREW_INSTALL_PACKAGES
 	brew link --force gettext
 	brew install --cask --no-quarantine perforce || {
 		# Update the definitions and try again
@@ -69,7 +67,6 @@ Documentation)
 	sudo apt-get -q update
 	sudo apt-get -q -y install asciidoc xmlto docbook-xsl-ns make
 
-	test -n "$ALREADY_HAVE_ASCIIDOCTOR" ||
 	sudo gem install --version 1.5.8 asciidoctor
 	;;
 linux-gcc-default)
ci/lib.sh +1 -12
index d12eaf338f9..c4b829c1eed 100755
@@ -29,18 +29,7 @@ export TERM=${TERM:-dumb}
 # Clear MAKEFLAGS that may come from the outside world.
 export MAKEFLAGS=
 
-if test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI"
-then
-	CI_TYPE=azure-pipelines
-	# We are running in Azure Pipelines
-	CC="${CC:-gcc}"
-
-	export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save"
-	export GIT_TEST_OPTS="--verbose-log -x --write-junit-xml"
-	MAKEFLAGS="$MAKEFLAGS --jobs=10"
-	test Windows_NT != "$AGENT_OS" ||
-	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
-elif test true = "$GITHUB_ACTIONS"
+if test "$GITHUB_ACTIONS" = "true"
 then
 	CI_TYPE=github-actions
 	CC="${CC:-gcc}"
ci/mount-fileshare.sh deleted +0 -25
deleted file mode 100755
index 26b58a80960..00000000000
@@ -1,25 +0,0 @@
-#!/bin/sh
-
-die () {
-	echo "$*" >&2
-	exit 1
-}
-
-test $# = 4 ||
-die "Usage: $0 <share> <username> <password> <mountpoint>"
-
-mkdir -p "$4" || die "Could not create $4"
-
-case "$(uname -s)" in
-Linux)
-	sudo mount -t cifs -o vers=3.0,username="$2",password="$3",dir_mode=0777,file_mode=0777,serverino "$1" "$4"
-	;;
-Darwin)
-	pass="$(echo "$3" | sed -e 's/\//%2F/g' -e 's/+/%2B/g')" &&
-	mount -t smbfs,soft "smb://$2:$pass@${1#//}" "$4"
-	;;
-*)
-	die "No support for $(uname -s)"
-	;;
-esac ||
-die "Could not mount $4"
ci/print-test-failures.sh +0 -4
index 740ebb7c529..47e2ea1ad10 100755
@@ -30,10 +30,6 @@ do
 		test_name="${test_name##*/}"
 		trash_dir="trash directory.$test_name"
 		case "$CI_TYPE" in
-		azure-pipelines)
-			mkdir -p failed-test-artifacts
-			mv "$trash_dir" failed-test-artifacts
-			;;
 		github-actions)
 			mkdir -p failed-test-artifacts
 			echo "FAILED_TEST_ARTIFACTS=t/failed-test-artifacts" >>$GITHUB_ENV
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#161 lore raw

PATCH v4 6/31 CI/lib.sh: stop adding leading whitespace to $MAKEFLAGS

In reply to Ævar Arnfjörð Bjarmason

In a8c51f77d16 (ci: clear and mark MAKEFLAGS exported just once, 2019-02-07) the assignment of MAKEFLAGS was done up-front to future-proof this code for the bug fixed in 406f93ae488 (ci: make sure we build Git parallel, 2019-02-07).

But since 4a6e4b96026 (CI: remove Travis CI support, 2021-11-23) the --jobs=10 has been an invariant here, so let's assign --jobs=$NPROC to the MAKEFLAGS right away, this gets rid of the leading whitespace from the $MAKEFLAGS variable, as it would previously be empty on the first assignment.

---
 ci/lib.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)
ci/lib.sh +5 -3
index c4b829c1eed..21c009757f4 100755
@@ -26,17 +26,19 @@ check_unignored_build_artifacts ()
 # GitHub Action doesn't set TERM, which is required by tput
 export TERM=${TERM:-dumb}
 
+# How many jobs to run in parallel?
+NPROC=10
+
 # Clear MAKEFLAGS that may come from the outside world.
-export MAKEFLAGS=
+export MAKEFLAGS=--jobs=$NPROC
 
 if test "$GITHUB_ACTIONS" = "true"
 then
 	CI_TYPE=github-actions
 	CC="${CC:-gcc}"
 
-	export GIT_PROVE_OPTS="--timer --jobs 10"
+	export GIT_PROVE_OPTS="--timer --jobs $NPROC"
 	export GIT_TEST_OPTS="--verbose-log -x"
-	MAKEFLAGS="$MAKEFLAGS --jobs=10"
 	test Windows != "$RUNNER_OS" ||
 	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
 else
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#162 lore raw

PATCH v4 7/31 CI: don't have "git grep" invoke a pager in tree content check

In reply to Ævar Arnfjörð Bjarmason

Fix the omission of not adding "-P" (a.k.a. "--no-pager") to the "git" invocation that spaws "git grep", added in 0e7696c64db (ci: disallow directional formatting, 2021-11-04).

This didn't matter for the GitHub CI, but does when running the check locally. A subsequent commit will start invoking it via a new check from the top-level Makefile, and without this we'd visibly spawn a pager when doing so.

---
 ci/check-directional-formatting.bash | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
ci/check-directional-formatting.bash +1 -1
index e6211b141aa..de8e6e486f0 100755
@@ -23,5 +23,5 @@
 # U+2066..U+2069: LRI, RLI, FSI and PDI
 regex='(\u202a|\u202b|\u202c|\u202d|\u202e|\u2066|\u2067|\u2068|\u2069)'
 
-! LC_CTYPE=C git grep -El "$(LC_CTYPE=C.UTF-8 printf "$regex")" \
+! LC_CTYPE=C git -P grep -El "$(LC_CTYPE=C.UTF-8 printf "$regex")" \
 	-- ':(exclude,attr:binary)' ':(exclude)*.po'
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#163 lore raw

PATCH v4 8/31 CI: have "static-analysis" run a "make ci-static-analysis" target

In reply to Ævar Arnfjörð Bjarmason

Make it easier to run the CI logic locally by creating "ci-static-analysis" target.

Now the "check-coccicheck" logic added in 0860a7641ba (travis-ci: fail if Coccinelle static analysis found something to transform, 2018-07-23) can be easily tested outside of CI, and likewise the sanity check added in 0e7696c64db (ci: disallow directional formatting, 2021-11-04).

These targets could be improved, the "hdr-check" target will re-do all of its work every time it's run, and could with a change similar to my c234e8a0ecf (Makefile: make the "sparse" target non-.PHONY, 2021-09-23). The same goes for the new "ci-check-directional-formatting" target.

But without those improvements being able to easily run a 1=1 equivalent of the checks we do in CI during a local build is already an improvement. Subsequent commits will expand on this ability.

This change will also make the CI check faster, since we can take advantage of parallelism. The "make coccicheck" takes a very long time, and at the tail end we might only have one job pegging 100% of one CPU core for a long time. Now any extra cores will be free to run the rest of the targets under "ci-static-analysis".

---
 .github/workflows/main.yml |  3 +--
 Makefile                   | 28 ++++++++++++++++++++++++++++
 ci/run-static-analysis.sh  | 30 ------------------------------
 shared.mak                 |  1 +
 4 files changed, 30 insertions(+), 32 deletions(-)
 delete mode 100755 ci/run-static-analysis.sh
.github/workflows/main.yml +1 -2
index 309b9141249..ad752010102 100644
@@ -317,8 +317,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/run-static-analysis.sh
-    - run: ci/check-directional-formatting.bash
+    - run: make ci-static-analysis
   sparse:
     needs: ci-config
     if: needs.ci-config.outputs.enabled == 'yes'
Makefile +28 -0
index f8bccfab5e9..8717ea7e745 100644
@@ -3031,6 +3031,20 @@ coccicheck: $(addsuffix .patch,$(filter-out %.pending.cocci,$(wildcard contrib/c
 # See contrib/coccinelle/README
 coccicheck-pending: $(addsuffix .patch,$(wildcard contrib/coccinelle/*.pending.cocci))
 
+.PHONY: check-coccicheck
+check-coccicheck: coccicheck
+	$(QUIET_CHECK)for cocci_patch in contrib/coccinelle/*.patch; do \
+		if test -s "$$cocci_patch"; then \
+			echo "Coccinelle suggests the following changes in '$$cocci_patch':"; \
+			cat "$$cocci_patch"; \
+			fail=UnfortunatelyYes; \
+		fi \
+	done; \
+	if test -n "$$fail"; then \
+		echo "error: Coccinelle suggested some changes"; \
+		exit 1; \
+	fi
+
 .PHONY: coccicheck coccicheck-pending
 
 ### Installation rules
@@ -3466,3 +3480,17 @@ $(FUZZ_PROGRAMS): all
 		$(XDIFF_OBJS) $(EXTLIBS) git.o [email protected] $(LIB_FUZZING_ENGINE) -o $@
 
 fuzz-all: $(FUZZ_PROGRAMS)
+
+### CI "check" targets
+#
+# These targets are run from the CI, see .github/workflows/main.yml,
+# but can also be run manually to run the same assertions locally.
+
+.PHONY: ci-check-directional-formatting
+ci-check-directional-formatting:
+	$(QUIET_CHECK)ci/check-directional-formatting.bash
+
+.PHONY: ci-static-analysis
+ci-static-analysis: ci-check-directional-formatting
+ci-static-analysis: check-coccicheck
+ci-static-analysis: hdr-check
ci/run-static-analysis.sh deleted +0 -30
deleted file mode 100755
index 5a87b1cac96..00000000000
@@ -1,30 +0,0 @@
-#!/bin/sh
-#
-# Perform various static code analysis checks
-#
-
-. ${0%/*}/lib.sh
-
-make coccicheck
-
-set +x
-
-fail=
-for cocci_patch in contrib/coccinelle/*.patch
-do
-	if test -s "$cocci_patch"
-	then
-		echo "$(tput setaf 1)Coccinelle suggests the following changes in '$cocci_patch':$(tput sgr0)"
-		cat "$cocci_patch"
-		fail=UnfortunatelyYes
-	fi
-done
-
-if test -n "$fail"
-then
-	echo "$(tput setaf 1)error: Coccinelle suggested some changes$(tput sgr0)"
-	exit 1
-fi
-
-make hdr-check ||
-exit 1
shared.mak +1 -0
index 50d4596f0d6..83e0aa16e7d 100644
@@ -68,6 +68,7 @@ ifndef V
 	QUIET_HDR      = @echo '   ' HDR $(<:hcc=h);
 	QUIET_RC       = @echo '   ' RC $@;
 	QUIET_SPATCH   = @echo '   ' SPATCH $<;
+	QUIET_CHECK    = @echo '   ' CHECK $@;
 
 ## Used in "Documentation/Makefile"
 	QUIET_ASCIIDOC	= @echo '   ' ASCIIDOC $@;
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#164 lore raw

PATCH v4 9/31 CI: have "static-analysis" run "check-builtins", not "documentation"

In reply to Ævar Arnfjörð Bjarmason

Move the "make check-builtins" check from the "documentation" job to the "static-analysis" job.

The "check-builtins" target added in c74390e4a1d (cherry is built-in, do not ship git-cherry.sh, 2006-11-05) is unrelated to the documentation, so it's odd that b98712b9aa9 (travis-ci: build documentation, 2016-05-04) added it to the "documentation" job.

Let's just move it to the "ci-static-analysis" target, and while we're at it improve the output with $(QUIET_CHECK).

---
 Makefile                 | 3 ++-
 ci/test-documentation.sh | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)
Makefile +2 -1
index 8717ea7e745..2e1ae6c6067 100644
@@ -3402,7 +3402,7 @@ check-docs::
 ### Make sure built-ins do not have dups and listed in git.c
 #
 check-builtins::
-	./check-builtins.sh
+	$(QUIET_CHECK)./check-builtins.sh
 
 ### Test suite coverage testing
 #
@@ -3492,5 +3492,6 @@ ci-check-directional-formatting:
 
 .PHONY: ci-static-analysis
 ci-static-analysis: ci-check-directional-formatting
+ci-static-analysis: check-builtins
 ci-static-analysis: check-coccicheck
 ci-static-analysis: hdr-check
ci/test-documentation.sh +0 -1
index 9e0652c30dd..41e2b126311 100755
@@ -14,7 +14,6 @@ filter_log () {
 	    "$1"
 }
 
-make check-builtins
 make check-docs
 
 # Build docs with AsciiDoc
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#165 lore raw

PATCH v4 10/31 CI: move p4 and git-lfs variables to ci/install-dependencies.sh

In reply to Ævar Arnfjörð Bjarmason

Move the declaration of variables that are only used by the "ubuntu-latest" block in "ci/install-dependencies.sh" there from "ci/lib.sh".

This makes the code easier to follow, and changes "ci/lib.sh" to a library that only exports CI variables for general use, we didn't need to export these $P4_PATH and $GIT_LFS_PATH variables.

---
 ci/install-dependencies.sh | 17 +++++++++++++++--
 ci/lib.sh                  | 12 ------------
 2 files changed, 15 insertions(+), 14 deletions(-)
ci/install-dependencies.sh +15 -2
index e7ea8799411..34b4400f38b 100755
@@ -5,14 +5,27 @@
 
 . ${0%/*}/lib.sh
 
-P4WHENCE=http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION
-LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION
 UBUNTU_COMMON_PKGS="make libssl-dev libcurl4-openssl-dev libexpat-dev
  tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl
  libemail-valid-perl libio-socket-ssl-perl libnet-smtp-ssl-perl"
 
 case "$runs_on_pool" in
 ubuntu-latest)
+	# The Linux build installs the defined dependency versions below.
+	# The OS X build installs much more recent versions, whichever
+	# were recorded in the Homebrew database upon creating the OS X
+	# image.
+	# Keep that in mind when you encounter a broken OS X build!
+	LINUX_P4_VERSION="16.2"
+	LINUX_GIT_LFS_VERSION="1.5.2"
+
+	P4_PATH="$HOME/custom/p4"
+	GIT_LFS_PATH="$HOME/custom/git-lfs"
+	export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH"
+
+	P4WHENCE=http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION
+	LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION
+
 	sudo apt-get -q update
 	sudo apt-get -q -y install language-pack-is libsvn-perl apache2 \
 		$UBUNTU_COMMON_PKGS $CC_PACKAGE
ci/lib.sh +0 -12
index 21c009757f4..5736a4d2e04 100755
@@ -67,18 +67,6 @@ ubuntu-latest)
 	fi
 
 	export GIT_TEST_HTTPD=true
-
-	# The Linux build installs the defined dependency versions below.
-	# The OS X build installs much more recent versions, whichever
-	# were recorded in the Homebrew database upon creating the OS X
-	# image.
-	# Keep that in mind when you encounter a broken OS X build!
-	export LINUX_P4_VERSION="16.2"
-	export LINUX_GIT_LFS_VERSION="1.5.2"
-
-	P4_PATH="$HOME/custom/p4"
-	GIT_LFS_PATH="$HOME/custom/git-lfs"
-	export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH"
 	;;
 macos-latest)
 	if [ "$jobname" = osx-gcc ]
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#166 lore raw

PATCH v4 11/31 CI: consistently use "export" in ci/lib.sh

In reply to Ævar Arnfjörð Bjarmason

Change the "ci/lib.sh" script co consistently use "export", for e.g. MAKEFLAGS we were exporting it, and then assigning to it, let's do it the other way around.

Right now this doesn't matter, since we in
e.g. "ci/install-dependencies.sh" source this file, and don't use something like "env(1)" to retrieve these variables. But in a subsequent commit we'll "export" these variables through a wrapper (to additionally write them to a GitHub CI-specific $GITHUB_ENV file). This change makes that subsequent change easier to read, as it won't need to do any control flow refactoring.

---
 ci/lib.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)
ci/lib.sh +4 -3
index 5736a4d2e04..5783eed1b20 100755
@@ -30,7 +30,7 @@ export TERM=${TERM:-dumb}
 NPROC=10
 
 # Clear MAKEFLAGS that may come from the outside world.
-export MAKEFLAGS=--jobs=$NPROC
+MAKEFLAGS=--jobs=$NPROC
 
 if test "$GITHUB_ACTIONS" = "true"
 then
@@ -38,9 +38,10 @@ then
 	CC="${CC:-gcc}"
 
 	export GIT_PROVE_OPTS="--timer --jobs $NPROC"
-	export GIT_TEST_OPTS="--verbose-log -x"
+	GIT_TEST_OPTS="--verbose-log -x"
 	test Windows != "$RUNNER_OS" ||
 	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
+	export GIT_TEST_OPTS
 else
 	echo "Could not identify CI type" >&2
 	env >&2
@@ -94,4 +95,4 @@ linux-leaks)
 	;;
 esac
 
-MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}"
+export MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}"
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#167 lore raw

PATCH v4 12/31 CI: export variables via a wrapper

In reply to Ævar Arnfjörð Bjarmason

Add and use a "setenv" wrapper to export the variables set in ci/lib.sh.

For now this merely adds debugging information about what variables are set, but in a subsequent commit we'll expand this to set variables in $GITHUB_ENV instead of the environment, and thus carry variable settings between steps in a job.

We'll also eventually be interested in what type of variable we're setting, and when it needs to be used, because we'll prominently expose what variables we set in what CI "step", so having only the ones we actually use will improve readability.

To that end call the "setenv" function with optional "--build", "--test" and "--all" parameters to indicate whether they're needed for building, testing or both.

We currently ignore the "varname" parameter, but will make use of it in a subsequent commit. Adding it here reduces the needless churn, as we won't need to change all these lines again for that subsequent change.

---
 ci/lib.sh                 | 51 ++++++++++++++++++++++++++++++---------
 ci/run-build-and-tests.sh | 34 +++++++++++++-------------
 2 files changed, 57 insertions(+), 28 deletions(-)
ci/lib.sh +40 -11
index 5783eed1b20..3727e6ec2f9 100755
@@ -13,6 +13,35 @@ then
 	exit 1
 fi
 
+# Helper functions
+setenv () {
+	while test $# != 0
+	do
+		case "$1" in
+		--build)
+			;;
+		--test)
+			;;
+		--all)
+			;;
+		-*)
+			echo "BUG: bad setenv() option '$1'" >&2
+			exit 1
+			;;
+		*)
+			break
+			;;
+		esac
+		shift
+	done
+
+	key=$1
+	val=$2
+	shift 2
+
+	eval "export $key=\"$val\""
+}
+
 check_unignored_build_artifacts ()
 {
 	! git ls-files --other --exclude-standard --error-unmatch \
@@ -24,7 +53,7 @@ check_unignored_build_artifacts ()
 }
 
 # GitHub Action doesn't set TERM, which is required by tput
-export TERM=${TERM:-dumb}
+setenv TERM ${TERM:-dumb}
 
 # How many jobs to run in parallel?
 NPROC=10
@@ -37,21 +66,21 @@ then
 	CI_TYPE=github-actions
 	CC="${CC:-gcc}"
 
-	export GIT_PROVE_OPTS="--timer --jobs $NPROC"
+	setenv --test GIT_PROVE_OPTS "--timer --jobs $NPROC"
 	GIT_TEST_OPTS="--verbose-log -x"
 	test Windows != "$RUNNER_OS" ||
 	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
-	export GIT_TEST_OPTS
+	setenv --test GIT_TEST_OPTS "$GIT_TEST_OPTS"
 else
 	echo "Could not identify CI type" >&2
 	env >&2
 	exit 1
 fi
 
-export DEVELOPER=1
-export DEFAULT_TEST_TARGET=prove
-export GIT_TEST_CLONE_2GB=true
-export SKIP_DASHED_BUILT_INS=YesPlease
+setenv --build DEVELOPER 1
+setenv --test DEFAULT_TEST_TARGET prove
+setenv --test GIT_TEST_CLONE_2GB true
+setenv --build SKIP_DASHED_BUILT_INS YesPlease
 
 case "$runs_on_pool" in
 ubuntu-latest)
@@ -67,7 +96,7 @@ ubuntu-latest)
 		MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python2"
 	fi
 
-	export GIT_TEST_HTTPD=true
+	setenv --test GIT_TEST_HTTPD true
 	;;
 macos-latest)
 	if [ "$jobname" = osx-gcc ]
@@ -90,9 +119,9 @@ linux-musl)
 	MAKEFLAGS="$MAKEFLAGS GIT_TEST_UTF8_LOCALE=C.UTF-8"
 	;;
 linux-leaks)
-	export SANITIZE=leak
-	export GIT_TEST_PASSING_SANITIZE_LEAK=true
+	setenv --build SANITIZE leak
+	setenv --test GIT_TEST_PASSING_SANITIZE_LEAK true
 	;;
 esac
 
-export MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}"
+setenv --all MAKEFLAGS "$MAKEFLAGS CC=${CC:-cc}"
ci/run-build-and-tests.sh +17 -17
index e5a21985b44..35d45a9373e 100755
@@ -9,33 +9,33 @@ export MAKE_TARGETS="all test"
 
 case "$jobname" in
 linux-gcc)
-	export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
+	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME main
 	;;
 linux-TEST-vars)
-	export GIT_TEST_SPLIT_INDEX=yes
-	export GIT_TEST_MERGE_ALGORITHM=recursive
-	export GIT_TEST_FULL_IN_PACK_ARRAY=true
-	export GIT_TEST_OE_SIZE=10
-	export GIT_TEST_OE_DELTA_SIZE=5
-	export GIT_TEST_COMMIT_GRAPH=1
-	export GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS=1
-	export GIT_TEST_MULTI_PACK_INDEX=1
-	export GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=1
-	export GIT_TEST_ADD_I_USE_BUILTIN=1
-	export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master
-	export GIT_TEST_WRITE_REV_INDEX=1
-	export GIT_TEST_CHECKOUT_WORKERS=2
+	setenv --test GIT_TEST_SPLIT_INDEX yes
+	setenv --test GIT_TEST_MERGE_ALGORITHM recursive
+	setenv --test GIT_TEST_FULL_IN_PACK_ARRAY true
+	setenv --test GIT_TEST_OE_SIZE 10
+	setenv --test GIT_TEST_OE_DELTA_SIZE 5
+	setenv --test GIT_TEST_COMMIT_GRAPH 1
+	setenv --test GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS 1
+	setenv --test GIT_TEST_MULTI_PACK_INDEX 1
+	setenv --test GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP 1
+	setenv --test GIT_TEST_ADD_I_USE_BUILTIN 1
+	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME master
+	setenv --test GIT_TEST_WRITE_REV_INDEX 1
+	setenv --test GIT_TEST_CHECKOUT_WORKERS 2
 	;;
 linux-clang)
-	export GIT_TEST_DEFAULT_HASH=sha1
+	setenv --test GIT_TEST_DEFAULT_HASH sha1
 	;;
 linux-sha256)
-	export GIT_TEST_DEFAULT_HASH=sha256
+	setenv --test GIT_TEST_DEFAULT_HASH sha256
 	;;
 pedantic)
 	# Don't run the tests; we only care about whether Git can be
 	# built.
-	export DEVOPTS=pedantic
+	setenv --build DEVOPTS pedantic
 	export MAKE_TARGETS=all
 	;;
 esac
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#168 lore raw

PATCH v4 13/31 CI: remove "run-build-and-tests.sh", run "make [test]" directly

In reply to Ævar Arnfjörð Bjarmason

Remove the already thin "ci/run-build-and-tests.sh" wrapper and instead make the CI run "make" or "make test" directly. By doing this we'll be able to easily see at a glance whether our failure was in the compilation or testing, whether that's via human eyes or improve machine readability.

We also need to run our new "ci/check-unignored-build-artifacts.sh" on success() in the CI now, just like we already had a step conditional on failure() running ci/print-test-failures.sh.

The reason we used a "ci/run-build-and-tests.sh" wrapper in the first place had to do with Travis CI-specific constraints that no longer apply to us, as the Travis CI support has been removed.

Instead we can configure the CI in an earlier step by running "ci/lib.sh", which under GitHub CI will write the environment variables we need to the "$GITHUB_ENV" file.

We'll then have access to them in subsequent steps, and crucially those variables will be prominently visible at the start of each step via an expandable drop-down in the UI.drop-do.

I.e. this changes the CI run from a top-down flow like (pseudocode):

- job:
- step1:
- use ci/lib.sh to set env vars
- run a script like ci/run-build-and-tests.sh
- step2:
- if: failure()
- use ci/lib.sh to set env vars
- run ci/print-test-failures.sh

To:

- job:
- step1:
- set variables in $GITHUB_ENV using ci/lib.sh
- step2:
- make
- step3:
- make test
- step4:
- if: failure()
- run ci/print-test-failures.sh
- step5:
- if: success()
- run ci/check-unignored-build-artifacts.sh

There is a proposal[2] to get some of the benefits of this approach by not re-arranging our variable setup in this way, but to instead use the GitHub CI grouping syntax to focus on the relevant parts of "make" or "make test" when we have failures.

Doing it this way makes for better looking GitHub CI UI, and lays much better ground work for our CI going forward. Because:

* The CI logic will be more portable to a future CI system, since a common feature of them is to run various commands in sequence, but a future system won't necessarily support the GitHub-specifics syntax of "grouping" output within a "step".

Even if those systems don't support a "$GITHUB_ENV" emulating will be much easier than to deal with some CI-specific grouping syntax.

* At the start of every step the GitHub CI presents an expandable list of environment variables from "$GITHUB_ENV". We'll now see exactly what variables affected that step (although we currently overshoot that a bit, and always define all variables).

* CI failures will be easier to reproduce locally, as this makes the relevant ci/* scripts something that sets up our environment, but leaves "make" and "make test" working as they do locally.

To reproduce a run the user only needs to set the variables discussed in the drop-down above, either manually or by running "ci/lib.sh".

* The output will be less verbose. The "ci/lib.sh" script uses "set -x", and before this e.g. "ci/static-analysis.sh" would start with 40 lines of trace output, culminating in using "export" to export the relevant environment variables.

Now that verbosity is in the earlier "ci/lib.sh" step, and not in any subsequent one. The "make" targets then start out with the relevant non-trace output right away.

* If we do want to use the grouping syntax within a "step" it'll now be easier to do so. It doesn't support nesting, so we'd have to make a choice between using it for e.g. "make" v.s. "make test", or individual test failures. See "sadly" in [3].

1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/
3. https://lore.kernel.org/git/9333ba781b8240f704e739b00d274f8c3d887e39.1643050574.git.gitgitgadget@gmail.com/

---
 .github/workflows/main.yml            | 14 ++++++--
 ci/check-unignored-build-artifacts.sh |  8 +++++
 ci/install-dependencies.sh            |  4 +++
 ci/lib.sh                             | 36 +++++++++++++++++++-
 ci/make-test-artifacts.sh             |  2 --
 ci/run-build-and-tests.sh             | 47 ---------------------------
 ci/run-test-slice.sh                  |  2 --
 ci/test-documentation.sh              |  1 -
 8 files changed, 58 insertions(+), 56 deletions(-)
 create mode 100755 ci/check-unignored-build-artifacts.sh
 delete mode 100755 ci/run-build-and-tests.sh
.github/workflows/main.yml +11 -3
index ad752010102..c83f6a21ca3 100644
@@ -268,7 +268,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/run-build-and-tests.sh
+    - run: ci/lib.sh
+    - run: make
+    - run: make test
+      if: success()
     - run: ci/print-test-failures.sh
       if: failure()
     - name: Upload failed tests' directories
@@ -292,6 +295,7 @@ jobs:
           image: daald/ubuntu32:xenial
         - jobname: pedantic
           image: fedora
+          skip-tests: no
     env:
       jobname: ${{matrix.vector.jobname}}
     runs-on: ubuntu-latest
@@ -299,9 +303,12 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - run: ci/install-docker-dependencies.sh
-    - run: ci/run-build-and-tests.sh
+    - run: ci/lib.sh
+    - run: make
+    - run: make test
+      if: success() && matrix.vector.skip-tests != 'no'
     - run: ci/print-test-failures.sh
-      if: failure()
+      if: failure() && matrix.vector.skip-tests != 'no'
     - name: Upload failed tests' directories
       if: failure() && env.FAILED_TEST_ARTIFACTS != ''
       uses: actions/upload-artifact@v1
@@ -317,6 +324,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
+    - run: ci/lib.sh
     - run: make ci-static-analysis
   sparse:
     needs: ci-config
ci/check-unignored-build-artifacts.sh new file +8 -0
new file mode 100755
index 00000000000..56d04b0db9a
@@ -0,0 +1,8 @@
+#!/bin/sh
+#
+# Check whether the build created anything not in our .gitignore
+#
+
+. ${0%/*}/lib.sh
+
+check_unignored_build_artifacts
ci/install-dependencies.sh +4 -0
index 34b4400f38b..01006379fd0 100755
@@ -22,6 +22,10 @@ ubuntu-latest)
 	P4_PATH="$HOME/custom/p4"
 	GIT_LFS_PATH="$HOME/custom/git-lfs"
 	export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH"
+	if test -n "$GITHUB_PATH"
+	then
+		echo "$PATH" >>"$GITHUB_PATH"
+	fi
 
 	P4WHENCE=http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION
 	LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION
ci/lib.sh +35 -1
index 3727e6ec2f9..cc9a368be42 100755
@@ -1,3 +1,5 @@
+#!/bin/sh
+
 # Library of functions shared by all CI scripts
 
 # Set 'exit on error' for all CI scripts to let the caller know that
@@ -39,7 +41,10 @@ setenv () {
 	val=$2
 	shift 2
 
-	eval "export $key=\"$val\""
+	if test -n "$GITHUB_ENV"
+	then
+		echo "$key=$val" >>"$GITHUB_ENV"
+	fi
 }
 
 check_unignored_build_artifacts ()
@@ -109,6 +114,35 @@ macos-latest)
 esac
 
 case "$jobname" in
+linux-gcc)
+	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME main
+	;;
+linux-TEST-vars)
+	setenv --test GIT_TEST_SPLIT_INDEX yes
+	setenv --test GIT_TEST_MERGE_ALGORITHM recursive
+	setenv --test GIT_TEST_FULL_IN_PACK_ARRAY true
+	setenv --test GIT_TEST_OE_SIZE 10
+	setenv --test GIT_TEST_OE_DELTA_SIZE 5
+	setenv --test GIT_TEST_COMMIT_GRAPH 1
+	setenv --test GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS 1
+	setenv --test GIT_TEST_MULTI_PACK_INDEX 1
+	setenv --test GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP 1
+	setenv --test GIT_TEST_ADD_I_USE_BUILTIN 1
+	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME master
+	setenv --test GIT_TEST_WRITE_REV_INDEX 1
+	setenv --test GIT_TEST_CHECKOUT_WORKERS 2
+	;;
+linux-clang)
+	setenv --test GIT_TEST_DEFAULT_HASH sha1
+	;;
+linux-sha256)
+	setenv --test GIT_TEST_DEFAULT_HASH sha256
+	;;
+pedantic)
+	# Don't run the tests; we only care about whether Git can be
+	# built.
+	setenv --build DEVOPTS pedantic
+	;;
 linux32)
 	CC=gcc
 	;;
ci/make-test-artifacts.sh +0 -2
index 646967481f6..45298562982 100755
@@ -8,5 +8,3 @@ mkdir -p "$1" # in case ci/lib.sh decides to quit early
 . ${0%/*}/lib.sh
 
 make artifacts-tar ARTIFACTS_DIRECTORY="$1"
-
-check_unignored_build_artifacts
ci/run-build-and-tests.sh deleted +0 -47

47 lines not on this page · raw message

ci/run-test-slice.sh +0 -2
index b9a682b4bcd..70326961454 100755
@@ -8,5 +8,3 @@
 make --quiet -C t T="$(cd t &&
 	./helper/test-tool path-utils slice-tests "$1" "$2" t[0-9]*.sh |
 	tr '\n' ' ')"
-
-check_unignored_build_artifacts
ci/test-documentation.sh +0 -1
index 41e2b126311..6b5cce03bd7 100755
@@ -39,4 +39,3 @@ test -s Documentation/git.html
 grep '<meta name="generator" content="Asciidoctor ' Documentation/git.html
 
 rm -f stdout.log stderr.log stderr.raw
-check_unignored_build_artifacts
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#169 lore raw

PATCH v4 14/31 ci/lib.sh: use "test" instead of "["

In reply to Ævar Arnfjörð Bjarmason

Change the ci/lib.sh script to use "if test" rather than "if [". Until the preceding commit this script relied on the non-POSIX "export X=Y" syntax, but now that it's POSIX portable let's use our usual style in it, and not the bash-specific style that was used in it historically.

---
 ci/lib.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
ci/lib.sh +2 -2
index cc9a368be42..ffc0e524aaa 100755
@@ -94,7 +94,7 @@ ubuntu-latest)
 		break
 	fi
 
-	if [ "$jobname" = linux-gcc ]
+	if test "$jobname" = linux-gcc
 	then
 		MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python3"
 	else
@@ -104,7 +104,7 @@ ubuntu-latest)
 	setenv --test GIT_TEST_HTTPD true
 	;;
 macos-latest)
-	if [ "$jobname" = osx-gcc ]
+	if test "$jobname" = osx-gcc
 	then
 		MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
 	else
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#170 lore raw

PATCH v4 15/31 CI: check ignored unignored build artifacts in "win[+VS] build" too

In reply to Ævar Arnfjörð Bjarmason

Use the newly created "ci/check-unignored-build-artifacts.sh" script in the "win build" and "win+VS build" steps too. These steps weren't doing this sanity check of untracked files against .gitignore before, but as the check passes for them let's keep doing that extra sanity check.

---
 .github/workflows/main.yml | 6 ++++++
 1 file changed, 6 insertions(+)
.github/workflows/main.yml +6 -0
index c83f6a21ca3..05c5771e7f6 100644
@@ -93,6 +93,9 @@ jobs:
         HOME: ${{runner.workspace}}
         NO_PERL: 1
       run: . /etc/profile && ci/make-test-artifacts.sh artifacts
+    - run: ci/check-unignored-build-artifacts.sh
+      if: success()
+      shell: bash
     - name: zip up tracked files
       run: git archive -o artifacts/tracked.tar.gz HEAD
     - name: upload tracked files and build artifacts
@@ -180,6 +183,9 @@ jobs:
       run: |
         mkdir -p artifacts &&
         eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts NO_GETTEXT=YesPlease 2>&1 | grep ^tar)"
+    - run: ci/check-unignored-build-artifacts.sh
+      if: success()
+      shell: bash
     - name: zip up tracked files
       run: git archive -o artifacts/tracked.tar.gz HEAD
     - name: upload tracked files and build artifacts
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#171 lore raw

PATCH v4 16/31 CI: invoke "make artifacts-tar" directly in windows-build

In reply to Ævar Arnfjörð Bjarmason

Change the windows-build job to invoke the "make artifacts-tar" step directly, instead of calling a "ci/make-test-artifacts.sh" script.

The script was needed because "ci/lib.sh" would set up various environment variables for us, but now we can instead use the "ci/lib.sh" in its script mode.

The "mkdir -p" added in b819f1d2cec (ci: parallelize testing on Windows, 2019-01-29) isn't needed, the same commit added that "mkdir -p" to the "artifacts-tar" rule itself, so we can have "make" create the directory for us.

This also has the benefit of making the "build" step less chatty, since it won't start with the verbose "set -x" output, that's now contained in the "ci/lib.sh" step.

---
 .github/workflows/main.yml |  5 ++++-
 ci/make-test-artifacts.sh  | 10 ----------
 2 files changed, 4 insertions(+), 11 deletions(-)
 delete mode 100755 ci/make-test-artifacts.sh
.github/workflows/main.yml +4 -1
index 05c5771e7f6..2c23a19eac2 100644
@@ -87,12 +87,15 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
+    - run: ci/lib.sh
+      shell: bash
     - name: build
       shell: bash
       env:
         HOME: ${{runner.workspace}}
         NO_PERL: 1
-      run: . /etc/profile && ci/make-test-artifacts.sh artifacts
+      run: . /etc/profile && make artifacts-tar ARTIFACTS_DIRECTORY=artifacts
+      if: success()
     - run: ci/check-unignored-build-artifacts.sh
       if: success()
       shell: bash
ci/make-test-artifacts.sh deleted +0 -10
deleted file mode 100755
index 45298562982..00000000000
@@ -1,10 +0,0 @@
-#!/bin/sh
-#
-# Build Git and store artifacts for testing
-#
-
-mkdir -p "$1" # in case ci/lib.sh decides to quit early
-
-. ${0%/*}/lib.sh
-
-make artifacts-tar ARTIFACTS_DIRECTORY="$1"
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#172 lore raw

PATCH v4 17/31 CI: split up and reduce "ci/test-documentation.sh"

In reply to Ævar Arnfjörð Bjarmason

Change the "ci/test-documentation.sh" script to run the bash-specific parts in as one command in the CI job itself, and to run the two "make doc" commands at the top-level.

It'll now be obvious from the title of the step if we failed in the asciidoc or asciidoctor step.

Since the "check_unignored_build_artifacts()" function is now only used in "ci/check-unignored-build-artifacts.sh" move that function there.

The recipe for the job in ".github/workflows/main.yml" is now a bit verbose because it's effectively the same job twice, with a "make clean" in-between. It would be better for the verbosity to run it via a matrix as done in the alternate approach in [1] does, but then we'd sacrifice overall CPU time for the brevity. It might still be worth doing, but let's go for this simpler approach for now.

1. https://lore.kernel.org/git/[email protected]/

---
 .github/workflows/main.yml            | 16 +++++++++++++-
 ci/check-unignored-build-artifacts.sh | 10 +++++++++
 ci/lib.sh                             | 10 ---------
 ci/test-documentation.sh              | 31 +++++++++------------------
 4 files changed, 35 insertions(+), 32 deletions(-)
.github/workflows/main.yml +15 -1
index 2c23a19eac2..92b914f16fd 100644
@@ -365,4 +365,18 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/test-documentation.sh
+    - run: ci/lib.sh
+    - run: make check-docs
+    - run: "make doc > >(tee stdout.log) 2> >(tee stderr.raw >&2)"
+      shell: bash
+    - run: ci/test-documentation.sh AsciiDoc
+      if: success()
+    - run: ci/check-unignored-build-artifacts.sh
+      if: success()
+    - run: make clean
+    - run: "make USE_ASCIIDOCTOR=1 doc > >(tee stdout.log) 2> >(tee stderr.raw >&2)"
+      shell: bash
+    - run: ci/test-documentation.sh Asciidoctor
+      if: success()
+    - run: ci/check-unignored-build-artifacts.sh
+      if: success()
ci/check-unignored-build-artifacts.sh +10 -0
index 56d04b0db9a..0bc04f32804 100755
@@ -5,4 +5,14 @@
 
 . ${0%/*}/lib.sh
 
+check_unignored_build_artifacts ()
+{
+	! git ls-files --other --exclude-standard --error-unmatch \
+		-- ':/*' 2>/dev/null ||
+	{
+		echo "$(tput setaf 1)error: found unignored build artifacts$(tput sgr0)"
+		false
+	}
+}
+
 check_unignored_build_artifacts
ci/lib.sh +0 -10
index ffc0e524aaa..35a972fa3f5 100755
@@ -47,16 +47,6 @@ setenv () {
 	fi
 }
 
-check_unignored_build_artifacts ()
-{
-	! git ls-files --other --exclude-standard --error-unmatch \
-		-- ':/*' 2>/dev/null ||
-	{
-		echo "$(tput setaf 1)error: found unignored build artifacts$(tput sgr0)"
-		false
-	}
-}
-
 # GitHub Action doesn't set TERM, which is required by tput
 setenv TERM ${TERM:-dumb}
 
ci/test-documentation.sh +10 -21
index 6b5cce03bd7..b8a6a6f664e 100755
@@ -1,10 +1,12 @@
-#!/usr/bin/env bash
+#!/bin/sh
 #
-# Perform sanity checks on documentation and build it.
+# Perform sanity checks on "make doc" output and built documentation
 #
 
 . ${0%/*}/lib.sh
 
+generator=$1
+
 filter_log () {
 	sed -e '/^GIT_VERSION = /d' \
 	    -e "/constant Gem::ConfigMap is deprecated/d" \
@@ -14,28 +16,15 @@ filter_log () {
 	    "$1"
 }
 
-make check-docs
-
-# Build docs with AsciiDoc
-make doc > >(tee stdout.log) 2> >(tee stderr.raw >&2)
-cat stderr.raw
-filter_log stderr.raw >stderr.log
-test ! -s stderr.log
-test -s Documentation/git.html
-test -s Documentation/git.xml
-test -s Documentation/git.1
-grep '<meta name="generator" content="AsciiDoc ' Documentation/git.html
-
-rm -f stdout.log stderr.log stderr.raw
-check_unignored_build_artifacts
-
-# Build docs with AsciiDoctor
-make clean
-make USE_ASCIIDOCTOR=1 doc > >(tee stdout.log) 2> >(tee stderr.raw >&2)
 cat stderr.raw
 filter_log stderr.raw >stderr.log
 test ! -s stderr.log
 test -s Documentation/git.html
-grep '<meta name="generator" content="Asciidoctor ' Documentation/git.html
+if test "$generator" = "Asciidoctor"
+then
+	test -s Documentation/git.xml
+	test -s Documentation/git.1
+fi
+grep "<meta name=\"generator\" content=\"$generator " Documentation/git.html
 
 rm -f stdout.log stderr.log stderr.raw
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#173 lore raw

PATCH v4 18/31 CI: make ci/install-dependencies.sh POSIX-compatible

In reply to Ævar Arnfjörð Bjarmason

Change the unportable constructs in the "ci/install-dependencies.sh" scripts to use POSIX replacements, and have it run under "/bin/sh" instead of "/usr/bin/env bash". We needed to:

* Replace pushd/popd with a sub-shell.
* Change "mkdir --parents" to "mkdir -p".
* Use POSIX "export" syntax.

This is in preparation for combining this script and "ci/install-docker-dependencies.sh", which doesn't have access to "bash".

---
 ci/install-dependencies.sh | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)
ci/install-dependencies.sh +16 -9
index 01006379fd0..db58de0a6fd 100755
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh
 #
 # Install dependencies required to build and test Git on Linux and macOS
 #
@@ -21,7 +21,8 @@ ubuntu-latest)
 
 	P4_PATH="$HOME/custom/p4"
 	GIT_LFS_PATH="$HOME/custom/git-lfs"
-	export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH"
+	PATH="$GIT_LFS_PATH:$P4_PATH:$PATH"
+	export PATH
 	if test -n "$GITHUB_PATH"
 	then
 		echo "$PATH" >>"$GITHUB_PATH"
@@ -33,22 +34,28 @@ ubuntu-latest)
 	sudo apt-get -q update
 	sudo apt-get -q -y install language-pack-is libsvn-perl apache2 \
 		$UBUNTU_COMMON_PKGS $CC_PACKAGE
-	mkdir --parents "$P4_PATH"
-	pushd "$P4_PATH"
+	mkdir -p "$P4_PATH"
+	(
+		cd "$P4_PATH"
 		wget --quiet "$P4WHENCE/bin.linux26x86_64/p4d"
 		wget --quiet "$P4WHENCE/bin.linux26x86_64/p4"
 		chmod u+x p4d
 		chmod u+x p4
-	popd
-	mkdir --parents "$GIT_LFS_PATH"
-	pushd "$GIT_LFS_PATH"
+	)
+	mkdir -p "$GIT_LFS_PATH"
+	(
+		cd "$GIT_LFS_PATH"
 		wget --quiet "$LFSWHENCE/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz"
 		tar --extract --gunzip --file "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz"
 		cp git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs .
-	popd
+	)
 	;;
 macos-latest)
-	export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1
+	HOMEBREW_NO_AUTO_UPDATE=1
+	export HOMEBREW_NO_AUTO_UPDATE
+	HOMEBREW_NO_INSTALL_CLEANUP=1
+	export HOMEBREW_NO_INSTALL_CLEANUP
+
 	# Uncomment this if you want to run perf tests:
 	# brew install gnu-time
 	brew link --force gettext
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#174 lore raw

PATCH v4 19/31 CI: combine ci/install{,-docker}-dependencies.sh

In reply to Ævar Arnfjörð Bjarmason

Combine the "ci/install-dependencies.sh" and "ci/install-docker-dependencies.sh" into one script. The "case" statement in the latter only cared about "$jobname", and can be folded into the same "case" statement in the former.

The reason they split up is historical, until a preceding commit "ci/lib.sh" required "bash", which might not have been available in "docker".

This also fixes issue in "ci/install-docker-dependencies.sh" where we'd hide errors due to not using "set -e". Now that we include "ci/lib.sh" we won't have that potential issue.

---
 .github/workflows/main.yml        |  2 +-
 ci/install-dependencies.sh        | 15 +++++++++++++++
 ci/install-docker-dependencies.sh | 22 ----------------------
 3 files changed, 16 insertions(+), 23 deletions(-)
 delete mode 100755 ci/install-docker-dependencies.sh
.github/workflows/main.yml +1 -1
index 92b914f16fd..3fd3e4f873c 100644
@@ -311,7 +311,7 @@ jobs:
     container: ${{matrix.vector.image}}
     steps:
     - uses: actions/checkout@v1
-    - run: ci/install-docker-dependencies.sh
+    - run: ci/install-dependencies.sh
     - run: ci/lib.sh
     - run: make
     - run: make test
ci/install-dependencies.sh +15 -0
index db58de0a6fd..8e41747ef6f 100755
@@ -97,6 +97,21 @@ linux-gcc-default)
 	sudo apt-get -q update
 	sudo apt-get -q -y install $UBUNTU_COMMON_PKGS
 	;;
+linux32)
+	linux32 --32bit i386 sh -c '
+		apt update >/dev/null &&
+		apt install -y build-essential libcurl4-openssl-dev \
+			libssl-dev libexpat-dev gettext python >/dev/null
+	'
+	;;
+linux-musl)
+	apk add --update build-base curl-dev openssl-dev expat-dev gettext \
+		pcre2-dev python3 musl-libintl perl-utils ncurses >/dev/null
+	;;
+pedantic)
+	dnf -yq update >/dev/null &&
+	dnf -yq install make gcc findutils diffutils perl python3 gettext zlib-devel expat-devel openssl-devel curl-devel pcre2-devel >/dev/null
+	;;
 esac
 
 if type p4d >/dev/null && type p4 >/dev/null
ci/install-docker-dependencies.sh deleted +0 -22
deleted file mode 100755
index 78b7e326da6..00000000000
@@ -1,22 +0,0 @@
-#!/bin/sh
-#
-# Install dependencies required to build and test Git inside container
-#
-
-case "$jobname" in
-linux32)
-	linux32 --32bit i386 sh -c '
-		apt update >/dev/null &&
-		apt install -y build-essential libcurl4-openssl-dev \
-			libssl-dev libexpat-dev gettext python >/dev/null
-	'
-	;;
-linux-musl)
-	apk add --update build-base curl-dev openssl-dev expat-dev gettext \
-		pcre2-dev python3 musl-libintl perl-utils ncurses >/dev/null
-	;;
-pedantic)
-	dnf -yq update >/dev/null &&
-	dnf -yq install make gcc findutils diffutils perl python3 gettext zlib-devel expat-devel openssl-devel curl-devel pcre2-devel >/dev/null
-	;;
-esac
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#175 lore raw

PATCH v4 20/31 CI: move "env" definitions into ci/lib.sh

In reply to Ævar Arnfjörð Bjarmason

Have the GitHub CI jobs use the "ci/lib".sh as a source of truth for environment variables that affect their builds and tests. This brings them in line with the rest of the jobs, and make it easier to see the entire CI configuration per jobname at a glance.

To do this we need to add a ci/lib.sh "step" to the remaining jobs that didn't have it added in preceding commits.

The Makefile parameters are luckily all accepted via the environment, so we can export these instead of adding them to MAKEFLAGS.

Let's also use the documented GIT_CONIFG_* mechanism for setting config instead of the internal GIT_CONFIG_PARAMETERS variable. This adjusts code added in 889cacb6897 (ci: configure GitHub Actions for CI/PR, 2020-04-11), we could probably use the GIT_AUTHOR_NAME etc, but let's keep setting this via config, just with the documented mechanism.

By setting "ARTIFACTS_DIRECTORY=artifacts" once we don't need to repeat it in various places, let's get it from the environment instead.

Finally, the "DEVELOPER: 1" was only needed by the "sparse" job, which wasn't using "ci/lib.sh" (and which sets DEVELOPER=1). Let's instead have the "sparse" job use the "ci/lib.sh" and remove DEVELOPER=1 from ".github/workflows/main.yml".

This substantially speeds up the "sparse" job, since it'll now pick up the "--jobs" setting in MAKEFLAGS that we use everywhere else. Before it ran in around 4m30s, now in around 2m30s.

---
 .github/workflows/main.yml | 29 +++++++++++------------------
 ci/lib.sh                  | 21 +++++++++++++++++++++
 2 files changed, 32 insertions(+), 18 deletions(-)
.github/workflows/main.yml +11 -18
index 3fd3e4f873c..fe37e0f1b36 100644
@@ -2,9 +2,6 @@ name: CI
 
 on: [push, pull_request]
 
-env:
-  DEVELOPER: 1
-
 jobs:
   ci-config:
     name: config
@@ -93,8 +90,7 @@ jobs:
       shell: bash
       env:
         HOME: ${{runner.workspace}}
-        NO_PERL: 1
-      run: . /etc/profile && make artifacts-tar ARTIFACTS_DIRECTORY=artifacts
+      run: . /etc/profile && make artifacts-tar
       if: success()
     - run: ci/check-unignored-build-artifacts.sh
       if: success()
@@ -105,7 +101,7 @@ jobs:
       uses: actions/upload-artifact@v2
       with:
         name: windows-artifacts
-        path: artifacts
+        path: ${{env.ARTIFACTS_DIRECTORY}}
   windows-test:
     name: win test
     env:
@@ -126,6 +122,8 @@ jobs:
       shell: bash
       run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
+    - run: ci/lib.sh
+      shell: bash
     - name: test
       shell: bash
       run: . /etc/profile && ci/run-test-slice.sh ${{matrix.nr}} 10
@@ -145,9 +143,6 @@ jobs:
       jobname: vs-build
     needs: ci-config
     if: needs.ci-config.outputs.enabled == 'yes'
-    env:
-      NO_PERL: 1
-      GIT_CONFIG_PARAMETERS: "'user.name=CI' 'user.email=ci@git'"
     runs-on: windows-latest
     steps:
     - uses: actions/checkout@v2
@@ -171,6 +166,8 @@ jobs:
     - name: copy dlls to root
       shell: cmd
       run: compat\vcbuild\vcpkg_copy_dlls.bat release
+    - run: ci/lib.sh
+      shell: bash
     - name: generate Visual Studio solution
       shell: bash
       run: |
@@ -181,21 +178,16 @@ jobs:
     - name: bundle artifact tar
       shell: bash
       env:
-        MSVC: 1
         VCPKG_ROOT: ${{github.workspace}}\compat\vcbuild\vcpkg
       run: |
-        mkdir -p artifacts &&
-        eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts NO_GETTEXT=YesPlease 2>&1 | grep ^tar)"
-    - run: ci/check-unignored-build-artifacts.sh
-      if: success()
-      shell: bash
+        eval "$(make -n artifacts-tar 2>&1 | grep -e ^mkdir -e ^tar)"
     - name: zip up tracked files
       run: git archive -o artifacts/tracked.tar.gz HEAD
     - name: upload tracked files and build artifacts
       uses: actions/upload-artifact@v2
       with:
         name: vs-artifacts
-        path: artifacts
+        path: ${{env.ARTIFACTS_DIRECTORY}}
   vs-test:
     name: win+VS test
     env:
@@ -216,10 +208,10 @@ jobs:
     - name: extract tracked files and build artifacts
       shell: bash
       run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz
+    - run: ci/lib.sh
+      shell: bash
     - name: test
       shell: bash
-      env:
-        NO_SVN_TESTS: 1
       run: . /etc/profile && ci/run-test-slice.sh ${{matrix.nr}} 10
     - name: ci/print-test-failures.sh
       if: failure()
@@ -354,6 +346,7 @@ jobs:
     - uses: actions/checkout@v2
     - name: Install other dependencies
       run: ci/install-dependencies.sh
+    - run: ci/lib.sh
     - run: make sparse
   documentation:
     name: documentation
ci/lib.sh +21 -0
index 35a972fa3f5..8c574b21a67 100755
@@ -104,6 +104,27 @@ macos-latest)
 esac
 
 case "$jobname" in
+windows-build)
+	setenv --build NO_PERL NoThanks
+	setenv --build ARTIFACTS_DIRECTORY artifacts
+	;;
+vs-build)
+	setenv --build NO_PERL NoThanks
+	setenv --build NO_GETTEXT NoThanks
+	setenv --build ARTIFACTS_DIRECTORY artifacts
+	setenv --build INCLUDE_DLLS_IN_ARTIFACTS YesPlease
+	setenv --build MSVC YesPlease
+
+	setenv --build GIT_CONFIG_COUNT 2
+	setenv --build GIT_CONFIG_KEY_0 user.name
+	setenv --build GIT_CONFIG_VALUE_0 CI
+	setenv --build GIT_CONFIG_KEY_1 user.emailname
+	setenv --build GIT_CONFIG_VALUE_1 ci@git
+	setenv --build GIT_CONFIG_VALUE_1 ci@git
+	;;
+vs-test)
+	setenv --test NO_SVN_TESTS YesPlease
+	;;
 linux-gcc)
 	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME main
 	;;
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#176 lore raw

PATCH v4 21/31 ci/run-test-slice.sh: replace shelling out with "echo"

In reply to Ævar Arnfjörð Bjarmason

Amend this code added in b819f1d2cec (ci: parallelize testing on Windows, 2019-01-29) to stop shelling out to "tr", all we're trying to do here is to turn '\n'-delimited output into a ' '-delimited list. To do that we can just use "echo".

None of these files will contain the odd characters using "echo" might choke on in the general case (i.e. '\'-escapes and the like).

This has the added benefit of not adding a redundant ' ' to the end of the T-parameter, which didn't matter in practice, but as we'll see in a subsequent commit makes for slightly nicer output, as we'll show this variable using a GitHub CI-native feature.

---
 ci/run-test-slice.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
ci/run-test-slice.sh +3 -3
index 70326961454..1de6a18ca47 100755
@@ -5,6 +5,6 @@
 
 . ${0%/*}/lib.sh
 
-make --quiet -C t T="$(cd t &&
-	./helper/test-tool path-utils slice-tests "$1" "$2" t[0-9]*.sh |
-	tr '\n' ' ')"
+tests=$(echo $(cd t && ./helper/test-tool path-utils slice-tests "$1" "$2" \
+	t[0-9]*.sh))
+make --quiet -C t T="$tests"
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#177 lore raw

PATCH v4 22/31 CI: pre-select test slice in Windows & VS tests

In reply to Ævar Arnfjörð Bjarmason

In preceding commits the tests have been changed to do their setup via $GITHUB_ENV in one step, and to then have subsequent steps that re-use that environment.

Let's change the "test slice" tests added in b819f1d2cec (ci:
parallelize testing on Windows, 2019-01-29) to do the same. These tests select 10% of the tests to run in 10 "test slices". Now we'll select those in a step that immediately precedes the testing step, and then simply invoke "make -C t -e".

This has the advantage that the tests to be run are now listed in the standard "Run" drop-down at the start of the "test" step.

Since the "T" variable in "t/Makefile" doesn't normally accept overrides from the environment we need to invoke "make" with the "-e" option (a.k.a. "--environment-overrides"). We could also make $(T) in t/Makefile be a "?=" assigned variable, but this way works, and is arguably clearer as it's more obvious that we're injecting a special list of tests that override the normal behavior of that Makefile.

Note that we cannot run the top-level "make test" here, because of how the Windows CI builds git, i.e. either via CMake or some option that would cause "make test" to recompile git itself. Instead we run "make -C t [...]".

---
 .github/workflows/main.yml                     | 10 ++++++++--
 ci/{run-test-slice.sh => select-test-slice.sh} |  4 ++--
 2 files changed, 10 insertions(+), 4 deletions(-)
 rename ci/{run-test-slice.sh => select-test-slice.sh} (57%)
.github/workflows/main.yml +8 -2
index fe37e0f1b36..1e13718b989 100644
@@ -124,9 +124,12 @@ jobs:
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
     - run: ci/lib.sh
       shell: bash
+    - name: select tests
+      run: . /etc/profile && ci/select-test-slice.sh ${{matrix.nr}} 10
+      shell: bash
     - name: test
       shell: bash
-      run: . /etc/profile && ci/run-test-slice.sh ${{matrix.nr}} 10
+      run: . /etc/profile && make -C t -e
     - name: ci/print-test-failures.sh
       if: failure()
       shell: bash
@@ -210,9 +213,12 @@ jobs:
       run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz
     - run: ci/lib.sh
       shell: bash
+    - name: select tests
+      run: . /etc/profile && ci/select-test-slice.sh ${{matrix.nr}} 10
+      shell: bash
     - name: test
       shell: bash
-      run: . /etc/profile && ci/run-test-slice.sh ${{matrix.nr}} 10
+      run: . /etc/profile && make -C t -e
     - name: ci/print-test-failures.sh
       if: failure()
       shell: bash
ci/select-test-slice.sh renamed from ci/run-test-slice.sh +2 -2
similarity index 57%
rename from ci/run-test-slice.sh
rename to ci/select-test-slice.sh
index 1de6a18ca47..ec602f8a053 100755
@@ -1,10 +1,10 @@
 #!/bin/sh
 #
-# Test Git in parallel
+# Select a portion of the tests for testing Git in parallel
 #
 
 . ${0%/*}/lib.sh
 
 tests=$(echo $(cd t && ./helper/test-tool path-utils slice-tests "$1" "$2" \
 	t[0-9]*.sh))
-make --quiet -C t T="$tests"
+echo T="$tests" >>$GITHUB_ENV
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#178 lore raw

PATCH v4 23/31 CI: only invoke ci/lib.sh as "steps" in main.yml

In reply to Ævar Arnfjörð Bjarmason

Change the scripts in ci/ to stop using ci/lib.sh as a library, now that the only thing it did for them was to "set -ex" and possibly set TERM=dumb.

Let's create a ci/lib-tput.sh for those that need to use "tput" instead, and have these scripts invoke "set -ex" themselves.

This makes their invocation a lot less verbose, since they'll be relying on an earlier step in the CI job to have set the variables in $GITHUB_ENV, and won't be spewing their own trace output to set those variables again.

Let's also create a ci/lib-ci-type.sh, and have ci/lib.sh and ci/print-test-failures.sh share the logic to discover the CI type. We could have set the CI_TYPE in the environment with "setenv", but let's avoid that verbosity for this purely internal variable.

The "ci/lib.sh" is now no longer a "Library of functions shared by all CI scripts", so let's remove that commentary, and the misleading comment about "set -ex" being for "installing dependencies", we're now no longer using it in "ci/install-dependencies.sh" (but it does its own "set -ex").

---
 ci/check-unignored-build-artifacts.sh |  4 +++-
 ci/install-dependencies.sh            |  2 +-
 ci/lib-ci-type.sh                     |  9 +++++++++
 ci/lib-tput.sh                        |  5 +++++
 ci/lib.sh                             | 27 ++++++++++-----------------
 ci/print-test-failures.sh             |  6 +++---
 ci/select-test-slice.sh               |  2 +-
 ci/test-documentation.sh              |  2 +-
 8 files changed, 33 insertions(+), 24 deletions(-)
 create mode 100644 ci/lib-ci-type.sh
 create mode 100644 ci/lib-tput.sh
ci/check-unignored-build-artifacts.sh +3 -1
index 0bc04f32804..c27d6a97f45 100755
@@ -3,7 +3,9 @@
 # Check whether the build created anything not in our .gitignore
 #
 
-. ${0%/*}/lib.sh
+set -ex
+
+. ${0%/*}/lib-tput.sh
 
 check_unignored_build_artifacts ()
 {
ci/install-dependencies.sh +1 -1
index 8e41747ef6f..52f2180d5b4 100755
@@ -3,7 +3,7 @@
 # Install dependencies required to build and test Git on Linux and macOS
 #
 
-. ${0%/*}/lib.sh
+set -ex
 
 UBUNTU_COMMON_PKGS="make libssl-dev libcurl4-openssl-dev libexpat-dev
  tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl
ci/lib-ci-type.sh new file +9 -0
new file mode 100644
index 00000000000..6f01fd9e5d9
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+if test "$GITHUB_ACTIONS" = "true"
+then
+	CI_TYPE=github-actions
+else
+	echo "Could not identify CI type" >&2
+	exit 1
+fi
ci/lib-tput.sh new file +5 -0
new file mode 100644
index 00000000000..b62ef13c44c
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# GitHub Action doesn't set TERM, which is required by tput
+TERM=${TERM:-dumb}
+export TERM
ci/lib.sh +10 -17
index 8c574b21a67..8f7e178f3ff 100755
@@ -1,13 +1,9 @@
 #!/bin/sh
-
-# Library of functions shared by all CI scripts
-
-# Set 'exit on error' for all CI scripts to let the caller know that
-# something went wrong.
-# Set tracing executed commands, primarily setting environment variables
-# and installing dependencies.
 set -ex
 
+# Helper libraries
+. ${0%/*}/lib-ci-type.sh
+
 # Starting assertions
 if test -z "$jobname"
 then
@@ -47,18 +43,14 @@ setenv () {
 	fi
 }
 
-# GitHub Action doesn't set TERM, which is required by tput
-setenv TERM ${TERM:-dumb}
-
 # How many jobs to run in parallel?
 NPROC=10
 
 # Clear MAKEFLAGS that may come from the outside world.
 MAKEFLAGS=--jobs=$NPROC
 
-if test "$GITHUB_ACTIONS" = "true"
-then
-	CI_TYPE=github-actions
+case "$CI_TYPE" in
+github-actions)
 	CC="${CC:-gcc}"
 
 	setenv --test GIT_PROVE_OPTS "--timer --jobs $NPROC"
@@ -66,11 +58,12 @@ then
 	test Windows != "$RUNNER_OS" ||
 	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
 	setenv --test GIT_TEST_OPTS "$GIT_TEST_OPTS"
-else
-	echo "Could not identify CI type" >&2
-	env >&2
+	;;
+*)
+	echo "Unhandled CI type: $CI_TYPE" >&2
 	exit 1
-fi
+	;;
+esac
 
 setenv --build DEVELOPER 1
 setenv --test DEFAULT_TEST_TARGET prove
ci/print-test-failures.sh +3 -3
index 47e2ea1ad10..29f8c332eca 100755
@@ -3,10 +3,10 @@
 # Print output of failing tests
 #
 
-. ${0%/*}/lib.sh
+set -e
 
-# Tracing executed commands would produce too much noise in the loop below.
-set +x
+. ${0%/*}/lib-ci-type.sh
+. ${0%/*}/lib-tput.sh
 
 cd t/
 
ci/select-test-slice.sh +1 -1
index ec602f8a053..a0332095bb3 100755
@@ -3,7 +3,7 @@
 # Select a portion of the tests for testing Git in parallel
 #
 
-. ${0%/*}/lib.sh
+set -ex
 
 tests=$(echo $(cd t && ./helper/test-tool path-utils slice-tests "$1" "$2" \
 	t[0-9]*.sh))
ci/test-documentation.sh +1 -1
index b8a6a6f664e..64ff212cdaa 100755
@@ -3,7 +3,7 @@
 # Perform sanity checks on "make doc" output and built documentation
 #
 
-. ${0%/*}/lib.sh
+set -ex
 
 generator=$1
 
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#179 lore raw

PATCH v4 24/31 CI: narrow down variable definitions in --build and --test

In reply to Ævar Arnfjörð Bjarmason

In a preceding step the "setvar" function was made to take a "--build", "--test" or "--all" argument to indicate where the variables it sets were used.

Let's make use of that by having the relevant parts of ".github/workflows/main.yml" invoke "ci/lib.sh" with those options.

By doing this the set of variables shown in build-only steps will be fewer, which makes diagnosing anything going on there easier, as we won't have to look at a deluge of e.g. GIT_TEST_* variables.

Since we'll now always run "--build" for "make" and "--test" for "make test" we can stop setting the "fat" MAKEFLAGS entirely on the "test" steps, i.e. the one with PYTHON_PATH, CC etc. These will all be carried over from the earlier --build step.

That won't be true in the case of the "windows-test" and "vs-test" jobs, since they run separately from the corresponding "build" step. So we'll need to make sure that we have the --jobs=N argument for those.

This doesn't matter that much, as we'll still have --jobs=N in GIT_PROVE_OPTS. So the only thing we'll use it for is parallelism in the t/Makefile before we get to running "prove". Still, it's good to be consistent for good measure, and to run the t/Makefile itself in parallel.

---
 .github/workflows/main.yml | 20 +++++++++++---------
 ci/lib.sh                  | 30 +++++++++++++++++++++++-------
 2 files changed, 34 insertions(+), 16 deletions(-)
.github/workflows/main.yml +11 -9
index 1e13718b989..0787cadc76b 100644
@@ -84,7 +84,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
       shell: bash
     - name: build
       shell: bash
@@ -122,7 +122,7 @@ jobs:
       shell: bash
       run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
-    - run: ci/lib.sh
+    - run: ci/lib.sh --test
       shell: bash
     - name: select tests
       run: . /etc/profile && ci/select-test-slice.sh ${{matrix.nr}} 10
@@ -169,7 +169,7 @@ jobs:
     - name: copy dlls to root
       shell: cmd
       run: compat\vcbuild\vcpkg_copy_dlls.bat release
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
       shell: bash
     - name: generate Visual Studio solution
       shell: bash
@@ -211,7 +211,7 @@ jobs:
     - name: extract tracked files and build artifacts
       shell: bash
       run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz
-    - run: ci/lib.sh
+    - run: ci/lib.sh --test
       shell: bash
     - name: select tests
       run: . /etc/profile && ci/select-test-slice.sh ${{matrix.nr}} 10
@@ -275,8 +275,9 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
     - run: make
+    - run: ci/lib.sh --test
     - run: make test
       if: success()
     - run: ci/print-test-failures.sh
@@ -310,8 +311,9 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - run: ci/install-dependencies.sh
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
     - run: make
+    - run: ci/lib.sh --test
     - run: make test
       if: success() && matrix.vector.skip-tests != 'no'
     - run: ci/print-test-failures.sh
@@ -331,7 +333,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
     - run: make ci-static-analysis
   sparse:
     needs: ci-config
@@ -352,7 +354,7 @@ jobs:
     - uses: actions/checkout@v2
     - name: Install other dependencies
       run: ci/install-dependencies.sh
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
     - run: make sparse
   documentation:
     name: documentation
@@ -364,7 +366,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
     - run: make check-docs
     - run: "make doc > >(tee stdout.log) 2> >(tee stderr.raw >&2)"
       shell: bash
ci/lib.sh +23 -7
index 8f7e178f3ff..50bc766254b 100755
@@ -5,6 +5,13 @@ set -ex
 . ${0%/*}/lib-ci-type.sh
 
 # Starting assertions
+mode=$1
+if test -z "$mode"
+then
+	echo "need a $0 mode, e.g. --build or --test" >&2
+	exit 1
+fi
+
 if test -z "$jobname"
 then
 	echo "must set a CI jobname in the environment" >&2
@@ -16,11 +23,12 @@ setenv () {
 	while test $# != 0
 	do
 		case "$1" in
-		--build)
-			;;
-		--test)
-			;;
-		--all)
+		--build | --test)
+			if test "$1" != "$mode"
+			then
+				return 0
+			fi
+
 			;;
 		-*)
 			echo "BUG: bad setenv() option '$1'" >&2
@@ -46,8 +54,12 @@ setenv () {
 # How many jobs to run in parallel?
 NPROC=10
 
+# For "--test" we carry the MAKEFLAGS over from earlier steps, except
+# in stand-alone jobs which will use $COMMON_MAKEFLAGS.
+COMMON_MAKEFLAGS=--jobs=$NPROC
+
 # Clear MAKEFLAGS that may come from the outside world.
-MAKEFLAGS=--jobs=$NPROC
+MAKEFLAGS=$COMMON_MAKEFLAGS
 
 case "$CI_TYPE" in
 github-actions)
@@ -101,6 +113,9 @@ windows-build)
 	setenv --build NO_PERL NoThanks
 	setenv --build ARTIFACTS_DIRECTORY artifacts
 	;;
+windows-test)
+	setenv --test MAKEFLAGS "$COMMON_MAKEFLAGS"
+	;;
 vs-build)
 	setenv --build NO_PERL NoThanks
 	setenv --build NO_GETTEXT NoThanks
@@ -117,6 +132,7 @@ vs-build)
 	;;
 vs-test)
 	setenv --test NO_SVN_TESTS YesPlease
+	setenv --test MAKEFLAGS "$COMMON_MAKEFLAGS"
 	;;
 linux-gcc)
 	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME main
@@ -162,4 +178,4 @@ linux-leaks)
 	;;
 esac
 
-setenv --all MAKEFLAGS "$MAKEFLAGS CC=${CC:-cc}"
+setenv --build MAKEFLAGS "$MAKEFLAGS CC=${CC:-cc}"
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#180 lore raw

PATCH v4 25/31 CI: add more variables to MAKEFLAGS, except under vs-build

In reply to Ævar Arnfjörð Bjarmason

It's clearer that "DEVELOPER" is a flag that affects the Makefile itself in particular if it's put into "MAKEFLAGS than" if it generically sits in the environment. Let's move both it and "SKIP_DASHED_BUILT_INS" to "MAKEFLAGS".

We can't do this under vs-build, since that invokes cmake. Let's have only that job set these in the environment.

---
 ci/lib.sh | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)
ci/lib.sh +10 -2
index 50bc766254b..8fb0bfd43e1 100755
@@ -61,6 +61,13 @@ COMMON_MAKEFLAGS=--jobs=$NPROC
 # Clear MAKEFLAGS that may come from the outside world.
 MAKEFLAGS=$COMMON_MAKEFLAGS
 
+# Use common options for "make" (cmake in "vs-build" below uses the
+# intermediate variables directly)
+DEVELOPER=1
+MAKEFLAGS="$MAKEFLAGS DEVELOPER=$DEVELOPER"
+SKIP_DASHED_BUILT_INS=YesPlease
+MAKEFLAGS="$MAKEFLAGS SKIP_DASHED_BUILT_INS=$SKIP_DASHED_BUILT_INS"
+
 case "$CI_TYPE" in
 github-actions)
 	CC="${CC:-gcc}"
@@ -77,10 +84,8 @@ github-actions)
 	;;
 esac
 
-setenv --build DEVELOPER 1
 setenv --test DEFAULT_TEST_TARGET prove
 setenv --test GIT_TEST_CLONE_2GB true
-setenv --build SKIP_DASHED_BUILT_INS YesPlease
 
 case "$runs_on_pool" in
 ubuntu-latest)
@@ -117,6 +122,9 @@ windows-test)
 	setenv --test MAKEFLAGS "$COMMON_MAKEFLAGS"
 	;;
 vs-build)
+	setenv --build DEVELOPER $DEVELOPER
+	setenv --build SKIP_DASHED_BUILT_INS $SKIP_DASHED_BUILT_INS
+
 	setenv --build NO_PERL NoThanks
 	setenv --build NO_GETTEXT NoThanks
 	setenv --build ARTIFACTS_DIRECTORY artifacts
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#181 lore raw

PATCH v4 26/31 CI: set CC in MAKEFLAGS directly, don't add it to the environment

In reply to Ævar Arnfjörð Bjarmason

Rather than pass a "$CC" in the environment to be picked up in ci/lib.sh let's instead have ci/lib.sh itself add it directly to MAKEFLAGS. For "$CC_PACKAGE" its setting and use can stay within ci/install-dependencies.sh.

Setting CC=gcc by default made for confusing trace output, and since a preceding change to carry it and others over across "steps" in the GitHub CI it's been even more misleading. E.g. the "win+VS build" job confusingly has CC=gcc set, even though it builds with MSVC.

Let's instead rely on the Makefile default of CC=cc, and only override it for those jobs where it's needed. This does mean that we'll need to set it for the "pedantic" job, which previously relied on the default CC=gcc in case "clang" become the default on that platform.

This partially reverts my 707d2f2fe86 (CI: use "$runs_on_pool", not "$jobname" to select packages & config, 2021-11-23), i.e. we're now aiming to only set those variables specific jobs need.

---
 .github/workflows/main.yml | 13 -------------
 ci/install-dependencies.sh | 10 ++++++++++
 ci/lib.sh                  | 23 ++++++++++++++++++++---
 3 files changed, 30 insertions(+), 16 deletions(-)
.github/workflows/main.yml +0 -13
index 0787cadc76b..6d25ec4ae3b 100644
@@ -238,37 +238,24 @@ jobs:
       matrix:
         vector:
           - jobname: linux-clang
-            cc: clang
             pool: ubuntu-latest
           - jobname: linux-sha256
-            cc: clang
             os: ubuntu
             pool: ubuntu-latest
           - jobname: linux-gcc
-            cc: gcc
-            cc_package: gcc-8
             pool: ubuntu-latest
           - jobname: linux-TEST-vars
-            cc: gcc
             os: ubuntu
-            cc_package: gcc-8
             pool: ubuntu-latest
           - jobname: osx-clang
-            cc: clang
             pool: macos-latest
           - jobname: osx-gcc
-            cc: gcc
-            cc_package: gcc-9
             pool: macos-latest
           - jobname: linux-gcc-default
-            cc: gcc
             pool: ubuntu-latest
           - jobname: linux-leaks
-            cc: gcc
             pool: ubuntu-latest
     env:
-      CC: ${{matrix.vector.cc}}
-      CC_PACKAGE: ${{matrix.vector.cc_package}}
       jobname: ${{matrix.vector.jobname}}
       runs_on_pool: ${{matrix.vector.pool}}
     runs-on: ${{matrix.vector.pool}}
ci/install-dependencies.sh +10 -0
index 52f2180d5b4..1d05f43d19e 100755
@@ -9,6 +9,16 @@ UBUNTU_COMMON_PKGS="make libssl-dev libcurl4-openssl-dev libexpat-dev
  tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl
  libemail-valid-perl libio-socket-ssl-perl libnet-smtp-ssl-perl"
 
+CC_PACKAGE=
+case "$jobname" in
+linux-gcc | linux-TEST-vars)
+	CC_PACKAGE=gcc-8
+	;;
+osx-gcc)
+	CC_PACKAGE=gcc-9
+	;;
+esac
+
 case "$runs_on_pool" in
 ubuntu-latest)
 	# The Linux build installs the defined dependency versions below.
ci/lib.sh +20 -3
index 8fb0bfd43e1..977ee6293ef 100755
@@ -51,6 +51,9 @@ setenv () {
 	fi
 }
 
+# Clear variables that may come from the outside world.
+CC=
+
 # How many jobs to run in parallel?
 NPROC=10
 
@@ -70,8 +73,6 @@ MAKEFLAGS="$MAKEFLAGS SKIP_DASHED_BUILT_INS=$SKIP_DASHED_BUILT_INS"
 
 case "$CI_TYPE" in
 github-actions)
-	CC="${CC:-gcc}"
-
 	setenv --test GIT_PROVE_OPTS "--timer --jobs $NPROC"
 	GIT_TEST_OPTS="--verbose-log -x"
 	test Windows != "$RUNNER_OS" ||
@@ -143,9 +144,14 @@ vs-test)
 	setenv --test MAKEFLAGS "$COMMON_MAKEFLAGS"
 	;;
 linux-gcc)
+	CC=gcc
 	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME main
 	;;
+linux-gcc-default)
+	CC=gcc
+	;;
 linux-TEST-vars)
+	CC=gcc
 	setenv --test GIT_TEST_SPLIT_INDEX yes
 	setenv --test GIT_TEST_MERGE_ALGORITHM recursive
 	setenv --test GIT_TEST_FULL_IN_PACK_ARRAY true
@@ -160,13 +166,22 @@ linux-TEST-vars)
 	setenv --test GIT_TEST_WRITE_REV_INDEX 1
 	setenv --test GIT_TEST_CHECKOUT_WORKERS 2
 	;;
+osx-gcc)
+	CC=gcc
+	;;
+osx-clang)
+	CC=clang
+	;;
 linux-clang)
+	CC=clang
 	setenv --test GIT_TEST_DEFAULT_HASH sha1
 	;;
 linux-sha256)
+	CC=clang
 	setenv --test GIT_TEST_DEFAULT_HASH sha256
 	;;
 pedantic)
+	CC=gcc
 	# Don't run the tests; we only care about whether Git can be
 	# built.
 	setenv --build DEVOPTS pedantic
@@ -181,9 +196,11 @@ linux-musl)
 	MAKEFLAGS="$MAKEFLAGS GIT_TEST_UTF8_LOCALE=C.UTF-8"
 	;;
 linux-leaks)
+	CC=gcc
 	setenv --build SANITIZE leak
 	setenv --test GIT_TEST_PASSING_SANITIZE_LEAK true
 	;;
 esac
 
-setenv --build MAKEFLAGS "$MAKEFLAGS CC=${CC:-cc}"
+MAKEFLAGS="$MAKEFLAGS${CC:+ CC=$CC}"
+setenv --build MAKEFLAGS "$MAKEFLAGS"
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#182 lore raw

PATCH v4 27/31 CI: set SANITIZE=leak in MAKEFLAGS directly

In reply to Ævar Arnfjörð Bjarmason

As in the preceding commit's change to do likewise for CC, let's not set SANITIZE in the environment. When I added the "linux-leaks" job in in 956d2e4639b (tests: add a test mode for SANITIZE=leak, run it in CI, 2021-09-23) I missed that I could do that.

---
 ci/lib.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
ci/lib.sh +1 -1
index 977ee6293ef..ac96b7a1c61 100755
@@ -197,7 +197,7 @@ linux-musl)
 	;;
 linux-leaks)
 	CC=gcc
-	setenv --build SANITIZE leak
+	MAKEFLAGS="$MAKEFLAGS SANITIZE=leak"
 	setenv --test GIT_TEST_PASSING_SANITIZE_LEAK true
 	;;
 esac
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#183 lore raw

PATCH v4 28/31 CI: set PYTHON_PATH setting for osx-{clang,gcc} into "$jobname" case

In reply to Ævar Arnfjörð Bjarmason

Move the setting of "PYTHON_PATH=[...]" to the "$jobname" case statement. This partially backs out of my 707d2f2fe86 (CI: use "$runs_on_pool", not "$jobname" to select packages & config, 2021-11-23), now that we have a "osx-{clang,gcc}" anyway for setting "$CC" we might as well do away with this part of the "$runs_on_pool" case.

---
 ci/lib.sh | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)
ci/lib.sh +2 -8
index ac96b7a1c61..1a7200451e4 100755
@@ -104,14 +104,6 @@ ubuntu-latest)
 
 	setenv --test GIT_TEST_HTTPD true
 	;;
-macos-latest)
-	if test "$jobname" = osx-gcc
-	then
-		MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
-	else
-		MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python2)"
-	fi
-	;;
 esac
 
 case "$jobname" in
@@ -167,9 +159,11 @@ linux-TEST-vars)
 	setenv --test GIT_TEST_CHECKOUT_WORKERS 2
 	;;
 osx-gcc)
+	MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
 	CC=gcc
 	;;
 osx-clang)
+	MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python2)"
 	CC=clang
 	;;
 linux-clang)
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#184 lore raw

PATCH v4 29/31 CI: don't use "set -x" in "ci/lib.sh" output

In reply to Ævar Arnfjörð Bjarmason

Remove the "set -x" from the "ci/lib.sh" output. Before preceding commits the logic in that file was much more complex, and likely to fail in some scenarios.

Now we only task "ci/lib.sh" with setting various variables for subsequent steps in our jobs, so we can start emitting more tailored debugging output, which makes what it's doing easier to read.

This change also changes the output of the "ci/print-test-failures.sh" script, since it's the only other user of "ci/lib-ci-type.sh". In that case it's also helpful to know what "$CI_TYPE" we're using, as that script doesn't "set -x" and will act differently depending on the $CI_TYPE.

---
 ci/lib.sh | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)
ci/lib.sh +21 -2
index 1a7200451e4..ab7546800e0 100755
@@ -1,5 +1,5 @@
 #!/bin/sh
-set -ex
+set -e
 
 # Helper libraries
 . ${0%/*}/lib-ci-type.sh
@@ -18,15 +18,26 @@ then
 	exit 1
 fi
 
+# Show our configuration
+echo "CONFIG: CI_TYPE=$CI_TYPE" >&2
+echo "CONFIG: jobname=$jobname" >&2
+echo "CONFIG: runs_on_pool=$runs_on_pool" >&2
+echo "CONFIG: GITHUB_ENV=$GITHUB_ENV" >&2
+if test -n "$GITHUB_ENV"
+then
+	echo "CONFIG: GITHUB_ENV=$GITHUB_ENV" >&2
+fi
+
 # Helper functions
 setenv () {
+	local skip=
 	while test $# != 0
 	do
 		case "$1" in
 		--build | --test)
 			if test "$1" != "$mode"
 			then
-				return 0
+				skip=t
 			fi
 
 			;;
@@ -45,10 +56,18 @@ setenv () {
 	val=$2
 	shift 2
 
+	if test -n "$skip"
+	then
+		echo "SKIP '$key=$val'" >&2
+		return 0
+	fi
+
 	if test -n "$GITHUB_ENV"
 	then
 		echo "$key=$val" >>"$GITHUB_ENV"
 	fi
+
+	echo "SET: '$key=$val'" >&2
 }
 
 # Clear variables that may come from the outside world.
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#185 lore raw

PATCH v4 30/31 CI: have osx-gcc use gcc, not clang

In reply to Ævar Arnfjörð Bjarmason

Fix a regression in 707d2f2fe86 (CI: use "$runs_on_pool", not "$jobname" to select packages & config, 2021-11-23). In that commit I changed CC=gcc from CC=gcc-9, but on OSX the "gcc" in $PATH points to clang, we need to use gcc-9 instead.

---
 ci/lib.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
ci/lib.sh +1 -1
index ab7546800e0..8f80be4c3df 100755
@@ -179,7 +179,7 @@ linux-TEST-vars)
 	;;
 osx-gcc)
 	MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
-	CC=gcc
+	CC=gcc-9
 	;;
 osx-clang)
 	MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python2)"
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#186 lore raw

Re: [PATCH v4 30/31] CI: have osx-gcc use gcc, not clang

Quoted 4 lines · Ævar Arnfjörð Bjarmason

Fix a regression in 707d2f2fe86 (CI: use "$runs_on_pool", not "$jobname" to select packages & config, 2021-11-23). In that commit I changed CC=gcc from CC=gcc-9, but on OSX the "gcc" in $PATH points to clang, we need to use gcc-9 instead.

As this is fixing a bug in master, perhaps it would be better as a separate patch that can be merged before this series (this series could be rebased on to the fix)

Best Wishes

Phillip

Quoted 19 lines
---
  ci/lib.sh | 2 +-
  1 file changed, 1 insertion(+), 1 deletion(-)
ci/lib.sh +1 -1
index ab7546800e0..8f80be4c3df 100755
@@ -179,7 +179,7 @@ linux-TEST-vars)
  	;;
  osx-gcc)
  	MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
-	CC=gcc
+	CC=gcc-9
  	;;
  osx-clang)
  	MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python2)"
#187 lore raw

Re: [PATCH v4 30/31] CI: have osx-gcc use gcc, not clang

Quoted 9 lines · Phillip Wood
Quoted 4 lines · Ævar Arnfjörð Bjarmason

Fix a regression in 707d2f2fe86 (CI: use "$runs_on_pool", not "$jobname" to select packages & config, 2021-11-23). In that commit I changed CC=gcc from CC=gcc-9, but on OSX the "gcc" in $PATH points to clang, we need to use gcc-9 instead.

As this is fixing a bug in master, perhaps it would be better as a separate patch that can be merged before this series (this series could be rebased on to the fix)

Yeah, that sounds quite sensible.

Or perhaps start from 'master', merge this (and other fixes, if there are, in this series), merge Dscho's CI updates and then use that as an updated base?

#188 lore raw

Re: [PATCH v4 30/31] CI: have osx-gcc use gcc, not clang

Quoted 12 lines · Junio C Hamano
Quoted 9 lines · Phillip Wood
Quoted 4 lines · Ævar Arnfjörð Bjarmason

Fix a regression in 707d2f2fe86 (CI: use "$runs_on_pool", not "$jobname" to select packages & config, 2021-11-23). In that commit I changed CC=gcc from CC=gcc-9, but on OSX the "gcc" in $PATH points to clang, we need to use gcc-9 instead.

As this is fixing a bug in master, perhaps it would be better as a separate patch that can be merged before this series (this series could be rebased on to the fix)

Quoted 5 lines

Yeah, that sounds quite sensible.

Or perhaps start from 'master', merge this (and other fixes, if there are, in this series), merge Dscho's CI updates and then use that as an updated base?

It would certainly be nice to get Dscho's updates merged sooner rather than later as I think they represent a more significant improvement for CI users. There is a complication though in that Dscho's series adds github markup to the build output and this series separates the build from the tests which means that is not necessary. I think it should be easy enough to change Dscho's series so it only uses github markup for the tests which is the main improvement and just wait for the build and tests to be separated in this series (ideally they'd be a short easy to review series that did just that).

Best Wishes

Phillip

#189 lore raw

Re: [PATCH v4 30/31] CI: have osx-gcc use gcc, not clang

Quoted 7 lines · Phillip Wood

... There is a complication though in that Dscho's series adds github markup to the build output and this series separates the build from the tests which means that is not necessary. I think it should be easy enough to change Dscho's series so it only uses github markup for the tests which is the main improvement and just wait for the build and tests to be separated in this series (ideally they'd be a short easy to review series that did just that).

OK, Dscho, Ævar, does that sound like a workable plan? See a reroll of Dscho's series (which Phillip considers "should be easy enough") first, get it solid enough and merge down to 'next', and then see the refactoring by Ævar on top, hopefully with a minumum churn that makes it impossible for people to review?

Thanks, all.

#190 lore raw

js/ci-github-workflow-markup, was Re: [PATCH v4 30/31] CI: have osx-gcc use gcc, not clang

Hi Junio,

Quoted 5 lines · Junio C Hamano

OK, Dscho, Ævar, does that sound like a workable plan? See a reroll of Dscho's series (which Phillip considers "should be easy enough") first, get it solid enough and merge down to 'next', and then see the refactoring by Ævar on top, hopefully with a minumum churn that makes it impossible for people to review?

That works for me. Give me a few more days to catch up and then I will send a new iteration of my patch series.

Thanks,
Dscho

#191 lore raw

Re: [PATCH v4 30/31] CI: have osx-gcc use gcc, not clang

In reply to Junio C Hamano
Quoted 10 lines · Junio C Hamano
Quoted 7 lines · Phillip Wood

... There is a complication though in that Dscho's series adds github markup to the build output and this series separates the build from the tests which means that is not necessary. I think it should be easy enough to change Dscho's series so it only uses github markup for the tests which is the main improvement and just wait for the build and tests to be separated in this series (ideally they'd be a short easy to review series that did just that).

Quoted 5 lines

OK, Dscho, Ævar, does that sound like a workable plan? See a reroll of Dscho's series (which Phillip considers "should be easy enough") first, get it solid enough and merge down to 'next', and then see the refactoring by Ævar on top, hopefully with a minumum churn that makes it impossible for people to review?

First. I see you re-queued a the v2 of Johannes's patches. I had some outstanding fixes for my series [2] + the derived version of his on top
[3] which I thought I'd submit regardless.

To reply a bit to the thread at large, quoting Phillip:

Quoted 9 lines · Phillip Wood

It would certainly be nice to get Dscho's updates merged sooner rather than later as I think they represent a more significant improvement for CI users. There is a complication though in that Dscho's series adds github markup to the build output and this series separates the build from the tests which means that is not necessary. I think it should be easy enough to change Dscho's series so it only uses github markup for the tests which is the main improvement and just wait for the build and tests to be separated in this series (ideally they'd be a short easy to review series that did just that).

Phillip: I'd be very happy to see a version of my [2] rebased on [2] (or [3]), rather than the other way around. I'm fairly sure that if you attempt to do so you won't find it to be all that easy.

If you do and come up with some version which Johannes is OK with I'd be very happy with it. Without that I don't see how I wouldn't be in the same rut of wanting to improve other parts of the CI and that being brought up as a perceived blocker, without a suggested way forward (per [4]).

But as to a way forward. Even though my series is larger I still think a much better way forward is to do it first.

There's been some bugs here & there (all of which I believe are resolved), but I don't think anyone's spotted anything that's fundamentally wrong with that needs to be addressed in some open-ended way.

Whereas Johannes's v2 changes some UX things arguably for the better, but also introduces severe performance regressions in the UX. See [5] and [6]. I was routinely seeing e.g. ~5s loading times go to ~20s.

Now, I'm not sure (I haven't re-done the measurements), but I have a hunch that basing it on top of my series made it much better.

That's because in Johannes's the GitHub folding JS needs to fold the equivalent of make+make test+his GitHub markup. If it's based my series the the "make" steps are in their own "step", which it looks as though the GitHub UX needs to do less work to fold/unfold.

1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/
3. https://lore.kernel.org/git/[email protected]/
4. https://lore.kernel.org/git/[email protected]/
5. https://lore.kernel.org/git/[email protected]/
6. https://lore.kernel.org/git/[email protected]/

#192 lore raw

Re: [PATCH v4 30/31] CI: have osx-gcc use gcc, not clang

Quoted 3 lines · Ævar Arnfjörð Bjarmason

First. I see you re-queued a the v2 of Johannes's patches. I had some outstanding fixes for my series [2] + the derived version of his on top
[3] which I thought I'd submit regardless.

I do not mind flip-flopping on alternate days for a week or so.

Quite honestly I do not care which ones come first all that much. All I care about is (1) we keep CI functional and (2) we do not butcher Makefile too badly in the process. Part of "functional" is "not unusably slow".

Making the CI process less locked into a particular vendor and allowing the build/test with a configuration as similar as what is run on CI locally are nice to have that can come at the same time, but I wouldn't shed tears if we defer them to later, as long as they eventually materialize.

If you can fix Dscho's series by adding a series on top, that sounds like one approach of lessor resistance.

#193 lore raw

PATCH v4 31/31 CI: make it easy to use ci/*.sh outside of CI

In reply to Ævar Arnfjörð Bjarmason

In preceding commits the ci/.sh scripts have lost most of their CI-specific assumptions. Let's go even further and explicitly support running ci/lib.sh outside of CI.

This was possible before by faking up enough CI-specific variables, but as shown in the new "help" output being added here using the ci/lib.sh to provide "CI-like" has now become trivial.

The ci/print-test-failures.sh scripts can now be used outside of CI as well, the only GitHub CI-specific part is now guarded by a check that we'll pass if outside of GitHub CI.

There's also a special-case here to not clobber $MAKEFLAGS in the environment if we're outside of CI, in case the user has e.g. "jN" or other flags to "make" that they'd prefer configured already.

Using "ci/lib.sh" as a stand-alone script is much more useful if it doesn't hardcode NPROC=10, let's provide a poor shellscript replacement for the online_cpus() we have in thread-utils.c to cover the most common OS's. It was suggested to use "2>&1" to invoke "command -v", but per my reading of [2] and my own testing that doesn't seem to be needed. Perhaps it's only needed for "which(1)"?

1. https://lore.kernel.org/git/[email protected]/
2. https://pubs.opengroup.org/onlinepubs/009604499/utilities/command.html

---
 ci/lib-ci-type.sh         |  3 ---
 ci/lib-online_cpus.sh     | 26 ++++++++++++++++++
 ci/lib.sh                 | 57 ++++++++++++++++++++++++++++++++++++---
 ci/print-test-failures.sh |  4 ---
 4 files changed, 79 insertions(+), 11 deletions(-)
 create mode 100644 ci/lib-online_cpus.sh
ci/lib-ci-type.sh +0 -3
index 6f01fd9e5d9..09acab7aaec 100644
@@ -3,7 +3,4 @@
 if test "$GITHUB_ACTIONS" = "true"
 then
 	CI_TYPE=github-actions
-else
-	echo "Could not identify CI type" >&2
-	exit 1
 fi
ci/lib-online_cpus.sh new file +26 -0
new file mode 100644
index 00000000000..6ce7ec5692d
@@ -0,0 +1,26 @@
+#!/bin/sh
+
+# TODO: Ideally we'd compile t/helper/test-online-cpus.c early, but
+# that presents a chicken & egg problem. But if we move it to a
+# stand-oline command...
+online_cpus() {
+	NPROC=
+
+	if command -v nproc >/dev/null
+	then
+		# GNU coreutils
+		NPROC=$(nproc)
+	elif command -v sysctl >/dev/null
+	then
+		# BSD & Mac OS X
+		NPROC=$(sysctl -n hw.ncpu)
+	elif test -n "$NUMBER_OF_PROCESSORS"
+	then
+		# Windows
+		NPROC="$NUMBER_OF_PROCESSORS"
+	else
+		NPROC=1
+	fi
+
+	echo $NPROC
+}
ci/lib.sh +53 -4

94 lines not on this page · raw message

ci/print-test-failures.sh +0 -4
index 29f8c332eca..ec395c79ccd 100755
@@ -36,10 +36,6 @@ do
 			cp "${TEST_EXIT%.exit}.out" failed-test-artifacts/
 			tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir"
 			;;
-		*)
-			echo "Unhandled CI type: $CI_TYPE" >&2
-			exit 1
-			;;
 		esac
 	fi
 done
Signature · 1 line
2.36.0.rc2.898.gdf6bbac50ff
#194 lore raw

Re: [PATCH v4 31/31] CI: make it easy to use ci/*.sh outside of CI

Quoted 16 lines · Ævar Arnfjörð Bjarmason

In preceding commits the ci/.sh scripts have lost most of their CI-specific assumptions. Let's go even further and explicitly support running ci/lib.sh outside of CI.

This was possible before by faking up enough CI-specific variables, but as shown in the new "help" output being added here using the ci/lib.sh to provide "CI-like" has now become trivial.

The ci/print-test-failures.sh scripts can now be used outside of CI as well, the only GitHub CI-specific part is now guarded by a check that we'll pass if outside of GitHub CI.

There's also a special-case here to not clobber $MAKEFLAGS in the environment if we're outside of CI, in case the user has e.g. "jN" or other flags to "make" that they'd prefer configured already.

Quoted 6 lines

Using "ci/lib.sh" as a stand-alone script is much more useful if it doesn't hardcode NPROC=10, let's provide a poor shellscript replacement for the online_cpus() we have in thread-utils.c to cover the most common OS's. It was suggested to use "2>&1" to invoke "command -v", but per my reading of [2] and my own testing that doesn't seem to be needed. Perhaps it's only needed for "which(1)"?

Not redirecting stderr makes sense. I almost certainly accidentally typed "type foo" reflexively even though I meant to type "command -v foo", which explains why I thought redirecting stderr was needed. The explanation above probably would have best been done simply as a reply to the review thread rather than as part of the commit message, but dropping it from the commit message is not worth a reroll.

ci/lib-online_cpus.sh line 3

Quoted 11 lines

1. https://lore.kernel.org/git/[email protected]/
2. https://pubs.opengroup.org/onlinepubs/009604499/utilities/command.html

---

ci/lib-online_cpus.sh +3 -0
@@ -0,0 +1,26 @@
+# TODO: Ideally we'd compile t/helper/test-online-cpus.c early, but
+# that presents a chicken & egg problem. But if we move it to a
+# stand-oline command...

Um, what's a "stand-oline"?

#195 lore raw

Re: [PATCH v4 31/31] CI: make it easy to use ci/*.sh outside of CI

Quoted 25 lines · Eric Sunshine
Quoted 16 lines · Ævar Arnfjörð Bjarmason

In preceding commits the ci/.sh scripts have lost most of their CI-specific assumptions. Let's go even further and explicitly support running ci/lib.sh outside of CI.

This was possible before by faking up enough CI-specific variables, but as shown in the new "help" output being added here using the ci/lib.sh to provide "CI-like" has now become trivial.

The ci/print-test-failures.sh scripts can now be used outside of CI as well, the only GitHub CI-specific part is now guarded by a check that we'll pass if outside of GitHub CI.

There's also a special-case here to not clobber $MAKEFLAGS in the environment if we're outside of CI, in case the user has e.g. "jN" or other flags to "make" that they'd prefer configured already.

Quoted 6 lines

Using "ci/lib.sh" as a stand-alone script is much more useful if it doesn't hardcode NPROC=10, let's provide a poor shellscript replacement for the online_cpus() we have in thread-utils.c to cover the most common OS's. It was suggested to use "2>&1" to invoke "command -v", but per my reading of [2] and my own testing that doesn't seem to be needed. Perhaps it's only needed for "which(1)"?

Quoted 6 lines

Not redirecting stderr makes sense. I almost certainly accidentally typed "type foo" reflexively even though I meant to type "command -v foo", which explains why I thought redirecting stderr was needed. The explanation above probably would have best been done simply as a reply to the review thread rather than as part of the commit message, but dropping it from the commit message is not worth a reroll.

Thanks. I figured including it was worthwhile, since I'd probably on another day have looked at it & mixed them up the same way, i.e. you need the redirection for "which"...

Quoted 13 lines

ci/lib-online_cpus.sh line 3

Quoted 11 lines

1. https://lore.kernel.org/git/[email protected]/
2. https://pubs.opengroup.org/onlinepubs/009604499/utilities/command.html

---

ci/lib-online_cpus.sh +3 -0
@@ -0,0 +1,26 @@
+# TODO: Ideally we'd compile t/helper/test-online-cpus.c early, but
+# that presents a chicken & egg problem. But if we move it to a
+# stand-oline command...

Um, what's a "stand-oline"?

Typo for stand-alone, sorry.

#196 lore raw

v5 cover CI: run "make" in CI "steps", improve UX

In reply to Ævar Arnfjörð Bjarmason

This series changes CI "step" targets that are shellscripts that do N things to instead be single command invocations at the "step" level, driven by the CI recipe itself.

To do that we need to pass state that we previously re-setup for every "step" via $GITHUB_ENV, whose state we then helpfully show (this is just a standard GitHub CI feature) in a drop-down at the start of every "step".

I.e. at the tip of this series you can reliably look at that $GITHUB_ENV view to see what the full and relevant environment was for that "make", "make test" or whatever.

Change since v4:

* Minor UX improvements for 29/29, e.g. "error: " prefixes for error messages.
* Addressed feedback from Phillip Wood at
https://lore.kernel.org/git/[email protected]/:

- That bash-specific ${../../} syntax is gone
- The inverted "skip-test: no" is now a "skip-test: yes", as it should be.

- The new "trace" output from ci/lib.sh now doesn't have "SKIP"

     lines, i.e. we just say what variables we do set, not what we
     don't set. E.g.:
     https://github.com/avar/git/runs/6093145633?check_suite_focus=true#step:4:13

- I looked at the "a lot of churn" comment, there's e.g. "CI:
export variables via a wrapper" where we first make a wrapper to "export" variables, then make further use of it, some of which involves touching the same lines again.

Likewise first invoking ci/lib.sh, then splitting that up into ci/lib.sh --build and ci/lib.sh --test.

I looked at squashing some of that together, but I think it really makes it harder to review & reason about to do N things at the same time, even if the net diff size is bigger.
* Squashed together two commits that were adjusting bash syntax to be POSIX syntax.
* For the pedantic job that skips tests the "ci/lib.sh --test" step of setting up variables for it is now also skipped. See:
https://github.com/avar/git/runs/6093144782?check_suite_focus=true
* Made ci/select-test-slice.sh do the same "are we in GitHub CI?" check as the rest of the scripts, it's now easier to run it stand-alone for testing.
* Adjusted usage output, made it friendlier, moved those UX bits over to a test_commit-like getopts parsing.
* Fixed a typo/commit message suggestion by Eric Sunshine.

* This series contained a regression fix for an issue already on "master", which I've split out in
https://lore.kernel.org/git/[email protected]/

When the two are merged there's a conflict that's resolved thusly, per a --remerge-diff (which I snipped away with [...]). I.e. gcc-8 and gcc-9 need to be added to the linux-gcc and osx-gcc part of ci/lib.sh:

	diff --git a/ci/lib.sh b/ci/lib.sh
	remerge CONFLICT (content): Merge conflict in ci/lib.sh
	index 407e994edcd..a4e9d51ad6c 100755
	--- a/ci/lib.sh
	+++ b/ci/lib.sh
	@@ -30,126 +30,6 @@ usage() {
	 	then
	 		echo "$CI_TYPE_HELP" >&2
	 	fi
	-<<<<<<< ca24940186b (Merge branch 'avar/ci-have-osx-p4-install-fall-back-on-sha256-no-check' into build-master)
	-=======
	[...snipped ~100 lines ... ]
	->>>>>>> 92acf9420a9 (CI: select CC based on CC_PACKAGE (again))
	 	exit 1
	 }
	 
	@@ -373,7 +253,7 @@ vs-test)
	 	setenv --test MAKEFLAGS "$COMMON_MAKEFLAGS"
	 	;;
	 linux-gcc)
	-	CC=gcc
	+	CC=gcc-8
	 	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME main
	 	;;
	 linux-gcc-default)
	@@ -397,7 +277,7 @@ linux-TEST-vars)
	 	;;
	 osx-gcc)
	 	MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
	-	CC=gcc
	+	CC=gcc-9
	 	;;
	 osx-clang)
	 	MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python2)"

Ævar Arnfjörð Bjarmason (29):
CI: run "set -ex" early in ci/lib.sh
CI: make "$jobname" explicit, remove fallback
CI: remove more dead Travis CI support
CI: remove dead "tree skipping" code
CI: remove unused Azure ci/* code
CI/lib.sh: stop adding leading whitespace to $MAKEFLAGS
CI: don't have "git grep" invoke a pager in tree content check
CI: have "static-analysis" run a "make ci-static-analysis" target
CI: have "static-analysis" run "check-builtins", not "documentation"
CI: move p4 and git-lfs variables to ci/install-dependencies.sh
CI: consistently use "export" in ci/lib.sh
CI: export variables via a wrapper
CI: remove "run-build-and-tests.sh", run "make [test]" directly
CI: make ci/{lib,install-dependencies}.sh POSIX-compatible
CI: check ignored unignored build artifacts in "win[+VS] build" too
CI: invoke "make artifacts-tar" directly in windows-build
CI: split up and reduce "ci/test-documentation.sh"
CI: combine ci/install{,-docker}-dependencies.sh
CI: move "env" definitions into ci/lib.sh ci/run-test-slice.sh: replace shelling out with "echo"
CI: pre-select test slice in Windows & VS tests
CI: only invoke ci/lib.sh as "steps" in main.yml
CI: narrow down variable definitions in --build and --test
CI: add more variables to MAKEFLAGS, except under vs-build
CI: set CC in MAKEFLAGS directly, don't add it to the environment
CI: set SANITIZE=leak in MAKEFLAGS directly
CI: set PYTHON_PATH setting for osx-{clang,gcc} into "$jobname" case
CI: don't use "set -x" in "ci/lib.sh" output
CI: make it easy to use ci/*.sh outside of CI

 .github/workflows/main.yml            | 101 +++---
 Makefile                              |  31 +-
 ci/check-directional-formatting.bash  |   2 +-
 ci/check-unignored-build-artifacts.sh |  20 ++
 ci/install-dependencies.sh            |  82 +++--
 ci/install-docker-dependencies.sh     |  22 --
 ci/lib-ci-type.sh                     |   6 +
 ci/lib-online_cpus.sh                 |  26 ++
 ci/lib-tput.sh                        |   5 +
 ci/lib.sh                             | 424 ++++++++++++++++----------
 ci/make-test-artifacts.sh             |  12 -
 ci/mount-fileshare.sh                 |  25 --
 ci/print-test-failures.sh             |  53 +---
 ci/run-build-and-tests.sh             |  54 ----
 ci/run-docker-build.sh                |  66 ----
 ci/run-docker.sh                      |  47 ---
 ci/run-static-analysis.sh             |  32 --
 ci/run-test-slice.sh                  |  17 --
 ci/select-test-slice.sh               |  13 +
 ci/test-documentation.sh              |  37 +--
 ci/util/extract-trash-dirs.sh         |  50 ---
 shared.mak                            |   1 +
 22 files changed, 508 insertions(+), 618 deletions(-)
 create mode 100755 ci/check-unignored-build-artifacts.sh
 delete mode 100755 ci/install-docker-dependencies.sh
 create mode 100644 ci/lib-ci-type.sh
 create mode 100644 ci/lib-online_cpus.sh
 create mode 100644 ci/lib-tput.sh
 delete mode 100755 ci/make-test-artifacts.sh
 delete mode 100755 ci/mount-fileshare.sh
 delete mode 100755 ci/run-build-and-tests.sh
 delete mode 100755 ci/run-docker-build.sh
 delete mode 100755 ci/run-docker.sh
 delete mode 100755 ci/run-static-analysis.sh
 delete mode 100755 ci/run-test-slice.sh
 create mode 100755 ci/select-test-slice.sh
 delete mode 100755 ci/util/extract-trash-dirs.sh

Range-diff against v4:

 1:  ed27ca0b7e7 =  1:  84d8098e365 CI: run "set -ex" early in ci/lib.sh
 2:  737fe1e2c4c !  2:  3f5c071b0f6 CI: make "$jobname" explicit, remove fallback
    @@ ci/lib.sh
     +# Starting assertions
     +if test -z "$jobname"
     +then
    -+	echo "must set a CI jobname in the environment" >&2
    ++	echo "error: must set a CI jobname in the environment" >&2
     +	exit 1
     +fi
     +
 3:  d875069fc28 =  3:  c64845a4efb CI: remove more dead Travis CI support
 4:  2590459db33 =  4:  45856e67d5d CI: remove dead "tree skipping" code
 5:  e311ea8b9b3 =  5:  d4671148eb7 CI: remove unused Azure ci/* code
 6:  151f28e3818 =  6:  45ea80d0495 CI/lib.sh: stop adding leading whitespace to $MAKEFLAGS
 7:  6c2a8ee9c4e =  7:  a264b40ba0d CI: don't have "git grep" invoke a pager in tree content check
 8:  64a0a1fc8ce =  8:  4a58f7be8d4 CI: have "static-analysis" run a "make ci-static-analysis" target
 9:  0f780bf9e3c =  9:  78c0c81af43 CI: have "static-analysis" run "check-builtins", not "documentation"
10:  3cdeae9c141 = 10:  07e36035a1e CI: move p4 and git-lfs variables to ci/install-dependencies.sh
11:  1d3d357dcef = 11:  272bd590a95 CI: consistently use "export" in ci/lib.sh
12:  5ef0756d095 = 12:  ea35327e24c CI: export variables via a wrapper
13:  e0ce614eb6d ! 13:  f6ff81f48ab CI: remove "run-build-and-tests.sh", run "make [test]" directly
    @@ .github/workflows/main.yml: jobs:
                image: daald/ubuntu32:xenial
              - jobname: pedantic
                image: fedora
    -+          skip-tests: no
    ++          skip-tests: yes
          env:
            jobname: ${{matrix.vector.jobname}}
          runs-on: ubuntu-latest
    @@ .github/workflows/main.yml: jobs:
     +    - run: ci/lib.sh
     +    - run: make
     +    - run: make test
    -+      if: success() && matrix.vector.skip-tests != 'no'
    ++      if: success() && matrix.vector.skip-tests != 'yes'
          - run: ci/print-test-failures.sh
     -      if: failure()
    -+      if: failure() && matrix.vector.skip-tests != 'no'
    ++      if: failure() && matrix.vector.skip-tests != 'yes'
          - name: Upload failed tests' directories
            if: failure() && env.FAILED_TEST_ARTIFACTS != ''
            uses: actions/upload-artifact@v1
14:  a8e70124929 <  -:  ----------- ci/lib.sh: use "test" instead of "["
18:  eec7a11376e ! 14:  533dbc50c4f CI: make ci/install-dependencies.sh POSIX-compatible

@@ Metadata
Author: Ævar Arnfjörð Bjarmason <[email protected]>

## Commit message ##
- CI: make ci/install-dependencies.sh POSIX-compatible
+ CI: make ci/{lib,install-dependencies}.sh POSIX-compatible

Change the unportable constructs in the "ci/install-dependencies.sh" scripts to use POSIX replacements, and have it run under "/bin/sh"
@@ Commit message
"ci/install-docker-dependencies.sh", which doesn't have access to "bash".

+ While we're at it change "if [" to "if test" in ci/lib.sh. This isn't
+ strictly speaking a case of POSIX-incompatibility, but that syntax was
+ being used there because this code was using idiomatic bash
+ constructs. Let's have it use our usual style of "if test" instead.
+

## ci/install-dependencies.sh ##
@@ ci/install-dependencies.sh: ubuntu-latest)
# Uncomment this if you want to run perf tests:
# brew install gnu-time
brew link --force gettext
+@@ ci/install-dependencies.sh: macos-latest) +
+ if test -n "$CC_PACKAGE"
+ then
+- BREW_PACKAGE=${CC_PACKAGE/-/@}
++ BREW_PACKAGE=$(echo $CC_PACKAGE | tr '-' '@')
+ brew install "$BREW_PACKAGE"
+ brew link "$BREW_PACKAGE"
+ fi
+
+ ## ci/lib.sh ##
+@@ ci/lib.sh: ubuntu-latest)
+ break
+ fi
+
+- if [ "$jobname" = linux-gcc ]
++ if test "$jobname" = linux-gcc
+ then
+ MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python3"
+ else
+@@ ci/lib.sh: ubuntu-latest)
+ setenv --test GIT_TEST_HTTPD true
+ ;;
+ macos-latest)
+- if [ "$jobname" = osx-gcc ]
++ if test "$jobname" = osx-gcc
+ then
+ MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
+ else

15:  bb335d1c0ca = 15:  6044c2b383f CI: check ignored unignored build artifacts in "win[+VS] build" too
16:  5458e6dab0b = 16:  4d1a9f88c32 CI: invoke "make artifacts-tar" directly in windows-build
17:  5e5a5b71700 = 17:  5c8f464a60f CI: split up and reduce "ci/test-documentation.sh"
19:  a9c98582de1 = 18:  a218be76819 CI: combine ci/install{,-docker}-dependencies.sh
20:  96e7eb68906 = 19:  83e21b4f501 CI: move "env" definitions into ci/lib.sh
21:  fdbe8554f5d = 20:  fd0e1610577 ci/run-test-slice.sh: replace shelling out with "echo"
22:  f257ab59971 ! 21:  0c7cd9d64ba CI: pre-select test slice in Windows & VS tests
    @@ ci/run-test-slice.sh => ci/select-test-slice.sh
      tests=$(echo $(cd t && ./helper/test-tool path-utils slice-tests "$1" "$2" \
      	t[0-9]*.sh))
     -make --quiet -C t T="$tests"
    -+echo T="$tests" >>$GITHUB_ENV
    ++if test -n "$GITHUB_ENV"
    ++then
    ++	echo T="$tests" >>$GITHUB_ENV
    ++fi
23:  1d95088da29 = 22:  1b077665352 CI: only invoke ci/lib.sh as "steps" in main.yml
24:  a7434b215db ! 23:  1867da7eddb CI: narrow down variable definitions in --build and --test
    @@ Commit message
         fewer, which makes diagnosing anything going on there easier, as we
         won't have to look at a deluge of e.g. GIT_TEST_* variables.
     
    +    For the "pedantic" job (which has no test phase) we won't run the
    +    "ci/lib.sh --test" step, which will be clearly visible as a skipped
    +    step in the UX.
    +
         Since we'll now always run "--build" for "make" and "--test" for "make
         test" we can stop setting the "fat" MAKEFLAGS entirely on the "test"
         steps, i.e. the one with PYTHON_PATH, CC etc. These will all be
    @@ .github/workflows/main.yml: jobs:
     +    - run: ci/lib.sh --build
          - run: make
     +    - run: ci/lib.sh --test
    ++      if: success() && matrix.vector.skip-tests != 'yes'
          - run: make test
    -       if: success() && matrix.vector.skip-tests != 'no'
    +       if: success() && matrix.vector.skip-tests != 'yes'
          - run: ci/print-test-failures.sh
     @@ .github/workflows/main.yml: jobs:
          steps:
    @@ .github/workflows/main.yml: jobs:
     
      ## ci/lib.sh ##
     @@ ci/lib.sh: set -ex
    + # Helper libraries
      . ${0%/*}/lib-ci-type.sh
      
    ++# Parse options
    ++mode_build=
    ++mode_test=
    ++while test $# != 0
    ++do
    ++	case "$1" in
    ++	--build)
    ++		mode_build=t
    ++		;;
    ++	--test)
    ++		mode_test=t
    ++		;;
    ++	-*)
    ++		echo "error: invalid option: $1" >&2
    ++		exit 1
    ++		;;
    ++	*)
    ++		echo "error: invalid argument: $1" >&2
    ++		exit 1
    ++		;;
    ++	esac
    ++	shift
    ++done
    ++
      # Starting assertions
    -+mode=$1
    -+if test -z "$mode"
    + if test -z "$jobname"
    + then
    +@@ ci/lib.sh: then
    + 	exit 1
    + fi
    + 
    ++if test "$mode_test$mode_build" != "t"
     +then
    -+	echo "need a $0 mode, e.g. --build or --test" >&2
    ++	echo "error: need one mode, e.g. --build or --test" >&2
     +	exit 1
     +fi
     +
    - if test -z "$jobname"
    - then
    - 	echo "must set a CI jobname in the environment" >&2
    -@@ ci/lib.sh: setenv () {
    + # Helper functions
    + setenv () {
      	while test $# != 0
      	do
      		case "$1" in
    --		--build)
    --			;;
    --		--test)
    + 		--build)
    ++			if test -z "$mode_build"
    ++			then
    ++				return 0
    ++			fi
    + 			;;
    + 		--test)
     -			;;
     -		--all)
    -+		--build | --test)
    -+			if test "$1" != "$mode"
    ++			if test -z "$mode_test"
     +			then
     +				return 0
     +			fi
    -+
      			;;
      		-*)
      			echo "BUG: bad setenv() option '$1'" >&2
25:  78f4609a9c1 = 24:  a4e3a9d2c28 CI: add more variables to MAKEFLAGS, except under vs-build
26:  26a34f1d4b9 ! 25:  48d99cee809 CI: set CC in MAKEFLAGS directly, don't add it to the environment

@@ Commit message
"$jobname" to select packages & config, 2021-11-23), i.e. we're now aiming to only set those variables specific jobs need.

+ Since we don't need to make this generic enough to handle "gcc-9"
+ turning into "gcc@9" for "brew install" let's remove what was
+ originally a bash-specific replacement in 707d2f2fe86 (CI: use
+ "$runs_on_pool", not "$jobname" to select packages & config,
+ 2021-11-23), and which a preceding commit changed to a "tr"
+ invocation.
+

## .github/workflows/main.yml ##
@@ ci/install-dependencies.sh: UBUNTU_COMMON_PKGS="make libssl-dev libcurl4-openssl
libemail-valid-perl libio-socket-ssl-perl libnet-smtp-ssl-perl"

+CC_PACKAGE=
++BREW_CC_PACKAGE=
+case "$jobname" in
+linux-gcc | linux-TEST-vars)
+ CC_PACKAGE=gcc-8
+ ;;
+osx-gcc)
-+ CC_PACKAGE=gcc-9
++ BREW_CC_PACKAGE=gcc@9
+ ;;
+esac
+
case "$runs_on_pool" in
ubuntu-latest)
# The Linux build installs the defined dependency versions below.
+@@ ci/install-dependencies.sh: macos-latest)
+ } ||
+ brew install homebrew/cask/perforce
+
+- if test -n "$CC_PACKAGE"
++ if test -n "$BREW_CC_PACKAGE"
+ then
+- BREW_PACKAGE=$(echo $CC_PACKAGE | tr '-' '@')
+- brew install "$BREW_PACKAGE"
+- brew link "$BREW_PACKAGE"
++ brew install "$BREW_CC_PACKAGE"
++ brew link "$BREW_CC_PACKAGE"
+ fi
+ ;;
+ esac

## ci/lib.sh ##
@@ ci/lib.sh: setenv () {

27:  514de8d16b0 = 26:  07a6c087297 CI: set SANITIZE=leak in MAKEFLAGS directly
28:  6501059c594 = 27:  50f21ffdffe CI: set PYTHON_PATH setting for osx-{clang,gcc} into "$jobname" case
29:  1bdcf1399d9 ! 28:  e35dff2bef3 CI: don't use "set -x" in "ci/lib.sh" output
    @@ Commit message
         script doesn't "set -x" and will act differently depending on the
         $CI_TYPE.
     
    +    It's useful when developing this script to see what variables
    +    it *doesn't* set, but such output isn't useful to users, so let's hide
    +    the "SKIP" output behind a --debug option.
    +
         Signed-off-by: Ævar Arnfjörð Bjarmason <[email protected]>
     
      ## ci/lib.sh ##
    @@ ci/lib.sh
      
      # Helper libraries
      . ${0%/*}/lib-ci-type.sh
    +@@ ci/lib.sh: set -ex
    + # Parse options
    + mode_build=
    + mode_test=
    ++mode_debug=
    + while test $# != 0
    + do
    + 	case "$1" in
    + 	--build)
    ++		echo "MODE: $1" >&2
    + 		mode_build=t
    + 		;;
    + 	--test)
    ++		echo "MODE: $1" >&2
    + 		mode_test=t
    + 		;;
    ++	--debug)
    ++		echo "DEBUG: $1" >&2
    ++		mode_debug=t
    ++		;;
    + 	-*)
    + 		echo "error: invalid option: $1" >&2
    + 		exit 1
     @@ ci/lib.sh: then
      	exit 1
      fi
    @@ ci/lib.sh: then
     +echo "CONFIG: CI_TYPE=$CI_TYPE" >&2
     +echo "CONFIG: jobname=$jobname" >&2
     +echo "CONFIG: runs_on_pool=$runs_on_pool" >&2
    -+echo "CONFIG: GITHUB_ENV=$GITHUB_ENV" >&2
     +if test -n "$GITHUB_ENV"
     +then
     +	echo "CONFIG: GITHUB_ENV=$GITHUB_ENV" >&2
     +fi
    ++echo >&2
     +
      # Helper functions
      setenv () {
    -+	local skip=
    ++	skip=
      	while test $# != 0
      	do
      		case "$1" in
    - 		--build | --test)
    - 			if test "$1" != "$mode"
    + 		--build)
    + 			if test -z "$mode_build"
    + 			then
    +-				return 0
    ++				skip=t
    + 			fi
    + 			;;
    + 		--test)
    + 			if test -z "$mode_test"
      			then
     -				return 0
     +				skip=t
      			fi
    - 
      			;;
    + 		-*)
     @@ ci/lib.sh: setenv () {
      	val=$2
      	shift 2
      
     +	if test -n "$skip"
     +	then
    -+		echo "SKIP '$key=$val'" >&2
    ++		if test -n "$mode_debug"
    ++		then
    ++			echo "SKIP: '$key=$val'" >&2
    ++		fi
     +		return 0
     +	fi
     +
30:  94abb826627 <  -:  ----------- CI: have osx-gcc use gcc, not clang
31:  4fc67e668da ! 29:  13d8c6997c3 CI: make it easy to use ci/*.sh outside of CI

@@ Commit message
Using "ci/lib.sh" as a stand-alone script is much more useful if it doesn't hardcode NPROC=10, let's provide a poor shellscript replacement for the online_cpus() we have in thread-utils.c to cover
- the most common OS's. It was suggested to use "2>&1" to invoke
- "command -v", but per my reading of [2] and my own testing that
- doesn't seem to be needed. Perhaps it's only needed for "which(1)"?
-
- 1. https://lore.kernel.org/git/[email protected]/
- 2. https://pubs.opengroup.org/onlinepubs/009604499/utilities/command.html
+ the most common OS's.

@@ ci/lib-online_cpus.sh (new)
+#!/bin/sh
+
+# TODO: Ideally we'd compile t/helper/test-online-cpus.c early, but
-+# that presents a chicken & egg problem. But if we move it to a -+# stand-oline command...
++# that currently presents a chicken & egg problem. We need this before ++# we build (much of) anything.
+online_cpus() {
+ NPROC=
+
@@ ci/lib.sh
#!/bin/sh
set -e

-+# Usage
++# Usage
+CI_TYPE_HELP_COMMANDS='
+ # run "make all test" like the "linux-leaks" job
+ (eval $(jobname=linux-leaks ci/lib.sh --all) && make test)
@@ ci/lib.sh
+'
+
+CI_TYPE_HELP="
-+running $0 outside of CI? You can use ci/lib.sh to set up your ++Running $0 outside of CI? You can use ci/lib.sh to set up your
+environment like a given CI job. E.g.:
+$CI_TYPE_HELP_COMMANDS
-+
-+note that some of these (e.g. the linux-musl one) may not work as ++Note that some of these (e.g. the linux-musl one) may not work as
+expected due to the CI job configuring a platform that may not match +yours."
+
- # Helper libraries
- . ${0%/*}/lib-ci-type.sh
-
-@@ ci/lib.sh: mode=$1
- if test -z "$mode"
- then
- echo "need a $0 mode, e.g. --build or --test" >&2
++usage() {
++ echo "usage: jobname=<job-name> [runs_on_pool=<pool-name>] $0 (--build | --test)" >&2
+ if test -z "$CI_TYPE"
+ then
+ echo "$CI_TYPE_HELP" >&2
+ fi
- exit 1
++ exit 1
++}
++
+ # Helper libraries
+ . ${0%/*}/lib-ci-type.sh
+
+ # Parse options
++mode_all=
+ mode_build=
+ mode_test=
+ mode_debug=
+ while test $# != 0
+ do
+ case "$1" in
++ --all)
++ echo "MODE: $1" >&2
++ mode_all=t
++ mode_build=t
++ mode_test=t
++ ;;
+ --build)
+ echo "MODE: $1" >&2
+ mode_build=t
+@@ ci/lib.sh: do
+ ;;
+ -*)
+ echo "error: invalid option: $1" >&2
+- exit 1
++ usage
+ ;;
+ *)
+ echo "error: invalid argument: $1" >&2
+- exit 1
++ usage
+ ;;
+ esac
+ shift
+@@ ci/lib.sh: done
+ if test -z "$jobname"
+ then
+ echo "error: must set a CI jobname in the environment" >&2
+- exit 1
++ usage
fi

+-if test "$mode_test$mode_build" != "t"
++if test "$mode_test$mode_build" != "t" && test -z "$mode_all"
+ then
+ echo "error: need one mode, e.g. --build or --test" >&2
+- exit 1
++ usage
+ fi
+
+ # Show our configuration
@@ ci/lib.sh: setenv () {
do
case "$1" in
- --build | --test)
-- if test "$1" != "$mode"
-+ if test "$1" != "$mode" && test "$mode" != "--all"
+ --build)
+- if test -z "$mode_build"
++ if test -z "$mode_build$mode_all"
+ then
+ skip=t
+ fi
+ ;;
+ --test)
+- if test -z "$mode_test"
++ if test -z "$mode_test$mode_all"
then
skip=t
fi

Signature · 1 line
2.36.0.879.g3659959fcca
#197 lore raw

PATCH v5 1/29 CI: run "set -ex" early in ci/lib.sh

Change ci/lib.sh to run "set -ex" before it does anything else, before this we were omitting a few "export" commands in our trace output that happened before we enabled tracing.

---
 ci/lib.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
ci/lib.sh +6 -6
index cbc2f8f1caa..57141d38a85 100755
@@ -1,5 +1,11 @@
 # Library of functions shared by all CI scripts
 
+# Set 'exit on error' for all CI scripts to let the caller know that
+# something went wrong.
+# Set tracing executed commands, primarily setting environment variables
+# and installing dependencies.
+set -ex
+
 skip_branch_tip_with_tag () {
 	# Sometimes, a branch is pushed at the same time the tag that points
 	# at the same commit as the tip of the branch is pushed, and building
@@ -85,12 +91,6 @@ export TERM=${TERM:-dumb}
 # Clear MAKEFLAGS that may come from the outside world.
 export MAKEFLAGS=
 
-# Set 'exit on error' for all CI scripts to let the caller know that
-# something went wrong.
-# Set tracing executed commands, primarily setting environment variables
-# and installing dependencies.
-set -ex
-
 if test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI"
 then
 	CI_TYPE=azure-pipelines
Signature · 1 line
2.36.0.879.g3659959fcca
#198 lore raw

PATCH v5 2/29 CI: make "$jobname" explicit, remove fallback

In reply to Ævar Arnfjörð Bjarmason

Remove the $jobname fallback last adjusted in b011fabd6e0 (ci/lib.sh:
encapsulate Travis-specific things, 2019-01-27).

This makes all of our CI jobs have an explicit $jobname, and removes the special-case of the Windows jobs implicitly setting one (that they don't currently use). In subsequent commits we'll use this new $jobname explicitly.

This is a follow-up to my 25715419bf4 (CI: don't run "make test" twice in one job, 2021-11-23), since then all the jobs except the Windows jobs have had an explicit $jobname.

In the case of the Windows jobs we were using a jobname of windows-gcc for both, due to picking "gcc" as a default fallback for "$CC" in ci/lib.sh.

Since we'll need this to run ci/lib.sh it makes sense to move the check to the top of the file. Subsequent commits will add more such assertions to it.

---
 .github/workflows/main.yml |  8 ++++++++
 ci/lib.sh                  | 12 +++++++-----
 2 files changed, 15 insertions(+), 5 deletions(-)
.github/workflows/main.yml +8 -0
index c35200defb9..309b9141249 100644
@@ -79,6 +79,8 @@ jobs:
 
   windows-build:
     name: win build
+    env:
+      jobname: windows-build
     needs: ci-config
     if: needs.ci-config.outputs.enabled == 'yes'
     runs-on: windows-latest
@@ -100,6 +102,8 @@ jobs:
         path: artifacts
   windows-test:
     name: win test
+    env:
+      jobname: windows-test
     runs-on: windows-latest
     needs: [windows-build]
     strategy:
@@ -131,6 +135,8 @@ jobs:
         path: ${{env.FAILED_TEST_ARTIFACTS}}
   vs-build:
     name: win+VS build
+    env:
+      jobname: vs-build
     needs: ci-config
     if: needs.ci-config.outputs.enabled == 'yes'
     env:
@@ -183,6 +189,8 @@ jobs:
         path: artifacts
   vs-test:
     name: win+VS test
+    env:
+      jobname: vs-test
     runs-on: windows-latest
     needs: vs-build
     strategy:
ci/lib.sh +7 -5
index 57141d38a85..6d76c2d406b 100755
@@ -6,6 +6,13 @@
 # and installing dependencies.
 set -ex
 
+# Starting assertions
+if test -z "$jobname"
+then
+	echo "error: must set a CI jobname in the environment" >&2
+	exit 1
+fi
+
 skip_branch_tip_with_tag () {
 	# Sometimes, a branch is pushed at the same time the tag that points
 	# at the same commit as the tip of the branch is pushed, and building
@@ -146,11 +153,6 @@ test -n "${DONT_SKIP_TAGS-}" ||
 skip_branch_tip_with_tag
 skip_good_tree
 
-if test -z "$jobname"
-then
-	jobname="$CI_OS_NAME-$CC"
-fi
-
 export DEVELOPER=1
 export DEFAULT_TEST_TARGET=prove
 export GIT_TEST_CLONE_2GB=true
Signature · 1 line
2.36.0.879.g3659959fcca
#199 lore raw

PATCH v5 3/29 CI: remove more dead Travis CI support

In reply to Ævar Arnfjörð Bjarmason

Remove code that I missed in 4a6e4b96026 (CI: remove Travis CI support, 2021-11-23). This code was only called from or used by the now-removed .travis.yml, or needed by the Travis CI environment.

For the symlinking in ci/run-build-and-tests.sh: Back when 3c93b829205 (travis-ci: build Git during the 'script' phase, 2018-01-08) and 4b060a4d973 (ci: use a junction on Windows instead of a symlink, 2019-01-27) adjusted this "ln" command, the Windows build would use ci/run-build-and-tests.sh.

As seen in 889cacb6897 (ci: configure GitHub Actions for CI/PR, 2020-04-11) the current windows build uses a different entry point under the GitHub CI, which doesn't use this .prove caching. Namely "ci/run-test-slice.sh".

We can be certain that it's never used in "ci/run-test-slice.sh" because to have a ".prove" file we'd need to use "--state=save", which we only do in the dead Azure codepath in ci/lib.sh. If it were used it would do the wrong thing, because the different test slices would each try to clobber the same "t/.prove" file.

If a subsequent run then used the "--state=failed,slow,save" it would defeat the purpose of "ci/run-test-slice.sh", since all slices would then run all tests. I.e. behavior of prove's "--state" options is to select tests to run from the provided "--state" file, in addition to those specified on the command-line.

For ci/run-docker{,-build}.sh: It was likewise last referenced in the .travis.yml removed in my 4a6e4b96026. The current "dockerized" run in ".github/workflows/main.yml" calls the same entry points as the main "regular" job.

For "ci/print-test-failures.sh" the creation of the tarball added in aea8879a6ac (travis-ci: include the trash directories of failed tests in the trace log, 2018-08-01) has been dead since my 4a6e4b96026, which removed the fall-through case from the "case/esac" statement.

---
 ci/lib.sh                     | 10 ++----
 ci/print-test-failures.sh     | 39 ---------------------
 ci/run-build-and-tests.sh     |  5 ---
 ci/run-docker-build.sh        | 66 -----------------------------------
 ci/run-docker.sh              | 47 -------------------------
 ci/run-test-slice.sh          |  5 ---
 ci/util/extract-trash-dirs.sh | 50 --------------------------
 7 files changed, 2 insertions(+), 220 deletions(-)
 delete mode 100755 ci/run-docker-build.sh
 delete mode 100755 ci/run-docker.sh
 delete mode 100755 ci/util/extract-trash-dirs.sh
ci/lib.sh +2 -8
index 6d76c2d406b..648940da877 100755
@@ -106,9 +106,6 @@ then
 	CI_COMMIT="$BUILD_SOURCEVERSION"
 	CI_JOB_ID="$BUILD_BUILDID"
 	CI_JOB_NUMBER="$BUILD_BUILDNUMBER"
-	CI_OS_NAME="$(echo "$AGENT_OS" | tr A-Z a-z)"
-	test darwin != "$CI_OS_NAME" || CI_OS_NAME=osx
-	CI_REPO_SLUG="$(expr "$BUILD_REPOSITORY_URI" : '.*/\([^/]*/[^/]*\)$')"
 	CC="${CC:-gcc}"
 
 	# use a subdirectory of the cache dir (because the file share is shared
@@ -118,16 +115,13 @@ then
 	export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save"
 	export GIT_TEST_OPTS="--verbose-log -x --write-junit-xml"
 	MAKEFLAGS="$MAKEFLAGS --jobs=10"
-	test windows_nt != "$CI_OS_NAME" ||
+	test Windows_NT != "$AGENT_OS" ||
 	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
 elif test true = "$GITHUB_ACTIONS"
 then
 	CI_TYPE=github-actions
 	CI_BRANCH="$GITHUB_REF"
 	CI_COMMIT="$GITHUB_SHA"
-	CI_OS_NAME="$(echo "$RUNNER_OS" | tr A-Z a-z)"
-	test macos != "$CI_OS_NAME" || CI_OS_NAME=osx
-	CI_REPO_SLUG="$GITHUB_REPOSITORY"
 	CI_JOB_ID="$GITHUB_RUN_ID"
 	CC="${CC:-gcc}"
 	DONT_SKIP_TAGS=t
@@ -137,7 +131,7 @@ then
 	export GIT_PROVE_OPTS="--timer --jobs 10"
 	export GIT_TEST_OPTS="--verbose-log -x"
 	MAKEFLAGS="$MAKEFLAGS --jobs=10"
-	test windows != "$CI_OS_NAME" ||
+	test Windows != "$RUNNER_OS" ||
 	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
 else
 	echo "Could not identify CI type" >&2
ci/print-test-failures.sh +0 -39
index 57277eefcd0..740ebb7c529 100755
@@ -16,15 +16,6 @@ then
 	exit
 fi
 
-case "$jobname" in
-osx-clang|osx-gcc)
-	# base64 in OSX doesn't wrap its output at 76 columns by
-	# default, but prints a single, very long line.
-	base64_opts="-b 76"
-	;;
-esac
-
-combined_trash_size=0
 for TEST_EXIT in test-results/*.exit
 do
 	if [ "$(cat "$TEST_EXIT")" != "0" ]
@@ -42,47 +33,17 @@ do
 		azure-pipelines)
 			mkdir -p failed-test-artifacts
 			mv "$trash_dir" failed-test-artifacts
-			continue
 			;;
 		github-actions)
 			mkdir -p failed-test-artifacts
 			echo "FAILED_TEST_ARTIFACTS=t/failed-test-artifacts" >>$GITHUB_ENV
 			cp "${TEST_EXIT%.exit}.out" failed-test-artifacts/
 			tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir"
-			continue
 			;;
 		*)
 			echo "Unhandled CI type: $CI_TYPE" >&2
 			exit 1
 			;;
 		esac
-		trash_tgz_b64="trash.$test_name.base64"
-		if [ -d "$trash_dir" ]
-		then
-			tar czp "$trash_dir" |base64 $base64_opts >"$trash_tgz_b64"
-
-			trash_size=$(wc -c <"$trash_tgz_b64")
-			if [ $trash_size -gt 1048576 ]
-			then
-				# larger than 1MB
-				echo "$(tput setaf 1)Didn't include the trash directory of '$test_name' in the trace log, it's too big$(tput sgr0)"
-				continue
-			fi
-
-			new_combined_trash_size=$(($combined_trash_size + $trash_size))
-			if [ $new_combined_trash_size -gt 1048576 ]
-			then
-				echo "$(tput setaf 1)Didn't include the trash directory of '$test_name' in the trace log, there is plenty of trash in there already.$(tput sgr0)"
-				continue
-			fi
-			combined_trash_size=$new_combined_trash_size
-
-			# DO NOT modify these two 'echo'-ed strings below
-			# without updating 'ci/util/extract-trash-dirs.sh'
-			# as well.
-			echo "$(tput setaf 1)Start of trash directory of '$test_name':$(tput sgr0)"
-			cat "$trash_tgz_b64"
-			echo "$(tput setaf 1)End of trash directory of '$test_name'$(tput sgr0)"
-		fi
 	fi
 done
ci/run-build-and-tests.sh +0 -5
index 280dda7d285..0a1ec8c2bae 100755
@@ -5,11 +5,6 @@
 
 . ${0%/*}/lib.sh
 
-case "$CI_OS_NAME" in
-windows*) cmd //c mklink //j t\\.prove "$(cygpath -aw "$cache_dir/.prove")";;
-*) ln -s "$cache_dir/.prove" t/.prove;;
-esac
-
 export MAKE_TARGETS="all test"
 
 case "$jobname" in
ci/run-docker-build.sh deleted +0 -66

66 lines not on this page · raw message

ci/run-docker.sh deleted +0 -47

47 lines not on this page · raw message

ci/run-test-slice.sh +0 -5
index f8c2c3106a2..b9a682b4bcd 100755
@@ -5,11 +5,6 @@
 
 . ${0%/*}/lib.sh
 
-case "$CI_OS_NAME" in
-windows*) cmd //c mklink //j t\\.prove "$(cygpath -aw "$cache_dir/.prove")";;
-*) ln -s "$cache_dir/.prove" t/.prove;;
-esac
-
 make --quiet -C t T="$(cd t &&
 	./helper/test-tool path-utils slice-tests "$1" "$2" t[0-9]*.sh |
 	tr '\n' ' ')"
ci/util/extract-trash-dirs.sh deleted +0 -50

50 lines not on this page · raw message

Signature · 1 line
2.36.0.879.g3659959fcca
#200 lore raw

PATCH v5 4/29 CI: remove dead "tree skipping" code

In reply to Ævar Arnfjörð Bjarmason

Remove the code related to skipping CI runs if the tree was successfully tested before. With the GitHub CI this is done with the "ci-config" step, see e76eec35540 (ci: allow per-branch config for GitHub Actions, 2020-05-07) and 7d78d5fc1a9 (ci: skip GitHub workflow runs for already-tested commits/trees, 2020-10-08).

This code hasn't been used since 4a6e4b96026 (CI: remove Travis CI support, 2021-11-23), and before that for the retired Azure pipeline support removed in 6081d3898fe (ci: retire the Azure Pipelines definition, 2020-04-11).

This change is needed because this and subsequent commits are turning "ci/lib.sh" into a dumber library that'll only be tasked with setting variables for CI jobs, or for specific steps within those jobs. Now we don't need to worry about it potentially skipping the run on its own.

This change also removes a subtle potential logic error introduced in 0e7696c64db (ci: disallow directional formatting, 2021-11-04). The "ci/check-directional-formatting.bash" script would have been made to run after the "save_good_tree" in invoked in "ci/run-static-analysis.sh". So if e.g. the "make coccicheck" failed we'd still mark the tree as good.

I.e. the addition of "ci/check-directional-formatting.bash" didn't take into account that the various users of the "save_good_tree" function in ci/*.sh made a hard assumption that they're the only ci/*.sh script that's being run (and they use "set -e").

This is entirely academic since we weren't actually running this code, but is something to be careful of if anyone ever needs to resurrect parts of this.

Finally, the cache_dir="$HOME/none" added in a3f2eec862b (ci/lib:
allow running in GitHub Actions, 2020-04-08) for the GitHub CI can be removed. It appears to have been added to appease the subsequent 'mkdir -p "$cache_dir"', which wasn't needed in GitHub CI.

1. https://lore.kernel.org/git/[email protected]/

---
 ci/lib.sh                 | 91 ---------------------------------------
 ci/run-build-and-tests.sh |  2 -
 ci/run-static-analysis.sh |  2 -
 ci/test-documentation.sh  |  2 -
 4 files changed, 97 deletions(-)
ci/lib.sh +0 -91
index 648940da877..a30ab35d9c2 100755
@@ -13,75 +13,6 @@ then
 	exit 1
 fi
 
-skip_branch_tip_with_tag () {
-	# Sometimes, a branch is pushed at the same time the tag that points
-	# at the same commit as the tip of the branch is pushed, and building
-	# both at the same time is a waste.
-	#
-	# When the build is triggered by a push to a tag, $CI_BRANCH will
-	# have that tagname, e.g. v2.14.0.  Let's see if $CI_BRANCH is
-	# exactly at a tag, and if so, if it is different from $CI_BRANCH.
-	# That way, we can tell if we are building the tip of a branch that
-	# is tagged and we can skip the build because we won't be skipping a
-	# build of a tag.
-
-	if TAG=$(git describe --exact-match "$CI_BRANCH" 2>/dev/null) &&
-		test "$TAG" != "$CI_BRANCH"
-	then
-		echo "$(tput setaf 2)Tip of $CI_BRANCH is exactly at $TAG$(tput sgr0)"
-		exit 0
-	fi
-}
-
-# Save some info about the current commit's tree, so we can skip the build
-# job if we encounter the same tree again and can provide a useful info
-# message.
-save_good_tree () {
-	echo "$(git rev-parse $CI_COMMIT^{tree}) $CI_COMMIT $CI_JOB_NUMBER $CI_JOB_ID" >>"$good_trees_file"
-	# limit the file size
-	tail -1000 "$good_trees_file" >"$good_trees_file".tmp
-	mv "$good_trees_file".tmp "$good_trees_file"
-}
-
-# Skip the build job if the same tree has already been built and tested
-# successfully before (e.g. because the branch got rebased, changing only
-# the commit messages).
-skip_good_tree () {
-	if test true = "$GITHUB_ACTIONS"
-	then
-		return
-	fi
-
-	if ! good_tree_info="$(grep "^$(git rev-parse $CI_COMMIT^{tree}) " "$good_trees_file")"
-	then
-		# Haven't seen this tree yet, or no cached good trees file yet.
-		# Continue the build job.
-		return
-	fi
-
-	echo "$good_tree_info" | {
-		read tree prev_good_commit prev_good_job_number prev_good_job_id
-
-		if test "$CI_JOB_ID" = "$prev_good_job_id"
-		then
-			cat <<-EOF
-			$(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0)
-			This commit has already been built and tested successfully by this build job.
-			To force a re-build delete the branch's cache and then hit 'Restart job'.
-			EOF
-		else
-			cat <<-EOF
-			$(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0)
-			This commit's tree has already been built and tested successfully in build job $prev_good_job_number for commit $prev_good_commit.
-			The log of that build job is available at $SYSTEM_TASKDEFINITIONSURI$SYSTEM_TEAMPROJECT/_build/results?buildId=$prev_good_job_id
-			To force a re-build delete the branch's cache and then hit 'Restart job'.
-			EOF
-		fi
-	}
-
-	exit 0
-}
-
 check_unignored_build_artifacts ()
 {
 	! git ls-files --other --exclude-standard --error-unmatch \
@@ -102,16 +33,8 @@ if test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI"
 then
 	CI_TYPE=azure-pipelines
 	# We are running in Azure Pipelines
-	CI_BRANCH="$BUILD_SOURCEBRANCH"
-	CI_COMMIT="$BUILD_SOURCEVERSION"
-	CI_JOB_ID="$BUILD_BUILDID"
-	CI_JOB_NUMBER="$BUILD_BUILDNUMBER"
 	CC="${CC:-gcc}"
 
-	# use a subdirectory of the cache dir (because the file share is shared
-	# among *all* phases)
-	cache_dir="$HOME/test-cache/$SYSTEM_PHASENAME"
-
 	export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save"
 	export GIT_TEST_OPTS="--verbose-log -x --write-junit-xml"
 	MAKEFLAGS="$MAKEFLAGS --jobs=10"
@@ -120,13 +43,7 @@ then
 elif test true = "$GITHUB_ACTIONS"
 then
 	CI_TYPE=github-actions
-	CI_BRANCH="$GITHUB_REF"
-	CI_COMMIT="$GITHUB_SHA"
-	CI_JOB_ID="$GITHUB_RUN_ID"
 	CC="${CC:-gcc}"
-	DONT_SKIP_TAGS=t
-
-	cache_dir="$HOME/none"
 
 	export GIT_PROVE_OPTS="--timer --jobs 10"
 	export GIT_TEST_OPTS="--verbose-log -x"
@@ -139,14 +56,6 @@ else
 	exit 1
 fi
 
-good_trees_file="$cache_dir/good-trees"
-
-mkdir -p "$cache_dir"
-
-test -n "${DONT_SKIP_TAGS-}" ||
-skip_branch_tip_with_tag
-skip_good_tree
-
 export DEVELOPER=1
 export DEFAULT_TEST_TARGET=prove
 export GIT_TEST_CLONE_2GB=true
ci/run-build-and-tests.sh +0 -2
index 0a1ec8c2bae..e5a21985b44 100755
@@ -45,5 +45,3 @@ esac
 # start running tests.
 make $MAKE_TARGETS
 check_unignored_build_artifacts
-
-save_good_tree
ci/run-static-analysis.sh +0 -2
index 65bcebda41a..5a87b1cac96 100755
@@ -28,5 +28,3 @@ fi
 
 make hdr-check ||
 exit 1
-
-save_good_tree
ci/test-documentation.sh +0 -2
index de41888430a..9e0652c30dd 100755
@@ -41,5 +41,3 @@ grep '<meta name="generator" content="Asciidoctor ' Documentation/git.html
 
 rm -f stdout.log stderr.log stderr.raw
 check_unignored_build_artifacts
-
-save_good_tree
Signature · 1 line
2.36.0.879.g3659959fcca
#201 lore raw

PATCH v5 5/29 CI: remove unused Azure ci/* code

In reply to Ævar Arnfjörð Bjarmason

Remove Azure-specific code that's been unused since 6081d3898fe (ci:
retire the Azure Pipelines definition, 2020-04-11). As noted in a larger removal of all of the Azure-supporting code in [1] (although that missed some of this) there is more of it in-tree, but let's focus on only the ci/* code for now.

This is needed because in subsequent commits this unused code would either need to be changed to accommodate changes in ci/*, or be removed.

As we'll see in those subsequent commits the end-state we're heading towards will actually make it easier to add new CI types in the future, even though the only one we're left with now is the GitHub CI. I.e. the "ci/*" framework will be made to do less, not more. We'll be offloading more of what it does to our generic build and test system.

While I'm at it (since the line needs to be touched anyway) change an odd 'if test true == "$GITHUB_ACTIONS"' to the more usual style used in other places of 'if test "$GITHUB_ACTIONS" = "true"'.

1. https://lore.kernel.org/git/[email protected]/

---
 ci/install-dependencies.sh |  3 ---
 ci/lib.sh                  | 13 +------------
 ci/mount-fileshare.sh      | 25 -------------------------
 ci/print-test-failures.sh  |  4 ----
 4 files changed, 1 insertion(+), 44 deletions(-)
 delete mode 100755 ci/mount-fileshare.sh
ci/install-dependencies.sh +0 -3
index dbcebad2fb2..e7ea8799411 100755
@@ -34,8 +34,6 @@ macos-latest)
 	export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1
 	# Uncomment this if you want to run perf tests:
 	# brew install gnu-time
-	test -z "$BREW_INSTALL_PACKAGES" ||
-	brew install $BREW_INSTALL_PACKAGES
 	brew link --force gettext
 	brew install --cask --no-quarantine perforce || {
 		# Update the definitions and try again
@@ -69,7 +67,6 @@ Documentation)
 	sudo apt-get -q update
 	sudo apt-get -q -y install asciidoc xmlto docbook-xsl-ns make
 
-	test -n "$ALREADY_HAVE_ASCIIDOCTOR" ||
 	sudo gem install --version 1.5.8 asciidoctor
 	;;
 linux-gcc-default)
ci/lib.sh +1 -12
index a30ab35d9c2..a6cdcfa014b 100755
@@ -29,18 +29,7 @@ export TERM=${TERM:-dumb}
 # Clear MAKEFLAGS that may come from the outside world.
 export MAKEFLAGS=
 
-if test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI"
-then
-	CI_TYPE=azure-pipelines
-	# We are running in Azure Pipelines
-	CC="${CC:-gcc}"
-
-	export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save"
-	export GIT_TEST_OPTS="--verbose-log -x --write-junit-xml"
-	MAKEFLAGS="$MAKEFLAGS --jobs=10"
-	test Windows_NT != "$AGENT_OS" ||
-	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
-elif test true = "$GITHUB_ACTIONS"
+if test "$GITHUB_ACTIONS" = "true"
 then
 	CI_TYPE=github-actions
 	CC="${CC:-gcc}"
ci/mount-fileshare.sh deleted +0 -25
deleted file mode 100755
index 26b58a80960..00000000000
@@ -1,25 +0,0 @@
-#!/bin/sh
-
-die () {
-	echo "$*" >&2
-	exit 1
-}
-
-test $# = 4 ||
-die "Usage: $0 <share> <username> <password> <mountpoint>"
-
-mkdir -p "$4" || die "Could not create $4"
-
-case "$(uname -s)" in
-Linux)
-	sudo mount -t cifs -o vers=3.0,username="$2",password="$3",dir_mode=0777,file_mode=0777,serverino "$1" "$4"
-	;;
-Darwin)
-	pass="$(echo "$3" | sed -e 's/\//%2F/g' -e 's/+/%2B/g')" &&
-	mount -t smbfs,soft "smb://$2:$pass@${1#//}" "$4"
-	;;
-*)
-	die "No support for $(uname -s)"
-	;;
-esac ||
-die "Could not mount $4"
ci/print-test-failures.sh +0 -4
index 740ebb7c529..47e2ea1ad10 100755
@@ -30,10 +30,6 @@ do
 		test_name="${test_name##*/}"
 		trash_dir="trash directory.$test_name"
 		case "$CI_TYPE" in
-		azure-pipelines)
-			mkdir -p failed-test-artifacts
-			mv "$trash_dir" failed-test-artifacts
-			;;
 		github-actions)
 			mkdir -p failed-test-artifacts
 			echo "FAILED_TEST_ARTIFACTS=t/failed-test-artifacts" >>$GITHUB_ENV
Signature · 1 line
2.36.0.879.g3659959fcca
#202 lore raw

PATCH v5 6/29 CI/lib.sh: stop adding leading whitespace to $MAKEFLAGS

In reply to Ævar Arnfjörð Bjarmason

In a8c51f77d16 (ci: clear and mark MAKEFLAGS exported just once, 2019-02-07) the assignment of MAKEFLAGS was done up-front to future-proof this code for the bug fixed in 406f93ae488 (ci: make sure we build Git parallel, 2019-02-07).

But since 4a6e4b96026 (CI: remove Travis CI support, 2021-11-23) the --jobs=10 has been an invariant here, so let's assign --jobs=$NPROC to the MAKEFLAGS right away, this gets rid of the leading whitespace from the $MAKEFLAGS variable, as it would previously be empty on the first assignment.

---
 ci/lib.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)
ci/lib.sh +5 -3
index a6cdcfa014b..edf26984aeb 100755
@@ -26,17 +26,19 @@ check_unignored_build_artifacts ()
 # GitHub Action doesn't set TERM, which is required by tput
 export TERM=${TERM:-dumb}
 
+# How many jobs to run in parallel?
+NPROC=10
+
 # Clear MAKEFLAGS that may come from the outside world.
-export MAKEFLAGS=
+export MAKEFLAGS=--jobs=$NPROC
 
 if test "$GITHUB_ACTIONS" = "true"
 then
 	CI_TYPE=github-actions
 	CC="${CC:-gcc}"
 
-	export GIT_PROVE_OPTS="--timer --jobs 10"
+	export GIT_PROVE_OPTS="--timer --jobs $NPROC"
 	export GIT_TEST_OPTS="--verbose-log -x"
-	MAKEFLAGS="$MAKEFLAGS --jobs=10"
 	test Windows != "$RUNNER_OS" ||
 	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
 else
Signature · 1 line
2.36.0.879.g3659959fcca
#203 lore raw

PATCH v5 7/29 CI: don't have "git grep" invoke a pager in tree content check

In reply to Ævar Arnfjörð Bjarmason

Fix the omission of not adding "-P" (a.k.a. "--no-pager") to the "git" invocation that spaws "git grep", added in 0e7696c64db (ci: disallow directional formatting, 2021-11-04).

This didn't matter for the GitHub CI, but does when running the check locally. A subsequent commit will start invoking it via a new check from the top-level Makefile, and without this we'd visibly spawn a pager when doing so.

---
 ci/check-directional-formatting.bash | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
ci/check-directional-formatting.bash +1 -1
index e6211b141aa..de8e6e486f0 100755
@@ -23,5 +23,5 @@
 # U+2066..U+2069: LRI, RLI, FSI and PDI
 regex='(\u202a|\u202b|\u202c|\u202d|\u202e|\u2066|\u2067|\u2068|\u2069)'
 
-! LC_CTYPE=C git grep -El "$(LC_CTYPE=C.UTF-8 printf "$regex")" \
+! LC_CTYPE=C git -P grep -El "$(LC_CTYPE=C.UTF-8 printf "$regex")" \
 	-- ':(exclude,attr:binary)' ':(exclude)*.po'
Signature · 1 line
2.36.0.879.g3659959fcca
#204 lore raw

PATCH v5 8/29 CI: have "static-analysis" run a "make ci-static-analysis" target

In reply to Ævar Arnfjörð Bjarmason

Make it easier to run the CI logic locally by creating "ci-static-analysis" target.

Now the "check-coccicheck" logic added in 0860a7641ba (travis-ci: fail if Coccinelle static analysis found something to transform, 2018-07-23) can be easily tested outside of CI, and likewise the sanity check added in 0e7696c64db (ci: disallow directional formatting, 2021-11-04).

These targets could be improved, the "hdr-check" target will re-do all of its work every time it's run, and could with a change similar to my c234e8a0ecf (Makefile: make the "sparse" target non-.PHONY, 2021-09-23). The same goes for the new "ci-check-directional-formatting" target.

But without those improvements being able to easily run a 1=1 equivalent of the checks we do in CI during a local build is already an improvement. Subsequent commits will expand on this ability.

This change will also make the CI check faster, since we can take advantage of parallelism. The "make coccicheck" takes a very long time, and at the tail end we might only have one job pegging 100% of one CPU core for a long time. Now any extra cores will be free to run the rest of the targets under "ci-static-analysis".

---
 .github/workflows/main.yml |  3 +--
 Makefile                   | 28 ++++++++++++++++++++++++++++
 ci/run-static-analysis.sh  | 30 ------------------------------
 shared.mak                 |  1 +
 4 files changed, 30 insertions(+), 32 deletions(-)
 delete mode 100755 ci/run-static-analysis.sh
.github/workflows/main.yml +1 -2
index 309b9141249..ad752010102 100644
@@ -317,8 +317,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/run-static-analysis.sh
-    - run: ci/check-directional-formatting.bash
+    - run: make ci-static-analysis
   sparse:
     needs: ci-config
     if: needs.ci-config.outputs.enabled == 'yes'
Makefile +28 -0
index f8bccfab5e9..8717ea7e745 100644
@@ -3031,6 +3031,20 @@ coccicheck: $(addsuffix .patch,$(filter-out %.pending.cocci,$(wildcard contrib/c
 # See contrib/coccinelle/README
 coccicheck-pending: $(addsuffix .patch,$(wildcard contrib/coccinelle/*.pending.cocci))
 
+.PHONY: check-coccicheck
+check-coccicheck: coccicheck
+	$(QUIET_CHECK)for cocci_patch in contrib/coccinelle/*.patch; do \
+		if test -s "$$cocci_patch"; then \
+			echo "Coccinelle suggests the following changes in '$$cocci_patch':"; \
+			cat "$$cocci_patch"; \
+			fail=UnfortunatelyYes; \
+		fi \
+	done; \
+	if test -n "$$fail"; then \
+		echo "error: Coccinelle suggested some changes"; \
+		exit 1; \
+	fi
+
 .PHONY: coccicheck coccicheck-pending
 
 ### Installation rules
@@ -3466,3 +3480,17 @@ $(FUZZ_PROGRAMS): all
 		$(XDIFF_OBJS) $(EXTLIBS) git.o [email protected] $(LIB_FUZZING_ENGINE) -o $@
 
 fuzz-all: $(FUZZ_PROGRAMS)
+
+### CI "check" targets
+#
+# These targets are run from the CI, see .github/workflows/main.yml,
+# but can also be run manually to run the same assertions locally.
+
+.PHONY: ci-check-directional-formatting
+ci-check-directional-formatting:
+	$(QUIET_CHECK)ci/check-directional-formatting.bash
+
+.PHONY: ci-static-analysis
+ci-static-analysis: ci-check-directional-formatting
+ci-static-analysis: check-coccicheck
+ci-static-analysis: hdr-check
ci/run-static-analysis.sh deleted +0 -30
deleted file mode 100755
index 5a87b1cac96..00000000000
@@ -1,30 +0,0 @@
-#!/bin/sh
-#
-# Perform various static code analysis checks
-#
-
-. ${0%/*}/lib.sh
-
-make coccicheck
-
-set +x
-
-fail=
-for cocci_patch in contrib/coccinelle/*.patch
-do
-	if test -s "$cocci_patch"
-	then
-		echo "$(tput setaf 1)Coccinelle suggests the following changes in '$cocci_patch':$(tput sgr0)"
-		cat "$cocci_patch"
-		fail=UnfortunatelyYes
-	fi
-done
-
-if test -n "$fail"
-then
-	echo "$(tput setaf 1)error: Coccinelle suggested some changes$(tput sgr0)"
-	exit 1
-fi
-
-make hdr-check ||
-exit 1
shared.mak +1 -0
index 50d4596f0d6..83e0aa16e7d 100644
@@ -68,6 +68,7 @@ ifndef V
 	QUIET_HDR      = @echo '   ' HDR $(<:hcc=h);
 	QUIET_RC       = @echo '   ' RC $@;
 	QUIET_SPATCH   = @echo '   ' SPATCH $<;
+	QUIET_CHECK    = @echo '   ' CHECK $@;
 
 ## Used in "Documentation/Makefile"
 	QUIET_ASCIIDOC	= @echo '   ' ASCIIDOC $@;
Signature · 1 line
2.36.0.879.g3659959fcca
#205 lore raw

PATCH v5 9/29 CI: have "static-analysis" run "check-builtins", not "documentation"

In reply to Ævar Arnfjörð Bjarmason

Move the "make check-builtins" check from the "documentation" job to the "static-analysis" job.

The "check-builtins" target added in c74390e4a1d (cherry is built-in, do not ship git-cherry.sh, 2006-11-05) is unrelated to the documentation, so it's odd that b98712b9aa9 (travis-ci: build documentation, 2016-05-04) added it to the "documentation" job.

Let's just move it to the "ci-static-analysis" target, and while we're at it improve the output with $(QUIET_CHECK).

---
 Makefile                 | 3 ++-
 ci/test-documentation.sh | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)
Makefile +2 -1
index 8717ea7e745..2e1ae6c6067 100644
@@ -3402,7 +3402,7 @@ check-docs::
 ### Make sure built-ins do not have dups and listed in git.c
 #
 check-builtins::
-	./check-builtins.sh
+	$(QUIET_CHECK)./check-builtins.sh
 
 ### Test suite coverage testing
 #
@@ -3492,5 +3492,6 @@ ci-check-directional-formatting:
 
 .PHONY: ci-static-analysis
 ci-static-analysis: ci-check-directional-formatting
+ci-static-analysis: check-builtins
 ci-static-analysis: check-coccicheck
 ci-static-analysis: hdr-check
ci/test-documentation.sh +0 -1
index 9e0652c30dd..41e2b126311 100755
@@ -14,7 +14,6 @@ filter_log () {
 	    "$1"
 }
 
-make check-builtins
 make check-docs
 
 # Build docs with AsciiDoc
Signature · 1 line
2.36.0.879.g3659959fcca
#206 lore raw

PATCH v5 10/29 CI: move p4 and git-lfs variables to ci/install-dependencies.sh

In reply to Ævar Arnfjörð Bjarmason

Move the declaration of variables that are only used by the "ubuntu-latest" block in "ci/install-dependencies.sh" there from "ci/lib.sh".

This makes the code easier to follow, and changes "ci/lib.sh" to a library that only exports CI variables for general use, we didn't need to export these $P4_PATH and $GIT_LFS_PATH variables.

---
 ci/install-dependencies.sh | 17 +++++++++++++++--
 ci/lib.sh                  | 12 ------------
 2 files changed, 15 insertions(+), 14 deletions(-)
ci/install-dependencies.sh +15 -2
index e7ea8799411..34b4400f38b 100755
@@ -5,14 +5,27 @@
 
 . ${0%/*}/lib.sh
 
-P4WHENCE=http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION
-LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION
 UBUNTU_COMMON_PKGS="make libssl-dev libcurl4-openssl-dev libexpat-dev
  tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl
  libemail-valid-perl libio-socket-ssl-perl libnet-smtp-ssl-perl"
 
 case "$runs_on_pool" in
 ubuntu-latest)
+	# The Linux build installs the defined dependency versions below.
+	# The OS X build installs much more recent versions, whichever
+	# were recorded in the Homebrew database upon creating the OS X
+	# image.
+	# Keep that in mind when you encounter a broken OS X build!
+	LINUX_P4_VERSION="16.2"
+	LINUX_GIT_LFS_VERSION="1.5.2"
+
+	P4_PATH="$HOME/custom/p4"
+	GIT_LFS_PATH="$HOME/custom/git-lfs"
+	export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH"
+
+	P4WHENCE=http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION
+	LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION
+
 	sudo apt-get -q update
 	sudo apt-get -q -y install language-pack-is libsvn-perl apache2 \
 		$UBUNTU_COMMON_PKGS $CC_PACKAGE
ci/lib.sh +0 -12
index edf26984aeb..3ceb36c7058 100755
@@ -67,18 +67,6 @@ ubuntu-latest)
 	fi
 
 	export GIT_TEST_HTTPD=true
-
-	# The Linux build installs the defined dependency versions below.
-	# The OS X build installs much more recent versions, whichever
-	# were recorded in the Homebrew database upon creating the OS X
-	# image.
-	# Keep that in mind when you encounter a broken OS X build!
-	export LINUX_P4_VERSION="16.2"
-	export LINUX_GIT_LFS_VERSION="1.5.2"
-
-	P4_PATH="$HOME/custom/p4"
-	GIT_LFS_PATH="$HOME/custom/git-lfs"
-	export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH"
 	;;
 macos-latest)
 	if [ "$jobname" = osx-gcc ]
Signature · 1 line
2.36.0.879.g3659959fcca
#207 lore raw

PATCH 1/1 fixup! CI: move p4 and git-lfs variables to ci/install-dependencies.sh

Once the code was moved into an ubuntu specific branch, it doesn't make sense anymore to refer to logic that only applies to a different branch

Cleanup the comment to allow for the implementation to change without making it stale, and while at it rename the variable used to track the P4 version to something slightly less confusing.

---
 ci/install-dependencies.sh | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)
ci/install-dependencies.sh +3 -5
index 098baeef3fb..7bea45fd339 100755
@@ -23,11 +23,9 @@ esac
 case "$runs_on_pool" in
 ubuntu-latest)
 	# The Linux build installs the defined dependency versions below.
-	# The OS X build installs much more recent versions, whichever
-	# were recorded in the Homebrew database upon creating the OS X
-	# image.
+	# The OS X build installs much more recent versions
 	# Keep that in mind when you encounter a broken OS X build!
-	LINUX_P4_VERSION="16.2"
+	P4_AT_LINUX_VERSION="16.2"
 	LINUX_GIT_LFS_VERSION="1.5.2"
 
 	P4_PATH="$HOME/custom/p4"
@@ -39,7 +37,7 @@ ubuntu-latest)
 		echo "$PATH" >>"$GITHUB_PATH"
 	fi
 
-	P4WHENCE=http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION
+	P4WHENCE=http://filehost.perforce.com/perforce/r$P4_AT_LINUX_VERSION
 	LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION
 
 	sudo apt-get -q update
Signature · 1 line
2.36.0.266.g59f845bde02
#208 lore raw

Re: [PATCH] fixup! CI: move p4 and git-lfs variables to ci/install-dependencies.sh

Quoted 11 lines · Carlo Marcelo Arenas Belón

Once the code was moved into an ubuntu specific branch, it doesn't make sense anymore to refer to logic that only applies to a different branch

Cleanup the comment to allow for the implementation to change without making it stale, and while at it rename the variable used to track the P4 version to something slightly less confusing.

---
 ci/install-dependencies.sh | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

OK. The place we run wget from is

    $P4WHENCE/bin.linux26x86_64/{p4,p4d}

for Linux. And P4WHENCE is defined to be

http://{filehost,cdist2}.perforce.com/perforce/r$LINUX_P4_VERSION

Unfortunately, what we grab are not these two files for macOS.

https://cdist2.perforce.com/perforce/r21.2/bin.macosx1015x86_64/helix-core-server.tgz

If p4 and p4d were the only files that we grabbed for macOS, then what I wanted to suggest in the previous message would have worked, i.e.

We define, per $runs_on_pool, two variables:

     P4VERSION (e.g. "r16.2", "r21.2")
     P4ARCH (e.g. "bin.macosx1015x86_64", "bin.linux26x86_64")

Then a single definition, after the "case $runs_on_pool ... esac", of a leading URL, i.e.

     P4DIST=https://cdist2.perforce.com/perforce/$P4VERSION/$P4ARCH/

can be used to help two wget commands, i.e.

     wget --quiet "$P4DIST/p4"
     wget --quiet "$P4DIST/p4d"

in install-dependencies.sh for both Linux-x86_64 and macOS.

But if we cannot unify the way the installation is done for macOS and Linux that way, I do not think renaming $LINUX_P4_VERSION to another name is worth doing (notice that the point of the above exercise was to abstract these into just two variables, VERSION and ARCH).

By the way, I spelunked https://filehost.perforce.com/perforce/ from a browser to see how the files are layed out over there, not cdist2. I am guessing that "filehost" is the name they want end-users like us to use when downloading what they distribute?

Quoted 27 lines
ci/install-dependencies.sh +3 -5
index 098baeef3fb..7bea45fd339 100755
@@ -23,11 +23,9 @@ esac
 case "$runs_on_pool" in
 ubuntu-latest)
 	# The Linux build installs the defined dependency versions below.
-	# The OS X build installs much more recent versions, whichever
-	# were recorded in the Homebrew database upon creating the OS X
-	# image.
+	# The OS X build installs much more recent versions
 	# Keep that in mind when you encounter a broken OS X build!
-	LINUX_P4_VERSION="16.2"
+	P4_AT_LINUX_VERSION="16.2"
 	LINUX_GIT_LFS_VERSION="1.5.2"
 
 	P4_PATH="$HOME/custom/p4"
@@ -39,7 +37,7 @@ ubuntu-latest)
 		echo "$PATH" >>"$GITHUB_PATH"
 	fi
 
-	P4WHENCE=http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION
+	P4WHENCE=http://filehost.perforce.com/perforce/r$P4_AT_LINUX_VERSION
 	LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION
 
 	sudo apt-get -q update
#209 lore raw

Re: [PATCH] fixup! CI: move p4 and git-lfs variables to ci/install-dependencies.sh

Quoted 3 lines · Junio C Hamano

If p4 and p4d were the only files that we grabbed for macOS, then what I wanted to suggest in the previous message would have worked,

true, but it can still work by making them both use the same version (which would be likely a CI variable) and pull the three files from that directory (which was my end objective as shown) in :

https://github.com/carenas/git/commit/4fd79fa1770bc9e8e57074dbac4e44de173aeb75

Quoted 4 lines

By the way, I spelunked https://filehost.perforce.com/perforce/ from a browser to see how the files are layed out over there, not cdist2. I am guessing that "filehost" is the name they want end-users like us to use when downloading what they distribute?

I presume that was the URL they were using when this code was written, FWIW brew uses cdist2, and that is what I get after going to their website at the end after clicking through their download forms so I assume it is less likely to break as well.

agree though that renaming the variable could be avoided but it looks nicer as an intermediate state, and we might even keep it if having two different versions is preferred :

https://github.com/carenas/git/commit/b6699fa3f082ef6d0e8c32718ce5142612003de8

Carlo

#210 lore raw

Re: [PATCH] fixup! CI: move p4 and git-lfs variables to ci/install-dependencies.sh

In reply to Junio C Hamano
Quoted 5 lines · Junio C Hamano

But if we cannot unify the way the installation is done for macOS and Linux that way, I do not think renaming $LINUX_P4_VERSION to another name is worth doing (notice that the point of the above exercise was to abstract these into just two variables, VERSION and ARCH).

BTW, I was intentionally holding those changes for the same reason; if renaming the variable causes any pain (my guess was that it will be unlikely since the code got moved already at that point, and so renaming it at the same time would be unlikely to cause further conflicts) then it shouldn't be done now (or maybe ever).

and those changes should only be done after all the refactoring going on now is gone, and only if discussed first with a lengthy period of RFC to make sure we get it right and go quickly.

Of course, the final version of it will be unlikely to NOT have a single function that does the right thing based on those two variables and those patches were ever going to be sent as RFC.

Carlo

#211 lore raw

PATCH v5 11/29 CI: consistently use "export" in ci/lib.sh

In reply to Ævar Arnfjörð Bjarmason

Change the "ci/lib.sh" script co consistently use "export", for e.g. MAKEFLAGS we were exporting it, and then assigning to it, let's do it the other way around.

Right now this doesn't matter, since we in
e.g. "ci/install-dependencies.sh" source this file, and don't use something like "env(1)" to retrieve these variables. But in a subsequent commit we'll "export" these variables through a wrapper (to additionally write them to a GitHub CI-specific $GITHUB_ENV file). This change makes that subsequent change easier to read, as it won't need to do any control flow refactoring.

---
 ci/lib.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)
ci/lib.sh +4 -3
index 3ceb36c7058..2e425cb787c 100755
@@ -30,7 +30,7 @@ export TERM=${TERM:-dumb}
 NPROC=10
 
 # Clear MAKEFLAGS that may come from the outside world.
-export MAKEFLAGS=--jobs=$NPROC
+MAKEFLAGS=--jobs=$NPROC
 
 if test "$GITHUB_ACTIONS" = "true"
 then
@@ -38,9 +38,10 @@ then
 	CC="${CC:-gcc}"
 
 	export GIT_PROVE_OPTS="--timer --jobs $NPROC"
-	export GIT_TEST_OPTS="--verbose-log -x"
+	GIT_TEST_OPTS="--verbose-log -x"
 	test Windows != "$RUNNER_OS" ||
 	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
+	export GIT_TEST_OPTS
 else
 	echo "Could not identify CI type" >&2
 	env >&2
@@ -94,4 +95,4 @@ linux-leaks)
 	;;
 esac
 
-MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}"
+export MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}"
Signature · 1 line
2.36.0.879.g3659959fcca
#212 lore raw

PATCH v5 12/29 CI: export variables via a wrapper

In reply to Ævar Arnfjörð Bjarmason

Add and use a "setenv" wrapper to export the variables set in ci/lib.sh.

For now this merely adds debugging information about what variables are set, but in a subsequent commit we'll expand this to set variables in $GITHUB_ENV instead of the environment, and thus carry variable settings between steps in a job.

We'll also eventually be interested in what type of variable we're setting, and when it needs to be used, because we'll prominently expose what variables we set in what CI "step", so having only the ones we actually use will improve readability.

To that end call the "setenv" function with optional "--build", "--test" and "--all" parameters to indicate whether they're needed for building, testing or both.

We currently ignore the "varname" parameter, but will make use of it in a subsequent commit. Adding it here reduces the needless churn, as we won't need to change all these lines again for that subsequent change.

---
 ci/lib.sh                 | 51 ++++++++++++++++++++++++++++++---------
 ci/run-build-and-tests.sh | 34 +++++++++++++-------------
 2 files changed, 57 insertions(+), 28 deletions(-)
ci/lib.sh +40 -11
index 2e425cb787c..e36e350e74c 100755
@@ -13,6 +13,35 @@ then
 	exit 1
 fi
 
+# Helper functions
+setenv () {
+	while test $# != 0
+	do
+		case "$1" in
+		--build)
+			;;
+		--test)
+			;;
+		--all)
+			;;
+		-*)
+			echo "BUG: bad setenv() option '$1'" >&2
+			exit 1
+			;;
+		*)
+			break
+			;;
+		esac
+		shift
+	done
+
+	key=$1
+	val=$2
+	shift 2
+
+	eval "export $key=\"$val\""
+}
+
 check_unignored_build_artifacts ()
 {
 	! git ls-files --other --exclude-standard --error-unmatch \
@@ -24,7 +53,7 @@ check_unignored_build_artifacts ()
 }
 
 # GitHub Action doesn't set TERM, which is required by tput
-export TERM=${TERM:-dumb}
+setenv TERM ${TERM:-dumb}
 
 # How many jobs to run in parallel?
 NPROC=10
@@ -37,21 +66,21 @@ then
 	CI_TYPE=github-actions
 	CC="${CC:-gcc}"
 
-	export GIT_PROVE_OPTS="--timer --jobs $NPROC"
+	setenv --test GIT_PROVE_OPTS "--timer --jobs $NPROC"
 	GIT_TEST_OPTS="--verbose-log -x"
 	test Windows != "$RUNNER_OS" ||
 	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
-	export GIT_TEST_OPTS
+	setenv --test GIT_TEST_OPTS "$GIT_TEST_OPTS"
 else
 	echo "Could not identify CI type" >&2
 	env >&2
 	exit 1
 fi
 
-export DEVELOPER=1
-export DEFAULT_TEST_TARGET=prove
-export GIT_TEST_CLONE_2GB=true
-export SKIP_DASHED_BUILT_INS=YesPlease
+setenv --build DEVELOPER 1
+setenv --test DEFAULT_TEST_TARGET prove
+setenv --test GIT_TEST_CLONE_2GB true
+setenv --build SKIP_DASHED_BUILT_INS YesPlease
 
 case "$runs_on_pool" in
 ubuntu-latest)
@@ -67,7 +96,7 @@ ubuntu-latest)
 		MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python2"
 	fi
 
-	export GIT_TEST_HTTPD=true
+	setenv --test GIT_TEST_HTTPD true
 	;;
 macos-latest)
 	if [ "$jobname" = osx-gcc ]
@@ -90,9 +119,9 @@ linux-musl)
 	MAKEFLAGS="$MAKEFLAGS GIT_TEST_UTF8_LOCALE=C.UTF-8"
 	;;
 linux-leaks)
-	export SANITIZE=leak
-	export GIT_TEST_PASSING_SANITIZE_LEAK=true
+	setenv --build SANITIZE leak
+	setenv --test GIT_TEST_PASSING_SANITIZE_LEAK true
 	;;
 esac
 
-export MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}"
+setenv --all MAKEFLAGS "$MAKEFLAGS CC=${CC:-cc}"
ci/run-build-and-tests.sh +17 -17
index e5a21985b44..35d45a9373e 100755
@@ -9,33 +9,33 @@ export MAKE_TARGETS="all test"
 
 case "$jobname" in
 linux-gcc)
-	export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
+	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME main
 	;;
 linux-TEST-vars)
-	export GIT_TEST_SPLIT_INDEX=yes
-	export GIT_TEST_MERGE_ALGORITHM=recursive
-	export GIT_TEST_FULL_IN_PACK_ARRAY=true
-	export GIT_TEST_OE_SIZE=10
-	export GIT_TEST_OE_DELTA_SIZE=5
-	export GIT_TEST_COMMIT_GRAPH=1
-	export GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS=1
-	export GIT_TEST_MULTI_PACK_INDEX=1
-	export GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=1
-	export GIT_TEST_ADD_I_USE_BUILTIN=1
-	export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master
-	export GIT_TEST_WRITE_REV_INDEX=1
-	export GIT_TEST_CHECKOUT_WORKERS=2
+	setenv --test GIT_TEST_SPLIT_INDEX yes
+	setenv --test GIT_TEST_MERGE_ALGORITHM recursive
+	setenv --test GIT_TEST_FULL_IN_PACK_ARRAY true
+	setenv --test GIT_TEST_OE_SIZE 10
+	setenv --test GIT_TEST_OE_DELTA_SIZE 5
+	setenv --test GIT_TEST_COMMIT_GRAPH 1
+	setenv --test GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS 1
+	setenv --test GIT_TEST_MULTI_PACK_INDEX 1
+	setenv --test GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP 1
+	setenv --test GIT_TEST_ADD_I_USE_BUILTIN 1
+	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME master
+	setenv --test GIT_TEST_WRITE_REV_INDEX 1
+	setenv --test GIT_TEST_CHECKOUT_WORKERS 2
 	;;
 linux-clang)
-	export GIT_TEST_DEFAULT_HASH=sha1
+	setenv --test GIT_TEST_DEFAULT_HASH sha1
 	;;
 linux-sha256)
-	export GIT_TEST_DEFAULT_HASH=sha256
+	setenv --test GIT_TEST_DEFAULT_HASH sha256
 	;;
 pedantic)
 	# Don't run the tests; we only care about whether Git can be
 	# built.
-	export DEVOPTS=pedantic
+	setenv --build DEVOPTS pedantic
 	export MAKE_TARGETS=all
 	;;
 esac
Signature · 1 line
2.36.0.879.g3659959fcca
#213 lore raw

PATCH v5 13/29 CI: remove "run-build-and-tests.sh", run "make [test]" directly

In reply to Ævar Arnfjörð Bjarmason

Remove the already thin "ci/run-build-and-tests.sh" wrapper and instead make the CI run "make" or "make test" directly. By doing this we'll be able to easily see at a glance whether our failure was in the compilation or testing, whether that's via human eyes or improve machine readability.

We also need to run our new "ci/check-unignored-build-artifacts.sh" on success() in the CI now, just like we already had a step conditional on failure() running ci/print-test-failures.sh.

The reason we used a "ci/run-build-and-tests.sh" wrapper in the first place had to do with Travis CI-specific constraints that no longer apply to us, as the Travis CI support has been removed.

Instead we can configure the CI in an earlier step by running "ci/lib.sh", which under GitHub CI will write the environment variables we need to the "$GITHUB_ENV" file.

We'll then have access to them in subsequent steps, and crucially those variables will be prominently visible at the start of each step via an expandable drop-down in the UI.drop-do.

I.e. this changes the CI run from a top-down flow like (pseudocode):

- job:
- step1:
- use ci/lib.sh to set env vars
- run a script like ci/run-build-and-tests.sh
- step2:
- if: failure()
- use ci/lib.sh to set env vars
- run ci/print-test-failures.sh

To:

- job:
- step1:
- set variables in $GITHUB_ENV using ci/lib.sh
- step2:
- make
- step3:
- make test
- step4:
- if: failure()
- run ci/print-test-failures.sh
- step5:
- if: success()
- run ci/check-unignored-build-artifacts.sh

There is a proposal[2] to get some of the benefits of this approach by not re-arranging our variable setup in this way, but to instead use the GitHub CI grouping syntax to focus on the relevant parts of "make" or "make test" when we have failures.

Doing it this way makes for better looking GitHub CI UI, and lays much better ground work for our CI going forward. Because:

* The CI logic will be more portable to a future CI system, since a common feature of them is to run various commands in sequence, but a future system won't necessarily support the GitHub-specifics syntax of "grouping" output within a "step".

Even if those systems don't support a "$GITHUB_ENV" emulating will be much easier than to deal with some CI-specific grouping syntax.

* At the start of every step the GitHub CI presents an expandable list of environment variables from "$GITHUB_ENV". We'll now see exactly what variables affected that step (although we currently overshoot that a bit, and always define all variables).

* CI failures will be easier to reproduce locally, as this makes the relevant ci/* scripts something that sets up our environment, but leaves "make" and "make test" working as they do locally.

To reproduce a run the user only needs to set the variables discussed in the drop-down above, either manually or by running "ci/lib.sh".

* The output will be less verbose. The "ci/lib.sh" script uses "set -x", and before this e.g. "ci/static-analysis.sh" would start with 40 lines of trace output, culminating in using "export" to export the relevant environment variables.

Now that verbosity is in the earlier "ci/lib.sh" step, and not in any subsequent one. The "make" targets then start out with the relevant non-trace output right away.

* If we do want to use the grouping syntax within a "step" it'll now be easier to do so. It doesn't support nesting, so we'd have to make a choice between using it for e.g. "make" v.s. "make test", or individual test failures. See "sadly" in [3].

1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/
3. https://lore.kernel.org/git/9333ba781b8240f704e739b00d274f8c3d887e39.1643050574.git.gitgitgadget@gmail.com/

---
 .github/workflows/main.yml            | 14 ++++++--
 ci/check-unignored-build-artifacts.sh |  8 +++++
 ci/install-dependencies.sh            |  4 +++
 ci/lib.sh                             | 36 +++++++++++++++++++-
 ci/make-test-artifacts.sh             |  2 --
 ci/run-build-and-tests.sh             | 47 ---------------------------
 ci/run-test-slice.sh                  |  2 --
 ci/test-documentation.sh              |  1 -
 8 files changed, 58 insertions(+), 56 deletions(-)
 create mode 100755 ci/check-unignored-build-artifacts.sh
 delete mode 100755 ci/run-build-and-tests.sh
.github/workflows/main.yml +11 -3
index ad752010102..57b2db06923 100644
@@ -268,7 +268,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/run-build-and-tests.sh
+    - run: ci/lib.sh
+    - run: make
+    - run: make test
+      if: success()
     - run: ci/print-test-failures.sh
       if: failure()
     - name: Upload failed tests' directories
@@ -292,6 +295,7 @@ jobs:
           image: daald/ubuntu32:xenial
         - jobname: pedantic
           image: fedora
+          skip-tests: yes
     env:
       jobname: ${{matrix.vector.jobname}}
     runs-on: ubuntu-latest
@@ -299,9 +303,12 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - run: ci/install-docker-dependencies.sh
-    - run: ci/run-build-and-tests.sh
+    - run: ci/lib.sh
+    - run: make
+    - run: make test
+      if: success() && matrix.vector.skip-tests != 'yes'
     - run: ci/print-test-failures.sh
-      if: failure()
+      if: failure() && matrix.vector.skip-tests != 'yes'
     - name: Upload failed tests' directories
       if: failure() && env.FAILED_TEST_ARTIFACTS != ''
       uses: actions/upload-artifact@v1
@@ -317,6 +324,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
+    - run: ci/lib.sh
     - run: make ci-static-analysis
   sparse:
     needs: ci-config
ci/check-unignored-build-artifacts.sh new file +8 -0
new file mode 100755
index 00000000000..56d04b0db9a
@@ -0,0 +1,8 @@
+#!/bin/sh
+#
+# Check whether the build created anything not in our .gitignore
+#
+
+. ${0%/*}/lib.sh
+
+check_unignored_build_artifacts
ci/install-dependencies.sh +4 -0
index 34b4400f38b..01006379fd0 100755
@@ -22,6 +22,10 @@ ubuntu-latest)
 	P4_PATH="$HOME/custom/p4"
 	GIT_LFS_PATH="$HOME/custom/git-lfs"
 	export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH"
+	if test -n "$GITHUB_PATH"
+	then
+		echo "$PATH" >>"$GITHUB_PATH"
+	fi
 
 	P4WHENCE=http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION
 	LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION
ci/lib.sh +35 -1
index e36e350e74c..43231862aa5 100755
@@ -1,3 +1,5 @@
+#!/bin/sh
+
 # Library of functions shared by all CI scripts
 
 # Set 'exit on error' for all CI scripts to let the caller know that
@@ -39,7 +41,10 @@ setenv () {
 	val=$2
 	shift 2
 
-	eval "export $key=\"$val\""
+	if test -n "$GITHUB_ENV"
+	then
+		echo "$key=$val" >>"$GITHUB_ENV"
+	fi
 }
 
 check_unignored_build_artifacts ()
@@ -109,6 +114,35 @@ macos-latest)
 esac
 
 case "$jobname" in
+linux-gcc)
+	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME main
+	;;
+linux-TEST-vars)
+	setenv --test GIT_TEST_SPLIT_INDEX yes
+	setenv --test GIT_TEST_MERGE_ALGORITHM recursive
+	setenv --test GIT_TEST_FULL_IN_PACK_ARRAY true
+	setenv --test GIT_TEST_OE_SIZE 10
+	setenv --test GIT_TEST_OE_DELTA_SIZE 5
+	setenv --test GIT_TEST_COMMIT_GRAPH 1
+	setenv --test GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS 1
+	setenv --test GIT_TEST_MULTI_PACK_INDEX 1
+	setenv --test GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP 1
+	setenv --test GIT_TEST_ADD_I_USE_BUILTIN 1
+	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME master
+	setenv --test GIT_TEST_WRITE_REV_INDEX 1
+	setenv --test GIT_TEST_CHECKOUT_WORKERS 2
+	;;
+linux-clang)
+	setenv --test GIT_TEST_DEFAULT_HASH sha1
+	;;
+linux-sha256)
+	setenv --test GIT_TEST_DEFAULT_HASH sha256
+	;;
+pedantic)
+	# Don't run the tests; we only care about whether Git can be
+	# built.
+	setenv --build DEVOPTS pedantic
+	;;
 linux32)
 	CC=gcc
 	;;
ci/make-test-artifacts.sh +0 -2
index 646967481f6..45298562982 100755
@@ -8,5 +8,3 @@ mkdir -p "$1" # in case ci/lib.sh decides to quit early
 . ${0%/*}/lib.sh
 
 make artifacts-tar ARTIFACTS_DIRECTORY="$1"
-
-check_unignored_build_artifacts
ci/run-build-and-tests.sh deleted +0 -47

47 lines not on this page · raw message

ci/run-test-slice.sh +0 -2
index b9a682b4bcd..70326961454 100755
@@ -8,5 +8,3 @@
 make --quiet -C t T="$(cd t &&
 	./helper/test-tool path-utils slice-tests "$1" "$2" t[0-9]*.sh |
 	tr '\n' ' ')"
-
-check_unignored_build_artifacts
ci/test-documentation.sh +0 -1
index 41e2b126311..6b5cce03bd7 100755
@@ -39,4 +39,3 @@ test -s Documentation/git.html
 grep '<meta name="generator" content="Asciidoctor ' Documentation/git.html
 
 rm -f stdout.log stderr.log stderr.raw
-check_unignored_build_artifacts
Signature · 1 line
2.36.0.879.g3659959fcca
#214 lore raw

PATCH v5 14/29 CI: make ci/{lib,install-dependencies}.sh POSIX-compatible

In reply to Ævar Arnfjörð Bjarmason

Change the unportable constructs in the "ci/install-dependencies.sh" scripts to use POSIX replacements, and have it run under "/bin/sh" instead of "/usr/bin/env bash". We needed to:

* Replace pushd/popd with a sub-shell.
* Change "mkdir --parents" to "mkdir -p".
* Use POSIX "export" syntax.

This is in preparation for combining this script and "ci/install-docker-dependencies.sh", which doesn't have access to "bash".

While we're at it change "if [" to "if test" in ci/lib.sh. This isn't strictly speaking a case of POSIX-incompatibility, but that syntax was being used there because this code was using idiomatic bash constructs. Let's have it use our usual style of "if test" instead.

---
 ci/install-dependencies.sh | 27 +++++++++++++++++----------
 ci/lib.sh                  |  4 ++--
 2 files changed, 19 insertions(+), 12 deletions(-)
ci/install-dependencies.sh +17 -10
index 01006379fd0..1ad4dd7db39 100755
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh
 #
 # Install dependencies required to build and test Git on Linux and macOS
 #
@@ -21,7 +21,8 @@ ubuntu-latest)
 
 	P4_PATH="$HOME/custom/p4"
 	GIT_LFS_PATH="$HOME/custom/git-lfs"
-	export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH"
+	PATH="$GIT_LFS_PATH:$P4_PATH:$PATH"
+	export PATH
 	if test -n "$GITHUB_PATH"
 	then
 		echo "$PATH" >>"$GITHUB_PATH"
@@ -33,22 +34,28 @@ ubuntu-latest)
 	sudo apt-get -q update
 	sudo apt-get -q -y install language-pack-is libsvn-perl apache2 \
 		$UBUNTU_COMMON_PKGS $CC_PACKAGE
-	mkdir --parents "$P4_PATH"
-	pushd "$P4_PATH"
+	mkdir -p "$P4_PATH"
+	(
+		cd "$P4_PATH"
 		wget --quiet "$P4WHENCE/bin.linux26x86_64/p4d"
 		wget --quiet "$P4WHENCE/bin.linux26x86_64/p4"
 		chmod u+x p4d
 		chmod u+x p4
-	popd
-	mkdir --parents "$GIT_LFS_PATH"
-	pushd "$GIT_LFS_PATH"
+	)
+	mkdir -p "$GIT_LFS_PATH"
+	(
+		cd "$GIT_LFS_PATH"
 		wget --quiet "$LFSWHENCE/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz"
 		tar --extract --gunzip --file "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz"
 		cp git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs .
-	popd
+	)
 	;;
 macos-latest)
-	export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1
+	HOMEBREW_NO_AUTO_UPDATE=1
+	export HOMEBREW_NO_AUTO_UPDATE
+	HOMEBREW_NO_INSTALL_CLEANUP=1
+	export HOMEBREW_NO_INSTALL_CLEANUP
+
 	# Uncomment this if you want to run perf tests:
 	# brew install gnu-time
 	brew link --force gettext
@@ -62,7 +69,7 @@ macos-latest)
 
 	if test -n "$CC_PACKAGE"
 	then
-		BREW_PACKAGE=${CC_PACKAGE/-/@}
+		BREW_PACKAGE=$(echo $CC_PACKAGE | tr '-' '@')
 		brew install "$BREW_PACKAGE"
 		brew link "$BREW_PACKAGE"
 	fi
ci/lib.sh +2 -2
index 43231862aa5..2fd022c94bc 100755
@@ -94,7 +94,7 @@ ubuntu-latest)
 		break
 	fi
 
-	if [ "$jobname" = linux-gcc ]
+	if test "$jobname" = linux-gcc
 	then
 		MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python3"
 	else
@@ -104,7 +104,7 @@ ubuntu-latest)
 	setenv --test GIT_TEST_HTTPD true
 	;;
 macos-latest)
-	if [ "$jobname" = osx-gcc ]
+	if test "$jobname" = osx-gcc
 	then
 		MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
 	else
Signature · 1 line
2.36.0.879.g3659959fcca
#215 lore raw

PATCH v5 15/29 CI: check ignored unignored build artifacts in "win[+VS] build" too

In reply to Ævar Arnfjörð Bjarmason

Use the newly created "ci/check-unignored-build-artifacts.sh" script in the "win build" and "win+VS build" steps too. These steps weren't doing this sanity check of untracked files against .gitignore before, but as the check passes for them let's keep doing that extra sanity check.

---
 .github/workflows/main.yml | 6 ++++++
 1 file changed, 6 insertions(+)
.github/workflows/main.yml +6 -0
index 57b2db06923..c97fbd0f0b3 100644
@@ -93,6 +93,9 @@ jobs:
         HOME: ${{runner.workspace}}
         NO_PERL: 1
       run: . /etc/profile && ci/make-test-artifacts.sh artifacts
+    - run: ci/check-unignored-build-artifacts.sh
+      if: success()
+      shell: bash
     - name: zip up tracked files
       run: git archive -o artifacts/tracked.tar.gz HEAD
     - name: upload tracked files and build artifacts
@@ -180,6 +183,9 @@ jobs:
       run: |
         mkdir -p artifacts &&
         eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts NO_GETTEXT=YesPlease 2>&1 | grep ^tar)"
+    - run: ci/check-unignored-build-artifacts.sh
+      if: success()
+      shell: bash
     - name: zip up tracked files
       run: git archive -o artifacts/tracked.tar.gz HEAD
     - name: upload tracked files and build artifacts
Signature · 1 line
2.36.0.879.g3659959fcca
#216 lore raw

PATCH v5 16/29 CI: invoke "make artifacts-tar" directly in windows-build

In reply to Ævar Arnfjörð Bjarmason

Change the windows-build job to invoke the "make artifacts-tar" step directly, instead of calling a "ci/make-test-artifacts.sh" script.

The script was needed because "ci/lib.sh" would set up various environment variables for us, but now we can instead use the "ci/lib.sh" in its script mode.

The "mkdir -p" added in b819f1d2cec (ci: parallelize testing on Windows, 2019-01-29) isn't needed, the same commit added that "mkdir -p" to the "artifacts-tar" rule itself, so we can have "make" create the directory for us.

This also has the benefit of making the "build" step less chatty, since it won't start with the verbose "set -x" output, that's now contained in the "ci/lib.sh" step.

---
 .github/workflows/main.yml |  5 ++++-
 ci/make-test-artifacts.sh  | 10 ----------
 2 files changed, 4 insertions(+), 11 deletions(-)
 delete mode 100755 ci/make-test-artifacts.sh
.github/workflows/main.yml +4 -1
index c97fbd0f0b3..ca993afd098 100644
@@ -87,12 +87,15 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
+    - run: ci/lib.sh
+      shell: bash
     - name: build
       shell: bash
       env:
         HOME: ${{runner.workspace}}
         NO_PERL: 1
-      run: . /etc/profile && ci/make-test-artifacts.sh artifacts
+      run: . /etc/profile && make artifacts-tar ARTIFACTS_DIRECTORY=artifacts
+      if: success()
     - run: ci/check-unignored-build-artifacts.sh
       if: success()
       shell: bash
ci/make-test-artifacts.sh deleted +0 -10
deleted file mode 100755
index 45298562982..00000000000
@@ -1,10 +0,0 @@
-#!/bin/sh
-#
-# Build Git and store artifacts for testing
-#
-
-mkdir -p "$1" # in case ci/lib.sh decides to quit early
-
-. ${0%/*}/lib.sh
-
-make artifacts-tar ARTIFACTS_DIRECTORY="$1"
Signature · 1 line
2.36.0.879.g3659959fcca
#217 lore raw

PATCH v5 17/29 CI: split up and reduce "ci/test-documentation.sh"

In reply to Ævar Arnfjörð Bjarmason

Change the "ci/test-documentation.sh" script to run the bash-specific parts in as one command in the CI job itself, and to run the two "make doc" commands at the top-level.

It'll now be obvious from the title of the step if we failed in the asciidoc or asciidoctor step.

Since the "check_unignored_build_artifacts()" function is now only used in "ci/check-unignored-build-artifacts.sh" move that function there.

The recipe for the job in ".github/workflows/main.yml" is now a bit verbose because it's effectively the same job twice, with a "make clean" in-between. It would be better for the verbosity to run it via a matrix as done in the alternate approach in [1] does, but then we'd sacrifice overall CPU time for the brevity. It might still be worth doing, but let's go for this simpler approach for now.

1. https://lore.kernel.org/git/[email protected]/

---
 .github/workflows/main.yml            | 16 +++++++++++++-
 ci/check-unignored-build-artifacts.sh | 10 +++++++++
 ci/lib.sh                             | 10 ---------
 ci/test-documentation.sh              | 31 +++++++++------------------
 4 files changed, 35 insertions(+), 32 deletions(-)
.github/workflows/main.yml +15 -1
index ca993afd098..88eb2ef2dad 100644
@@ -365,4 +365,18 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/test-documentation.sh
+    - run: ci/lib.sh
+    - run: make check-docs
+    - run: "make doc > >(tee stdout.log) 2> >(tee stderr.raw >&2)"
+      shell: bash
+    - run: ci/test-documentation.sh AsciiDoc
+      if: success()
+    - run: ci/check-unignored-build-artifacts.sh
+      if: success()
+    - run: make clean
+    - run: "make USE_ASCIIDOCTOR=1 doc > >(tee stdout.log) 2> >(tee stderr.raw >&2)"
+      shell: bash
+    - run: ci/test-documentation.sh Asciidoctor
+      if: success()
+    - run: ci/check-unignored-build-artifacts.sh
+      if: success()
ci/check-unignored-build-artifacts.sh +10 -0
index 56d04b0db9a..0bc04f32804 100755
@@ -5,4 +5,14 @@
 
 . ${0%/*}/lib.sh
 
+check_unignored_build_artifacts ()
+{
+	! git ls-files --other --exclude-standard --error-unmatch \
+		-- ':/*' 2>/dev/null ||
+	{
+		echo "$(tput setaf 1)error: found unignored build artifacts$(tput sgr0)"
+		false
+	}
+}
+
 check_unignored_build_artifacts
ci/lib.sh +0 -10
index 2fd022c94bc..63c14f504b3 100755
@@ -47,16 +47,6 @@ setenv () {
 	fi
 }
 
-check_unignored_build_artifacts ()
-{
-	! git ls-files --other --exclude-standard --error-unmatch \
-		-- ':/*' 2>/dev/null ||
-	{
-		echo "$(tput setaf 1)error: found unignored build artifacts$(tput sgr0)"
-		false
-	}
-}
-
 # GitHub Action doesn't set TERM, which is required by tput
 setenv TERM ${TERM:-dumb}
 
ci/test-documentation.sh +10 -21
index 6b5cce03bd7..b8a6a6f664e 100755
@@ -1,10 +1,12 @@
-#!/usr/bin/env bash
+#!/bin/sh
 #
-# Perform sanity checks on documentation and build it.
+# Perform sanity checks on "make doc" output and built documentation
 #
 
 . ${0%/*}/lib.sh
 
+generator=$1
+
 filter_log () {
 	sed -e '/^GIT_VERSION = /d' \
 	    -e "/constant Gem::ConfigMap is deprecated/d" \
@@ -14,28 +16,15 @@ filter_log () {
 	    "$1"
 }
 
-make check-docs
-
-# Build docs with AsciiDoc
-make doc > >(tee stdout.log) 2> >(tee stderr.raw >&2)
-cat stderr.raw
-filter_log stderr.raw >stderr.log
-test ! -s stderr.log
-test -s Documentation/git.html
-test -s Documentation/git.xml
-test -s Documentation/git.1
-grep '<meta name="generator" content="AsciiDoc ' Documentation/git.html
-
-rm -f stdout.log stderr.log stderr.raw
-check_unignored_build_artifacts
-
-# Build docs with AsciiDoctor
-make clean
-make USE_ASCIIDOCTOR=1 doc > >(tee stdout.log) 2> >(tee stderr.raw >&2)
 cat stderr.raw
 filter_log stderr.raw >stderr.log
 test ! -s stderr.log
 test -s Documentation/git.html
-grep '<meta name="generator" content="Asciidoctor ' Documentation/git.html
+if test "$generator" = "Asciidoctor"
+then
+	test -s Documentation/git.xml
+	test -s Documentation/git.1
+fi
+grep "<meta name=\"generator\" content=\"$generator " Documentation/git.html
 
 rm -f stdout.log stderr.log stderr.raw
Signature · 1 line
2.36.0.879.g3659959fcca
#218 lore raw

PATCH v5 18/29 CI: combine ci/install{,-docker}-dependencies.sh

In reply to Ævar Arnfjörð Bjarmason

Combine the "ci/install-dependencies.sh" and "ci/install-docker-dependencies.sh" into one script. The "case" statement in the latter only cared about "$jobname", and can be folded into the same "case" statement in the former.

The reason they split up is historical, until a preceding commit "ci/lib.sh" required "bash", which might not have been available in "docker".

This also fixes issue in "ci/install-docker-dependencies.sh" where we'd hide errors due to not using "set -e". Now that we include "ci/lib.sh" we won't have that potential issue.

---
 .github/workflows/main.yml        |  2 +-
 ci/install-dependencies.sh        | 15 +++++++++++++++
 ci/install-docker-dependencies.sh | 22 ----------------------
 3 files changed, 16 insertions(+), 23 deletions(-)
 delete mode 100755 ci/install-docker-dependencies.sh
.github/workflows/main.yml +1 -1
index 88eb2ef2dad..42944020436 100644
@@ -311,7 +311,7 @@ jobs:
     container: ${{matrix.vector.image}}
     steps:
     - uses: actions/checkout@v1
-    - run: ci/install-docker-dependencies.sh
+    - run: ci/install-dependencies.sh
     - run: ci/lib.sh
     - run: make
     - run: make test
ci/install-dependencies.sh +15 -0
index 1ad4dd7db39..adece989491 100755
@@ -97,6 +97,21 @@ linux-gcc-default)
 	sudo apt-get -q update
 	sudo apt-get -q -y install $UBUNTU_COMMON_PKGS
 	;;
+linux32)
+	linux32 --32bit i386 sh -c '
+		apt update >/dev/null &&
+		apt install -y build-essential libcurl4-openssl-dev \
+			libssl-dev libexpat-dev gettext python >/dev/null
+	'
+	;;
+linux-musl)
+	apk add --update build-base curl-dev openssl-dev expat-dev gettext \
+		pcre2-dev python3 musl-libintl perl-utils ncurses >/dev/null
+	;;
+pedantic)
+	dnf -yq update >/dev/null &&
+	dnf -yq install make gcc findutils diffutils perl python3 gettext zlib-devel expat-devel openssl-devel curl-devel pcre2-devel >/dev/null
+	;;
 esac
 
 if type p4d >/dev/null && type p4 >/dev/null
ci/install-docker-dependencies.sh deleted +0 -22
deleted file mode 100755
index 78b7e326da6..00000000000
@@ -1,22 +0,0 @@
-#!/bin/sh
-#
-# Install dependencies required to build and test Git inside container
-#
-
-case "$jobname" in
-linux32)
-	linux32 --32bit i386 sh -c '
-		apt update >/dev/null &&
-		apt install -y build-essential libcurl4-openssl-dev \
-			libssl-dev libexpat-dev gettext python >/dev/null
-	'
-	;;
-linux-musl)
-	apk add --update build-base curl-dev openssl-dev expat-dev gettext \
-		pcre2-dev python3 musl-libintl perl-utils ncurses >/dev/null
-	;;
-pedantic)
-	dnf -yq update >/dev/null &&
-	dnf -yq install make gcc findutils diffutils perl python3 gettext zlib-devel expat-devel openssl-devel curl-devel pcre2-devel >/dev/null
-	;;
-esac
Signature · 1 line
2.36.0.879.g3659959fcca
#219 lore raw

PATCH v5 19/29 CI: move "env" definitions into ci/lib.sh

In reply to Ævar Arnfjörð Bjarmason

Have the GitHub CI jobs use the "ci/lib".sh as a source of truth for environment variables that affect their builds and tests. This brings them in line with the rest of the jobs, and make it easier to see the entire CI configuration per jobname at a glance.

To do this we need to add a ci/lib.sh "step" to the remaining jobs that didn't have it added in preceding commits.

The Makefile parameters are luckily all accepted via the environment, so we can export these instead of adding them to MAKEFLAGS.

Let's also use the documented GIT_CONIFG_* mechanism for setting config instead of the internal GIT_CONFIG_PARAMETERS variable. This adjusts code added in 889cacb6897 (ci: configure GitHub Actions for CI/PR, 2020-04-11), we could probably use the GIT_AUTHOR_NAME etc, but let's keep setting this via config, just with the documented mechanism.

By setting "ARTIFACTS_DIRECTORY=artifacts" once we don't need to repeat it in various places, let's get it from the environment instead.

Finally, the "DEVELOPER: 1" was only needed by the "sparse" job, which wasn't using "ci/lib.sh" (and which sets DEVELOPER=1). Let's instead have the "sparse" job use the "ci/lib.sh" and remove DEVELOPER=1 from ".github/workflows/main.yml".

This substantially speeds up the "sparse" job, since it'll now pick up the "--jobs" setting in MAKEFLAGS that we use everywhere else. Before it ran in around 4m30s, now in around 2m30s.

---
 .github/workflows/main.yml | 29 +++++++++++------------------
 ci/lib.sh                  | 21 +++++++++++++++++++++
 2 files changed, 32 insertions(+), 18 deletions(-)
.github/workflows/main.yml +11 -18
index 42944020436..5f167ac5be6 100644
@@ -2,9 +2,6 @@ name: CI
 
 on: [push, pull_request]
 
-env:
-  DEVELOPER: 1
-
 jobs:
   ci-config:
     name: config
@@ -93,8 +90,7 @@ jobs:
       shell: bash
       env:
         HOME: ${{runner.workspace}}
-        NO_PERL: 1
-      run: . /etc/profile && make artifacts-tar ARTIFACTS_DIRECTORY=artifacts
+      run: . /etc/profile && make artifacts-tar
       if: success()
     - run: ci/check-unignored-build-artifacts.sh
       if: success()
@@ -105,7 +101,7 @@ jobs:
       uses: actions/upload-artifact@v2
       with:
         name: windows-artifacts
-        path: artifacts
+        path: ${{env.ARTIFACTS_DIRECTORY}}
   windows-test:
     name: win test
     env:
@@ -126,6 +122,8 @@ jobs:
       shell: bash
       run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
+    - run: ci/lib.sh
+      shell: bash
     - name: test
       shell: bash
       run: . /etc/profile && ci/run-test-slice.sh ${{matrix.nr}} 10
@@ -145,9 +143,6 @@ jobs:
       jobname: vs-build
     needs: ci-config
     if: needs.ci-config.outputs.enabled == 'yes'
-    env:
-      NO_PERL: 1
-      GIT_CONFIG_PARAMETERS: "'user.name=CI' 'user.email=ci@git'"
     runs-on: windows-latest
     steps:
     - uses: actions/checkout@v2
@@ -171,6 +166,8 @@ jobs:
     - name: copy dlls to root
       shell: cmd
       run: compat\vcbuild\vcpkg_copy_dlls.bat release
+    - run: ci/lib.sh
+      shell: bash
     - name: generate Visual Studio solution
       shell: bash
       run: |
@@ -181,21 +178,16 @@ jobs:
     - name: bundle artifact tar
       shell: bash
       env:
-        MSVC: 1
         VCPKG_ROOT: ${{github.workspace}}\compat\vcbuild\vcpkg
       run: |
-        mkdir -p artifacts &&
-        eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts NO_GETTEXT=YesPlease 2>&1 | grep ^tar)"
-    - run: ci/check-unignored-build-artifacts.sh
-      if: success()
-      shell: bash
+        eval "$(make -n artifacts-tar 2>&1 | grep -e ^mkdir -e ^tar)"
     - name: zip up tracked files
       run: git archive -o artifacts/tracked.tar.gz HEAD
     - name: upload tracked files and build artifacts
       uses: actions/upload-artifact@v2
       with:
         name: vs-artifacts
-        path: artifacts
+        path: ${{env.ARTIFACTS_DIRECTORY}}
   vs-test:
     name: win+VS test
     env:
@@ -216,10 +208,10 @@ jobs:
     - name: extract tracked files and build artifacts
       shell: bash
       run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz
+    - run: ci/lib.sh
+      shell: bash
     - name: test
       shell: bash
-      env:
-        NO_SVN_TESTS: 1
       run: . /etc/profile && ci/run-test-slice.sh ${{matrix.nr}} 10
     - name: ci/print-test-failures.sh
       if: failure()
@@ -354,6 +346,7 @@ jobs:
     - uses: actions/checkout@v2
     - name: Install other dependencies
       run: ci/install-dependencies.sh
+    - run: ci/lib.sh
     - run: make sparse
   documentation:
     name: documentation
ci/lib.sh +21 -0
index 63c14f504b3..cae324e2fd7 100755
@@ -104,6 +104,27 @@ macos-latest)
 esac
 
 case "$jobname" in
+windows-build)
+	setenv --build NO_PERL NoThanks
+	setenv --build ARTIFACTS_DIRECTORY artifacts
+	;;
+vs-build)
+	setenv --build NO_PERL NoThanks
+	setenv --build NO_GETTEXT NoThanks
+	setenv --build ARTIFACTS_DIRECTORY artifacts
+	setenv --build INCLUDE_DLLS_IN_ARTIFACTS YesPlease
+	setenv --build MSVC YesPlease
+
+	setenv --build GIT_CONFIG_COUNT 2
+	setenv --build GIT_CONFIG_KEY_0 user.name
+	setenv --build GIT_CONFIG_VALUE_0 CI
+	setenv --build GIT_CONFIG_KEY_1 user.emailname
+	setenv --build GIT_CONFIG_VALUE_1 ci@git
+	setenv --build GIT_CONFIG_VALUE_1 ci@git
+	;;
+vs-test)
+	setenv --test NO_SVN_TESTS YesPlease
+	;;
 linux-gcc)
 	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME main
 	;;
Signature · 1 line
2.36.0.879.g3659959fcca
#220 lore raw

PATCH v5 20/29 ci/run-test-slice.sh: replace shelling out with "echo"

In reply to Ævar Arnfjörð Bjarmason

Amend this code added in b819f1d2cec (ci: parallelize testing on Windows, 2019-01-29) to stop shelling out to "tr", all we're trying to do here is to turn '\n'-delimited output into a ' '-delimited list. To do that we can just use "echo".

None of these files will contain the odd characters using "echo" might choke on in the general case (i.e. '\'-escapes and the like).

This has the added benefit of not adding a redundant ' ' to the end of the T-parameter, which didn't matter in practice, but as we'll see in a subsequent commit makes for slightly nicer output, as we'll show this variable using a GitHub CI-native feature.

---
 ci/run-test-slice.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
ci/run-test-slice.sh +3 -3
index 70326961454..1de6a18ca47 100755
@@ -5,6 +5,6 @@
 
 . ${0%/*}/lib.sh
 
-make --quiet -C t T="$(cd t &&
-	./helper/test-tool path-utils slice-tests "$1" "$2" t[0-9]*.sh |
-	tr '\n' ' ')"
+tests=$(echo $(cd t && ./helper/test-tool path-utils slice-tests "$1" "$2" \
+	t[0-9]*.sh))
+make --quiet -C t T="$tests"
Signature · 1 line
2.36.0.879.g3659959fcca
#221 lore raw

PATCH v5 21/29 CI: pre-select test slice in Windows & VS tests

In reply to Ævar Arnfjörð Bjarmason

In preceding commits the tests have been changed to do their setup via $GITHUB_ENV in one step, and to then have subsequent steps that re-use that environment.

Let's change the "test slice" tests added in b819f1d2cec (ci:
parallelize testing on Windows, 2019-01-29) to do the same. These tests select 10% of the tests to run in 10 "test slices". Now we'll select those in a step that immediately precedes the testing step, and then simply invoke "make -C t -e".

This has the advantage that the tests to be run are now listed in the standard "Run" drop-down at the start of the "test" step.

Since the "T" variable in "t/Makefile" doesn't normally accept overrides from the environment we need to invoke "make" with the "-e" option (a.k.a. "--environment-overrides"). We could also make $(T) in t/Makefile be a "?=" assigned variable, but this way works, and is arguably clearer as it's more obvious that we're injecting a special list of tests that override the normal behavior of that Makefile.

Note that we cannot run the top-level "make test" here, because of how the Windows CI builds git, i.e. either via CMake or some option that would cause "make test" to recompile git itself. Instead we run "make -C t [...]".

---
 .github/workflows/main.yml                     | 10 ++++++++--
 ci/{run-test-slice.sh => select-test-slice.sh} |  7 +++++--
 2 files changed, 13 insertions(+), 4 deletions(-)
 rename ci/{run-test-slice.sh => select-test-slice.sh} (50%)
.github/workflows/main.yml +8 -2
index 5f167ac5be6..83e0aa1f469 100644
@@ -124,9 +124,12 @@ jobs:
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
     - run: ci/lib.sh
       shell: bash
+    - name: select tests
+      run: . /etc/profile && ci/select-test-slice.sh ${{matrix.nr}} 10
+      shell: bash
     - name: test
       shell: bash
-      run: . /etc/profile && ci/run-test-slice.sh ${{matrix.nr}} 10
+      run: . /etc/profile && make -C t -e
     - name: ci/print-test-failures.sh
       if: failure()
       shell: bash
@@ -210,9 +213,12 @@ jobs:
       run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz
     - run: ci/lib.sh
       shell: bash
+    - name: select tests
+      run: . /etc/profile && ci/select-test-slice.sh ${{matrix.nr}} 10
+      shell: bash
     - name: test
       shell: bash
-      run: . /etc/profile && ci/run-test-slice.sh ${{matrix.nr}} 10
+      run: . /etc/profile && make -C t -e
     - name: ci/print-test-failures.sh
       if: failure()
       shell: bash
ci/select-test-slice.sh renamed from ci/run-test-slice.sh +5 -2
similarity index 50%
rename from ci/run-test-slice.sh
rename to ci/select-test-slice.sh
index 1de6a18ca47..f59d8cadda5 100755
@@ -1,10 +1,13 @@
 #!/bin/sh
 #
-# Test Git in parallel
+# Select a portion of the tests for testing Git in parallel
 #
 
 . ${0%/*}/lib.sh
 
 tests=$(echo $(cd t && ./helper/test-tool path-utils slice-tests "$1" "$2" \
 	t[0-9]*.sh))
-make --quiet -C t T="$tests"
+if test -n "$GITHUB_ENV"
+then
+	echo T="$tests" >>$GITHUB_ENV
+fi
Signature · 1 line
2.36.0.879.g3659959fcca
#222 lore raw

PATCH v5 22/29 CI: only invoke ci/lib.sh as "steps" in main.yml

In reply to Ævar Arnfjörð Bjarmason

Change the scripts in ci/ to stop using ci/lib.sh as a library, now that the only thing it did for them was to "set -ex" and possibly set TERM=dumb.

Let's create a ci/lib-tput.sh for those that need to use "tput" instead, and have these scripts invoke "set -ex" themselves.

This makes their invocation a lot less verbose, since they'll be relying on an earlier step in the CI job to have set the variables in $GITHUB_ENV, and won't be spewing their own trace output to set those variables again.

Let's also create a ci/lib-ci-type.sh, and have ci/lib.sh and ci/print-test-failures.sh share the logic to discover the CI type. We could have set the CI_TYPE in the environment with "setenv", but let's avoid that verbosity for this purely internal variable.

The "ci/lib.sh" is now no longer a "Library of functions shared by all CI scripts", so let's remove that commentary, and the misleading comment about "set -ex" being for "installing dependencies", we're now no longer using it in "ci/install-dependencies.sh" (but it does its own "set -ex").

---
 ci/check-unignored-build-artifacts.sh |  4 +++-
 ci/install-dependencies.sh            |  2 +-
 ci/lib-ci-type.sh                     |  9 +++++++++
 ci/lib-tput.sh                        |  5 +++++
 ci/lib.sh                             | 27 ++++++++++-----------------
 ci/print-test-failures.sh             |  6 +++---
 ci/select-test-slice.sh               |  2 +-
 ci/test-documentation.sh              |  2 +-
 8 files changed, 33 insertions(+), 24 deletions(-)
 create mode 100644 ci/lib-ci-type.sh
 create mode 100644 ci/lib-tput.sh
ci/check-unignored-build-artifacts.sh +3 -1
index 0bc04f32804..c27d6a97f45 100755
@@ -3,7 +3,9 @@
 # Check whether the build created anything not in our .gitignore
 #
 
-. ${0%/*}/lib.sh
+set -ex
+
+. ${0%/*}/lib-tput.sh
 
 check_unignored_build_artifacts ()
 {
ci/install-dependencies.sh +1 -1
index adece989491..4b4cdbe26e0 100755
@@ -3,7 +3,7 @@
 # Install dependencies required to build and test Git on Linux and macOS
 #
 
-. ${0%/*}/lib.sh
+set -ex
 
 UBUNTU_COMMON_PKGS="make libssl-dev libcurl4-openssl-dev libexpat-dev
  tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl
ci/lib-ci-type.sh new file +9 -0
new file mode 100644
index 00000000000..6f01fd9e5d9
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+if test "$GITHUB_ACTIONS" = "true"
+then
+	CI_TYPE=github-actions
+else
+	echo "Could not identify CI type" >&2
+	exit 1
+fi
ci/lib-tput.sh new file +5 -0
new file mode 100644
index 00000000000..b62ef13c44c
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# GitHub Action doesn't set TERM, which is required by tput
+TERM=${TERM:-dumb}
+export TERM
ci/lib.sh +10 -17
index cae324e2fd7..fea45a1a8d3 100755
@@ -1,13 +1,9 @@
 #!/bin/sh
-
-# Library of functions shared by all CI scripts
-
-# Set 'exit on error' for all CI scripts to let the caller know that
-# something went wrong.
-# Set tracing executed commands, primarily setting environment variables
-# and installing dependencies.
 set -ex
 
+# Helper libraries
+. ${0%/*}/lib-ci-type.sh
+
 # Starting assertions
 if test -z "$jobname"
 then
@@ -47,18 +43,14 @@ setenv () {
 	fi
 }
 
-# GitHub Action doesn't set TERM, which is required by tput
-setenv TERM ${TERM:-dumb}
-
 # How many jobs to run in parallel?
 NPROC=10
 
 # Clear MAKEFLAGS that may come from the outside world.
 MAKEFLAGS=--jobs=$NPROC
 
-if test "$GITHUB_ACTIONS" = "true"
-then
-	CI_TYPE=github-actions
+case "$CI_TYPE" in
+github-actions)
 	CC="${CC:-gcc}"
 
 	setenv --test GIT_PROVE_OPTS "--timer --jobs $NPROC"
@@ -66,11 +58,12 @@ then
 	test Windows != "$RUNNER_OS" ||
 	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
 	setenv --test GIT_TEST_OPTS "$GIT_TEST_OPTS"
-else
-	echo "Could not identify CI type" >&2
-	env >&2
+	;;
+*)
+	echo "Unhandled CI type: $CI_TYPE" >&2
 	exit 1
-fi
+	;;
+esac
 
 setenv --build DEVELOPER 1
 setenv --test DEFAULT_TEST_TARGET prove
ci/print-test-failures.sh +3 -3
index 47e2ea1ad10..29f8c332eca 100755
@@ -3,10 +3,10 @@
 # Print output of failing tests
 #
 
-. ${0%/*}/lib.sh
+set -e
 
-# Tracing executed commands would produce too much noise in the loop below.
-set +x
+. ${0%/*}/lib-ci-type.sh
+. ${0%/*}/lib-tput.sh
 
 cd t/
 
ci/select-test-slice.sh +1 -1
index f59d8cadda5..eccf9a48104 100755
@@ -3,7 +3,7 @@
 # Select a portion of the tests for testing Git in parallel
 #
 
-. ${0%/*}/lib.sh
+set -ex
 
 tests=$(echo $(cd t && ./helper/test-tool path-utils slice-tests "$1" "$2" \
 	t[0-9]*.sh))
ci/test-documentation.sh +1 -1
index b8a6a6f664e..64ff212cdaa 100755
@@ -3,7 +3,7 @@
 # Perform sanity checks on "make doc" output and built documentation
 #
 
-. ${0%/*}/lib.sh
+set -ex
 
 generator=$1
 
Signature · 1 line
2.36.0.879.g3659959fcca
#223 lore raw

PATCH v5 23/29 CI: narrow down variable definitions in --build and --test

In reply to Ævar Arnfjörð Bjarmason

In a preceding step the "setvar" function was made to take a "--build", "--test" or "--all" argument to indicate where the variables it sets were used.

Let's make use of that by having the relevant parts of ".github/workflows/main.yml" invoke "ci/lib.sh" with those options.

By doing this the set of variables shown in build-only steps will be fewer, which makes diagnosing anything going on there easier, as we won't have to look at a deluge of e.g. GIT_TEST_* variables.

For the "pedantic" job (which has no test phase) we won't run the "ci/lib.sh --test" step, which will be clearly visible as a skipped step in the UX.

Since we'll now always run "--build" for "make" and "--test" for "make test" we can stop setting the "fat" MAKEFLAGS entirely on the "test" steps, i.e. the one with PYTHON_PATH, CC etc. These will all be carried over from the earlier --build step.

That won't be true in the case of the "windows-test" and "vs-test" jobs, since they run separately from the corresponding "build" step. So we'll need to make sure that we have the --jobs=N argument for those.

This doesn't matter that much, as we'll still have --jobs=N in GIT_PROVE_OPTS. So the only thing we'll use it for is parallelism in the t/Makefile before we get to running "prove". Still, it's good to be consistent for good measure, and to run the t/Makefile itself in parallel.

---
 .github/workflows/main.yml | 21 ++++++++-------
 ci/lib.sh                  | 52 +++++++++++++++++++++++++++++++++++---
 2 files changed, 60 insertions(+), 13 deletions(-)
.github/workflows/main.yml +12 -9
index 83e0aa1f469..bb62b4ff725 100644
@@ -84,7 +84,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
       shell: bash
     - name: build
       shell: bash
@@ -122,7 +122,7 @@ jobs:
       shell: bash
       run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
-    - run: ci/lib.sh
+    - run: ci/lib.sh --test
       shell: bash
     - name: select tests
       run: . /etc/profile && ci/select-test-slice.sh ${{matrix.nr}} 10
@@ -169,7 +169,7 @@ jobs:
     - name: copy dlls to root
       shell: cmd
       run: compat\vcbuild\vcpkg_copy_dlls.bat release
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
       shell: bash
     - name: generate Visual Studio solution
       shell: bash
@@ -211,7 +211,7 @@ jobs:
     - name: extract tracked files and build artifacts
       shell: bash
       run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz
-    - run: ci/lib.sh
+    - run: ci/lib.sh --test
       shell: bash
     - name: select tests
       run: . /etc/profile && ci/select-test-slice.sh ${{matrix.nr}} 10
@@ -275,8 +275,9 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
     - run: make
+    - run: ci/lib.sh --test
     - run: make test
       if: success()
     - run: ci/print-test-failures.sh
@@ -310,8 +311,10 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - run: ci/install-dependencies.sh
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
     - run: make
+    - run: ci/lib.sh --test
+      if: success() && matrix.vector.skip-tests != 'yes'
     - run: make test
       if: success() && matrix.vector.skip-tests != 'yes'
     - run: ci/print-test-failures.sh
@@ -331,7 +334,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
     - run: make ci-static-analysis
   sparse:
     needs: ci-config
@@ -352,7 +355,7 @@ jobs:
     - uses: actions/checkout@v2
     - name: Install other dependencies
       run: ci/install-dependencies.sh
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
     - run: make sparse
   documentation:
     name: documentation
@@ -364,7 +367,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/lib.sh
+    - run: ci/lib.sh --build
     - run: make check-docs
     - run: "make doc > >(tee stdout.log) 2> >(tee stderr.raw >&2)"
       shell: bash
ci/lib.sh +48 -4

94 lines not on this page · raw message

Signature · 1 line
2.36.0.879.g3659959fcca
#224 lore raw

PATCH v5 24/29 CI: add more variables to MAKEFLAGS, except under vs-build

In reply to Ævar Arnfjörð Bjarmason

It's clearer that "DEVELOPER" is a flag that affects the Makefile itself in particular if it's put into "MAKEFLAGS than" if it generically sits in the environment. Let's move both it and "SKIP_DASHED_BUILT_INS" to "MAKEFLAGS".

We can't do this under vs-build, since that invokes cmake. Let's have only that job set these in the environment.

---
 ci/lib.sh | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)
ci/lib.sh +10 -2
index c875208817f..c8cc37c50b4 100755
@@ -89,6 +89,13 @@ COMMON_MAKEFLAGS=--jobs=$NPROC
 # Clear MAKEFLAGS that may come from the outside world.
 MAKEFLAGS=$COMMON_MAKEFLAGS
 
+# Use common options for "make" (cmake in "vs-build" below uses the
+# intermediate variables directly)
+DEVELOPER=1
+MAKEFLAGS="$MAKEFLAGS DEVELOPER=$DEVELOPER"
+SKIP_DASHED_BUILT_INS=YesPlease
+MAKEFLAGS="$MAKEFLAGS SKIP_DASHED_BUILT_INS=$SKIP_DASHED_BUILT_INS"
+
 case "$CI_TYPE" in
 github-actions)
 	CC="${CC:-gcc}"
@@ -105,10 +112,8 @@ github-actions)
 	;;
 esac
 
-setenv --build DEVELOPER 1
 setenv --test DEFAULT_TEST_TARGET prove
 setenv --test GIT_TEST_CLONE_2GB true
-setenv --build SKIP_DASHED_BUILT_INS YesPlease
 
 case "$runs_on_pool" in
 ubuntu-latest)
@@ -145,6 +150,9 @@ windows-test)
 	setenv --test MAKEFLAGS "$COMMON_MAKEFLAGS"
 	;;
 vs-build)
+	setenv --build DEVELOPER $DEVELOPER
+	setenv --build SKIP_DASHED_BUILT_INS $SKIP_DASHED_BUILT_INS
+
 	setenv --build NO_PERL NoThanks
 	setenv --build NO_GETTEXT NoThanks
 	setenv --build ARTIFACTS_DIRECTORY artifacts
Signature · 1 line
2.36.0.879.g3659959fcca
#225 lore raw

PATCH v5 25/29 CI: set CC in MAKEFLAGS directly, don't add it to the environment

In reply to Ævar Arnfjörð Bjarmason

Rather than pass a "$CC" in the environment to be picked up in ci/lib.sh let's instead have ci/lib.sh itself add it directly to MAKEFLAGS. For "$CC_PACKAGE" its setting and use can stay within ci/install-dependencies.sh.

Setting CC=gcc by default made for confusing trace output, and since a preceding change to carry it and others over across "steps" in the GitHub CI it's been even more misleading. E.g. the "win+VS build" job confusingly has CC=gcc set, even though it builds with MSVC.

Let's instead rely on the Makefile default of CC=cc, and only override it for those jobs where it's needed. This does mean that we'll need to set it for the "pedantic" job, which previously relied on the default CC=gcc in case "clang" become the default on that platform.

This partially reverts my 707d2f2fe86 (CI: use "$runs_on_pool", not "$jobname" to select packages & config, 2021-11-23), i.e. we're now aiming to only set those variables specific jobs need.

Since we don't need to make this generic enough to handle "gcc-9" turning into "gcc@9" for "brew install" let's remove what was originally a bash-specific replacement in 707d2f2fe86 (CI: use "$runs_on_pool", not "$jobname" to select packages & config, 2021-11-23), and which a preceding commit changed to a "tr" invocation.

---
 .github/workflows/main.yml | 13 -------------
 ci/install-dependencies.sh | 18 ++++++++++++++----
 ci/lib.sh                  | 23 ++++++++++++++++++++---
 3 files changed, 34 insertions(+), 20 deletions(-)
.github/workflows/main.yml +0 -13
index bb62b4ff725..6835e942280 100644
@@ -238,37 +238,24 @@ jobs:
       matrix:
         vector:
           - jobname: linux-clang
-            cc: clang
             pool: ubuntu-latest
           - jobname: linux-sha256
-            cc: clang
             os: ubuntu
             pool: ubuntu-latest
           - jobname: linux-gcc
-            cc: gcc
-            cc_package: gcc-8
             pool: ubuntu-latest
           - jobname: linux-TEST-vars
-            cc: gcc
             os: ubuntu
-            cc_package: gcc-8
             pool: ubuntu-latest
           - jobname: osx-clang
-            cc: clang
             pool: macos-latest
           - jobname: osx-gcc
-            cc: gcc
-            cc_package: gcc-9
             pool: macos-latest
           - jobname: linux-gcc-default
-            cc: gcc
             pool: ubuntu-latest
           - jobname: linux-leaks
-            cc: gcc
             pool: ubuntu-latest
     env:
-      CC: ${{matrix.vector.cc}}
-      CC_PACKAGE: ${{matrix.vector.cc_package}}
       jobname: ${{matrix.vector.jobname}}
       runs_on_pool: ${{matrix.vector.pool}}
     runs-on: ${{matrix.vector.pool}}
ci/install-dependencies.sh +14 -4
index 4b4cdbe26e0..65548bf0b25 100755
@@ -9,6 +9,17 @@ UBUNTU_COMMON_PKGS="make libssl-dev libcurl4-openssl-dev libexpat-dev
  tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl
  libemail-valid-perl libio-socket-ssl-perl libnet-smtp-ssl-perl"
 
+CC_PACKAGE=
+BREW_CC_PACKAGE=
+case "$jobname" in
+linux-gcc | linux-TEST-vars)
+	CC_PACKAGE=gcc-8
+	;;
+osx-gcc)
+	BREW_CC_PACKAGE=gcc@9
+	;;
+esac
+
 case "$runs_on_pool" in
 ubuntu-latest)
 	# The Linux build installs the defined dependency versions below.
@@ -67,11 +78,10 @@ macos-latest)
 	} ||
 	brew install homebrew/cask/perforce
 
-	if test -n "$CC_PACKAGE"
+	if test -n "$BREW_CC_PACKAGE"
 	then
-		BREW_PACKAGE=$(echo $CC_PACKAGE | tr '-' '@')
-		brew install "$BREW_PACKAGE"
-		brew link "$BREW_PACKAGE"
+		brew install "$BREW_CC_PACKAGE"
+		brew link "$BREW_CC_PACKAGE"
 	fi
 	;;
 esac
ci/lib.sh +20 -3
index c8cc37c50b4..8d19ca5ced8 100755
@@ -79,6 +79,9 @@ setenv () {
 	fi
 }
 
+# Clear variables that may come from the outside world.
+CC=
+
 # How many jobs to run in parallel?
 NPROC=10
 
@@ -98,8 +101,6 @@ MAKEFLAGS="$MAKEFLAGS SKIP_DASHED_BUILT_INS=$SKIP_DASHED_BUILT_INS"
 
 case "$CI_TYPE" in
 github-actions)
-	CC="${CC:-gcc}"
-
 	setenv --test GIT_PROVE_OPTS "--timer --jobs $NPROC"
 	GIT_TEST_OPTS="--verbose-log -x"
 	test Windows != "$RUNNER_OS" ||
@@ -171,9 +172,14 @@ vs-test)
 	setenv --test MAKEFLAGS "$COMMON_MAKEFLAGS"
 	;;
 linux-gcc)
+	CC=gcc
 	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME main
 	;;
+linux-gcc-default)
+	CC=gcc
+	;;
 linux-TEST-vars)
+	CC=gcc
 	setenv --test GIT_TEST_SPLIT_INDEX yes
 	setenv --test GIT_TEST_MERGE_ALGORITHM recursive
 	setenv --test GIT_TEST_FULL_IN_PACK_ARRAY true
@@ -188,13 +194,22 @@ linux-TEST-vars)
 	setenv --test GIT_TEST_WRITE_REV_INDEX 1
 	setenv --test GIT_TEST_CHECKOUT_WORKERS 2
 	;;
+osx-gcc)
+	CC=gcc
+	;;
+osx-clang)
+	CC=clang
+	;;
 linux-clang)
+	CC=clang
 	setenv --test GIT_TEST_DEFAULT_HASH sha1
 	;;
 linux-sha256)
+	CC=clang
 	setenv --test GIT_TEST_DEFAULT_HASH sha256
 	;;
 pedantic)
+	CC=gcc
 	# Don't run the tests; we only care about whether Git can be
 	# built.
 	setenv --build DEVOPTS pedantic
@@ -209,9 +224,11 @@ linux-musl)
 	MAKEFLAGS="$MAKEFLAGS GIT_TEST_UTF8_LOCALE=C.UTF-8"
 	;;
 linux-leaks)
+	CC=gcc
 	setenv --build SANITIZE leak
 	setenv --test GIT_TEST_PASSING_SANITIZE_LEAK true
 	;;
 esac
 
-setenv --build MAKEFLAGS "$MAKEFLAGS CC=${CC:-cc}"
+MAKEFLAGS="$MAKEFLAGS${CC:+ CC=$CC}"
+setenv --build MAKEFLAGS "$MAKEFLAGS"
Signature · 1 line
2.36.0.879.g3659959fcca
#226 lore raw

PATCH v5 26/29 CI: set SANITIZE=leak in MAKEFLAGS directly

In reply to Ævar Arnfjörð Bjarmason

As in the preceding commit's change to do likewise for CC, let's not set SANITIZE in the environment. When I added the "linux-leaks" job in in 956d2e4639b (tests: add a test mode for SANITIZE=leak, run it in CI, 2021-09-23) I missed that I could do that.

---
 ci/lib.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
ci/lib.sh +1 -1
index 8d19ca5ced8..b6e7a6647bb 100755
@@ -225,7 +225,7 @@ linux-musl)
 	;;
 linux-leaks)
 	CC=gcc
-	setenv --build SANITIZE leak
+	MAKEFLAGS="$MAKEFLAGS SANITIZE=leak"
 	setenv --test GIT_TEST_PASSING_SANITIZE_LEAK true
 	;;
 esac
Signature · 1 line
2.36.0.879.g3659959fcca
#227 lore raw

PATCH v5 27/29 CI: set PYTHON_PATH setting for osx-{clang,gcc} into "$jobname" case

In reply to Ævar Arnfjörð Bjarmason

Move the setting of "PYTHON_PATH=[...]" to the "$jobname" case statement. This partially backs out of my 707d2f2fe86 (CI: use "$runs_on_pool", not "$jobname" to select packages & config, 2021-11-23), now that we have a "osx-{clang,gcc}" anyway for setting "$CC" we might as well do away with this part of the "$runs_on_pool" case.

---
 ci/lib.sh | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)
ci/lib.sh +2 -8
index b6e7a6647bb..f2f75901a9c 100755
@@ -132,14 +132,6 @@ ubuntu-latest)
 
 	setenv --test GIT_TEST_HTTPD true
 	;;
-macos-latest)
-	if test "$jobname" = osx-gcc
-	then
-		MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
-	else
-		MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python2)"
-	fi
-	;;
 esac
 
 case "$jobname" in
@@ -195,9 +187,11 @@ linux-TEST-vars)
 	setenv --test GIT_TEST_CHECKOUT_WORKERS 2
 	;;
 osx-gcc)
+	MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
 	CC=gcc
 	;;
 osx-clang)
+	MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python2)"
 	CC=clang
 	;;
 linux-clang)
Signature · 1 line
2.36.0.879.g3659959fcca
#228 lore raw

PATCH v5 28/29 CI: don't use "set -x" in "ci/lib.sh" output

In reply to Ævar Arnfjörð Bjarmason

Remove the "set -x" from the "ci/lib.sh" output. Before preceding commits the logic in that file was much more complex, and likely to fail in some scenarios.

Now we only task "ci/lib.sh" with setting various variables for subsequent steps in our jobs, so we can start emitting more tailored debugging output, which makes what it's doing easier to read.

This change also changes the output of the "ci/print-test-failures.sh" script, since it's the only other user of "ci/lib-ci-type.sh". In that case it's also helpful to know what "$CI_TYPE" we're using, as that script doesn't "set -x" and will act differently depending on the $CI_TYPE.

It's useful when developing this script to see what variables it *doesn't* set, but such output isn't useful to users, so let's hide the "SKIP" output behind a --debug option.

---
 ci/lib.sh | 35 ++++++++++++++++++++++++++++++++---
 1 file changed, 32 insertions(+), 3 deletions(-)
ci/lib.sh +32 -3
index f2f75901a9c..9e38c893890 100755
@@ -1,5 +1,5 @@
 #!/bin/sh
-set -ex
+set -e
 
 # Helper libraries
 . ${0%/*}/lib-ci-type.sh
@@ -7,15 +7,22 @@ set -ex
 # Parse options
 mode_build=
 mode_test=
+mode_debug=
 while test $# != 0
 do
 	case "$1" in
 	--build)
+		echo "MODE: $1" >&2
 		mode_build=t
 		;;
 	--test)
+		echo "MODE: $1" >&2
 		mode_test=t
 		;;
+	--debug)
+		echo "DEBUG: $1" >&2
+		mode_debug=t
+		;;
 	-*)
 		echo "error: invalid option: $1" >&2
 		exit 1
@@ -41,21 +48,32 @@ then
 	exit 1
 fi
 
+# Show our configuration
+echo "CONFIG: CI_TYPE=$CI_TYPE" >&2
+echo "CONFIG: jobname=$jobname" >&2
+echo "CONFIG: runs_on_pool=$runs_on_pool" >&2
+if test -n "$GITHUB_ENV"
+then
+	echo "CONFIG: GITHUB_ENV=$GITHUB_ENV" >&2
+fi
+echo >&2
+
 # Helper functions
 setenv () {
+	skip=
 	while test $# != 0
 	do
 		case "$1" in
 		--build)
 			if test -z "$mode_build"
 			then
-				return 0
+				skip=t
 			fi
 			;;
 		--test)
 			if test -z "$mode_test"
 			then
-				return 0
+				skip=t
 			fi
 			;;
 		-*)
@@ -73,10 +91,21 @@ setenv () {
 	val=$2
 	shift 2
 
+	if test -n "$skip"
+	then
+		if test -n "$mode_debug"
+		then
+			echo "SKIP: '$key=$val'" >&2
+		fi
+		return 0
+	fi
+
 	if test -n "$GITHUB_ENV"
 	then
 		echo "$key=$val" >>"$GITHUB_ENV"
 	fi
+
+	echo "SET: '$key=$val'" >&2
 }
 
 # Clear variables that may come from the outside world.
Signature · 1 line
2.36.0.879.g3659959fcca
#229 lore raw

PATCH v5 29/29 CI: make it easy to use ci/*.sh outside of CI

In reply to Ævar Arnfjörð Bjarmason

In preceding commits the ci/.sh scripts have lost most of their CI-specific assumptions. Let's go even further and explicitly support running ci/lib.sh outside of CI.

This was possible before by faking up enough CI-specific variables, but as shown in the new "help" output being added here using the ci/lib.sh to provide "CI-like" has now become trivial.

The ci/print-test-failures.sh scripts can now be used outside of CI as well, the only GitHub CI-specific part is now guarded by a check that we'll pass if outside of GitHub CI.

There's also a special-case here to not clobber $MAKEFLAGS in the environment if we're outside of CI, in case the user has e.g. "jN" or other flags to "make" that they'd prefer configured already.

Using "ci/lib.sh" as a stand-alone script is much more useful if it doesn't hardcode NPROC=10, let's provide a poor shellscript replacement for the online_cpus() we have in thread-utils.c to cover the most common OS's.

---
 ci/lib-ci-type.sh         |  3 --
 ci/lib-online_cpus.sh     | 26 +++++++++++++
 ci/lib.sh                 | 80 ++++++++++++++++++++++++++++++++++-----
 ci/print-test-failures.sh |  4 --
 4 files changed, 96 insertions(+), 17 deletions(-)
 create mode 100644 ci/lib-online_cpus.sh
ci/lib-ci-type.sh +0 -3
index 6f01fd9e5d9..09acab7aaec 100644
@@ -3,7 +3,4 @@
 if test "$GITHUB_ACTIONS" = "true"
 then
 	CI_TYPE=github-actions
-else
-	echo "Could not identify CI type" >&2
-	exit 1
 fi
ci/lib-online_cpus.sh new file +26 -0
new file mode 100644
index 00000000000..7cf0cd042fa
@@ -0,0 +1,26 @@
+#!/bin/sh
+
+# TODO: Ideally we'd compile t/helper/test-online-cpus.c early, but
+# that currently presents a chicken & egg problem. We need this before
+# we build (much of) anything.
+online_cpus() {
+	NPROC=
+
+	if command -v nproc >/dev/null
+	then
+		# GNU coreutils
+		NPROC=$(nproc)
+	elif command -v sysctl >/dev/null
+	then
+		# BSD & Mac OS X
+		NPROC=$(sysctl -n hw.ncpu)
+	elif test -n "$NUMBER_OF_PROCESSORS"
+	then
+		# Windows
+		NPROC="$NUMBER_OF_PROCESSORS"
+	else
+		NPROC=1
+	fi
+
+	echo $NPROC
+}
ci/lib.sh +70 -10

145 lines not on this page · raw message

ci/print-test-failures.sh +0 -4
index 29f8c332eca..ec395c79ccd 100755
@@ -36,10 +36,6 @@ do
 			cp "${TEST_EXIT%.exit}.out" failed-test-artifacts/
 			tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir"
 			;;
-		*)
-			echo "Unhandled CI type: $CI_TYPE" >&2
-			exit 1
-			;;
 		esac
 	fi
 done
Signature · 1 line
2.36.0.879.g3659959fcca
#230 lore raw

v6 cover CI: run "make" in CI "steps", improve UX

In reply to Ævar Arnfjörð Bjarmason

This series changes CI "step" targets that are shellscripts that do N things to instead be single command invocations at the "step" level, driven by the CI recipe itself.

To do that we need to pass state that we previously re-setup for every "step" via $GITHUB_ENV, whose state we then helpfully show (this is just a standard GitHub CI feature) in a drop-down at the start of every "step".

I.e. at the tip of this series you can reliably look at that $GITHUB_ENV view to see what the full and relevant environment was for that "make", "make test" or whatever.

The only changes since v5 are to rebase this on an updated "master", which had some small fixes in the ci/ directory. For the last iteration see:
https://lore.kernel.org/git/[email protected]/

I'll follow this up with a re-roll of
https://lore.kernel.org/git/[email protected]/
which is a replacement for
https://lore.kernel.org/git/[email protected]/
based on this topic.

To view the CI output differences this topic makes see these two runs:

master: https://github.com/avar/git/actions/runs/2377697338
this: https://github.com/avar/git/actions/runs/2377257487

These runs have a commit on top to trigger CI build and test failures in various areas. One example is here:

master: https://github.com/avar/git/runs/6573059000?check_suite_focus=true
this: https://github.com/avar/git/runs/6571804790?check_suite_focus=true

This shows that we fail on "make" instead of a monolothic "ci/run-build-and-test.sh" step. Furthermore we can see that a preceding "ci/lib.sh --build" step has been added to set & export (via $GITHUB_ENV) the full set of variables "make" will use in the failing step. Those are then visible at (click to expand it):

https://github.com/avar/git/runs/6571804790?check_suite_focus=true#step:5:1

For every step we now do the same, i.e. it's immediately obvious what parameters go into running that step. The last step in this series then makes it easy to run builds and tests "like CI" using the same tooling:

	$ ci/lib.sh 
	error: must set a CI jobname in the environment
	usage: jobname=<job-name> [runs_on_pool=<pool-name>] ci/lib.sh (--build | --test)

Running ci/lib.sh outside of CI? You can use ci/lib.sh to set up your environment like a given CI job. E.g.:

# run "make all test" like the "linux-leaks" job (eval $(jobname=linux-leaks ci/lib.sh --all) && make test)

# run "make all test" like the "linux-musl" job (eval $(jobname=linux-musl ci/lib.sh --all) && make test)

# run "make test" like the "linux-TEST-vars" job (uses various GIT_TEST_* modes) make && (eval $(jobname=linux-TEST-vars ci/lib.sh --test) && make test)

# run "make test" like the "linux-sha256" job
make && (eval $(jobname=linux-sha256 ci/lib.sh --test) && make test)

Note that some of these (e.g. the linux-musl one) may not work as expected due to the CI job configuring a platform that may not match yours.

As noted in the usage this can then be directly eval'd:

	$ jobname=osx-gcc ci/lib.sh --build 2>/dev/null 
	MAKEFLAGS="j8 DEVELOPER=1 SKIP_DASHED_BUILT_INS=YesPlease PYTHON_PATH=/usr/bin/python3 CC=gcc"
	export MAKEFLAGS

Of course YMMV when doing so, running that on my Linux box won't magically give me an OSX environment. But it's very useful to be able to to e.g.:

	jobname=linux-TEST-vars ci/lib.sh --test

Which allows myself and others to tweak our local build scripts to test things locally with setups that mirror these CI setups as far as GIT_TEST_* etc. parameters are concerned.

Ævar Arnfjörð Bjarmason (29):
CI: run "set -ex" early in ci/lib.sh
CI: make "$jobname" explicit, remove fallback
CI: remove more dead Travis CI support
CI: remove dead "tree skipping" code
CI: remove unused Azure ci/* code
CI/lib.sh: stop adding leading whitespace to $MAKEFLAGS
CI: don't have "git grep" invoke a pager in tree content check
CI: have "static-analysis" run a "make ci-static-analysis" target
CI: have "static-analysis" run "check-builtins", not "documentation"
CI: move p4 and git-lfs variables to ci/install-dependencies.sh
CI: consistently use "export" in ci/lib.sh
CI: export variables via a wrapper
CI: remove "run-build-and-tests.sh", run "make [test]" directly
CI: make ci/{lib,install-dependencies}.sh POSIX-compatible
CI: check ignored unignored build artifacts in "win[+VS] build" too
CI: invoke "make artifacts-tar" directly in windows-build
CI: split up and reduce "ci/test-documentation.sh"
CI: combine ci/install{,-docker}-dependencies.sh
CI: move "env" definitions into ci/lib.sh ci/run-test-slice.sh: replace shelling out with "echo"
CI: pre-select test slice in Windows & VS tests
CI: only invoke ci/lib.sh as "steps" in main.yml
CI: narrow down variable definitions in --build and --test
CI: add more variables to MAKEFLAGS, except under vs-build
CI: set CC in MAKEFLAGS directly, don't add it to the environment
CI: set SANITIZE=leak in MAKEFLAGS directly
CI: set PYTHON_PATH setting for osx-{clang,gcc} into "$jobname" case
CI: don't use "set -x" in "ci/lib.sh" output
CI: make it easy to use ci/*.sh outside of CI

 .github/workflows/main.yml            | 101 +++---
 Makefile                              |  31 +-
 ci/check-directional-formatting.bash  |   2 +-
 ci/check-unignored-build-artifacts.sh |  20 ++
 ci/install-dependencies.sh            |  82 +++--
 ci/install-docker-dependencies.sh     |  22 --
 ci/lib-ci-type.sh                     |   6 +
 ci/lib-online_cpus.sh                 |  26 ++
 ci/lib-tput.sh                        |   5 +
 ci/lib.sh                             | 424 ++++++++++++++++----------
 ci/make-test-artifacts.sh             |  12 -
 ci/mount-fileshare.sh                 |  25 --
 ci/print-test-failures.sh             |  53 +---
 ci/run-build-and-tests.sh             |  54 ----
 ci/run-docker-build.sh                |  66 ----
 ci/run-docker.sh                      |  47 ---
 ci/run-static-analysis.sh             |  32 --
 ci/run-test-slice.sh                  |  17 --
 ci/select-test-slice.sh               |  13 +
 ci/test-documentation.sh              |  37 +--
 ci/util/extract-trash-dirs.sh         |  50 ---
 shared.mak                            |   1 +
 22 files changed, 508 insertions(+), 618 deletions(-)
 create mode 100755 ci/check-unignored-build-artifacts.sh
 delete mode 100755 ci/install-docker-dependencies.sh
 create mode 100644 ci/lib-ci-type.sh
 create mode 100644 ci/lib-online_cpus.sh
 create mode 100644 ci/lib-tput.sh
 delete mode 100755 ci/make-test-artifacts.sh
 delete mode 100755 ci/mount-fileshare.sh
 delete mode 100755 ci/run-build-and-tests.sh
 delete mode 100755 ci/run-docker-build.sh
 delete mode 100755 ci/run-docker.sh
 delete mode 100755 ci/run-static-analysis.sh
 delete mode 100755 ci/run-test-slice.sh
 create mode 100755 ci/select-test-slice.sh
 delete mode 100755 ci/util/extract-trash-dirs.sh

Range-diff against v5:

 1:  84d8098e365 =  1:  25cb08ad06c CI: run "set -ex" early in ci/lib.sh
 2:  3f5c071b0f6 =  2:  1be7910c728 CI: make "$jobname" explicit, remove fallback
 3:  c64845a4efb !  3:  234e248e831 CI: remove more dead Travis CI support
    @@ ci/lib.sh: then
     -	test macos != "$CI_OS_NAME" || CI_OS_NAME=osx
     -	CI_REPO_SLUG="$GITHUB_REPOSITORY"
      	CI_JOB_ID="$GITHUB_RUN_ID"
    - 	CC="${CC:-gcc}"
    + 	CC="${CC_PACKAGE:-${CC:-gcc}}"
      	DONT_SKIP_TAGS=t
     @@ ci/lib.sh: then
      	export GIT_PROVE_OPTS="--timer --jobs 10"
 4:  45856e67d5d !  4:  36a6287ee9b CI: remove dead "tree skipping" code
    @@ ci/lib.sh: then
     -	CI_BRANCH="$GITHUB_REF"
     -	CI_COMMIT="$GITHUB_SHA"
     -	CI_JOB_ID="$GITHUB_RUN_ID"
    - 	CC="${CC:-gcc}"
    + 	CC="${CC_PACKAGE:-${CC:-gcc}}"
     -	DONT_SKIP_TAGS=t
     -
     -	cache_dir="$HOME/none"

5: d4671148eb7 ! 5: 66b0a383daf CI: remove unused Azure ci/* code
@@ ci/install-dependencies.sh: macos-latest)
- test -z "$BREW_INSTALL_PACKAGES" ||
- brew install $BREW_INSTALL_PACKAGES
brew link --force gettext
- brew install --cask --no-quarantine perforce || {
- # Update the definitions and try again
+ mkdir -p $HOME/bin
+ (
@@ ci/install-dependencies.sh: Documentation)
sudo apt-get -q update
sudo apt-get -q -y install asciidoc xmlto docbook-xsl-ns make
@@ ci/lib.sh: export TERM=${TERM:-dumb}
+if test "$GITHUB_ACTIONS" = "true"
then
CI_TYPE=github-actions
- CC="${CC:-gcc}"
+ CC="${CC_PACKAGE:-${CC:-gcc}}"

## ci/mount-fileshare.sh (deleted) ##
@@
6: 45ea80d0495 ! 6: d96333c1bd8 CI/lib.sh: stop adding leading whitespace to $MAKEFLAGS
@@ ci/lib.sh: check_unignored_build_artifacts ()
if test "$GITHUB_ACTIONS" = "true"
then
CI_TYPE=github-actions
- CC="${CC:-gcc}"
+ CC="${CC_PACKAGE:-${CC:-gcc}}"

- export GIT_PROVE_OPTS="--timer --jobs 10"
+ export GIT_PROVE_OPTS="--timer --jobs $NPROC"

 7:  a264b40ba0d =  7:  d0780ab4a96 CI: don't have "git grep" invoke a pager in tree content check
 8:  4a58f7be8d4 =  8:  84e368736e2 CI: have "static-analysis" run a "make ci-static-analysis" target
 9:  78c0c81af43 =  9:  1a9c98b909c CI: have "static-analysis" run "check-builtins", not "documentation"
10:  07e36035a1e ! 10:  e208a9ab1e2 CI: move p4 and git-lfs variables to ci/install-dependencies.sh
    @@ ci/install-dependencies.sh
      
      . ${0%/*}/lib.sh
      
    --P4WHENCE=http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION
    +-P4WHENCE=https://cdist2.perforce.com/perforce/r$LINUX_P4_VERSION
     -LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION
      UBUNTU_COMMON_PKGS="make libssl-dev libcurl4-openssl-dev libexpat-dev
       tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl
    @@ ci/install-dependencies.sh
     +	GIT_LFS_PATH="$HOME/custom/git-lfs"
     +	export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH"
     +
    -+	P4WHENCE=http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION
    ++	P4WHENCE=https://cdist2.perforce.com/perforce/r$LINUX_P4_VERSION
     +	LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION
     +
      	sudo apt-get -q update
11:  272bd590a95 ! 11:  9655bae9e21 CI: consistently use "export" in ci/lib.sh
    @@ ci/lib.sh: export TERM=${TERM:-dumb}
      if test "$GITHUB_ACTIONS" = "true"
      then
     @@ ci/lib.sh: then
    - 	CC="${CC:-gcc}"
    + 	CC="${CC_PACKAGE:-${CC:-gcc}}"
      
      	export GIT_PROVE_OPTS="--timer --jobs $NPROC"
     -	export GIT_TEST_OPTS="--verbose-log -x"
12:  ea35327e24c ! 12:  2da0aa1d40e CI: export variables via a wrapper
    @@ ci/lib.sh: check_unignored_build_artifacts ()
      NPROC=10
     @@ ci/lib.sh: then
      	CI_TYPE=github-actions
    - 	CC="${CC:-gcc}"
    + 	CC="${CC_PACKAGE:-${CC:-gcc}}"
      
     -	export GIT_PROVE_OPTS="--timer --jobs $NPROC"
     +	setenv --test GIT_PROVE_OPTS "--timer --jobs $NPROC"

13: f6ff81f48ab ! 13: 6e97633652e CI: remove "run-build-and-tests.sh", run "make [test]" directly
@@ ci/install-dependencies.sh: ubuntu-latest)
+ echo "$PATH" >>"$GITHUB_PATH"
+ fi

- P4WHENCE=http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION
+ P4WHENCE=https://cdist2.perforce.com/perforce/r$LINUX_P4_VERSION
LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION

## ci/lib.sh ##

14:  533dbc50c4f = 14:  0488e29859b CI: make ci/{lib,install-dependencies}.sh POSIX-compatible
15:  6044c2b383f = 15:  f7ac6c33044 CI: check ignored unignored build artifacts in "win[+VS] build" too
16:  4d1a9f88c32 = 16:  f89346f11eb CI: invoke "make artifacts-tar" directly in windows-build
17:  5c8f464a60f = 17:  7fa9c69e3ca CI: split up and reduce "ci/test-documentation.sh"
18:  a218be76819 ! 18:  7925b2610fc CI: combine ci/install{,-docker}-dependencies.sh

@@ ci/install-dependencies.sh: linux-gcc-default)
+ ;;
esac

- if type p4d >/dev/null && type p4 >/dev/null
+ if type p4d >/dev/null 2>&1 && type p4 >/dev/null 2>&1

## ci/install-docker-dependencies.sh (deleted) ##
@@

19:  83e21b4f501 = 19:  b00abc07637 CI: move "env" definitions into ci/lib.sh
20:  fd0e1610577 = 20:  bb5960355fe ci/run-test-slice.sh: replace shelling out with "echo"
21:  0c7cd9d64ba = 21:  1eeb2e8e7f0 CI: pre-select test slice in Windows & VS tests
22:  1b077665352 ! 22:  39979d9887d CI: only invoke ci/lib.sh as "steps" in main.yml
    @@ ci/lib.sh: setenv () {
     -	CI_TYPE=github-actions
     +case "$CI_TYPE" in
     +github-actions)
    - 	CC="${CC:-gcc}"
    + 	CC="${CC_PACKAGE:-${CC:-gcc}}"
      
      	setenv --test GIT_PROVE_OPTS "--timer --jobs $NPROC"
     @@ ci/lib.sh: then
23:  1867da7eddb = 23:  e2b3bf032d6 CI: narrow down variable definitions in --build and --test
24:  a4e3a9d2c28 ! 24:  c6f47d52823 CI: add more variables to MAKEFLAGS, except under vs-build

@@ ci/lib.sh: COMMON_MAKEFLAGS=--jobs=$NPROC
+
case "$CI_TYPE" in
github-actions)
- CC="${CC:-gcc}"
+ CC="${CC_PACKAGE:-${CC:-gcc}}"
@@ ci/lib.sh: github-actions)
;;
esac
25: 48d99cee809 ! 25: 377460e8024 CI: set CC in MAKEFLAGS directly, don't add it to the environment
@@ ci/install-dependencies.sh: UBUNTU_COMMON_PKGS="make libssl-dev libcurl4-openssl
ubuntu-latest)
# The Linux build installs the defined dependency versions below.
@@ ci/install-dependencies.sh: macos-latest)
- } ||
- brew install homebrew/cask/perforce
+ PATH="$PATH:${HOME}/bin"
+ export PATH

- if test -n "$CC_PACKAGE"
+ if test -n "$BREW_CC_PACKAGE"
@@ ci/lib.sh: MAKEFLAGS="$MAKEFLAGS SKIP_DASHED_BUILT_INS=$SKIP_DASHED_BUILT_INS"

case "$CI_TYPE" in
github-actions)
-- CC="${CC:-gcc}"
+- CC="${CC_PACKAGE:-${CC:-gcc}}"
-
setenv --test GIT_PROVE_OPTS "--timer --jobs $NPROC" GIT_TEST_OPTS="--verbose-log -x"

26:  07a6c087297 = 26:  98e320d5e67 CI: set SANITIZE=leak in MAKEFLAGS directly
27:  50f21ffdffe = 27:  282b7c89a4f CI: set PYTHON_PATH setting for osx-{clang,gcc} into "$jobname" case
28:  e35dff2bef3 = 28:  7a208927819 CI: don't use "set -x" in "ci/lib.sh" output
29:  13d8c6997c3 = 29:  4d8a8f3be07 CI: make it easy to use ci/*.sh outside of CI
Signature · 1 line
2.36.1.1045.gf356b5617dd
#231 lore raw

PATCH v6 1/29 CI: run "set -ex" early in ci/lib.sh

Change ci/lib.sh to run "set -ex" before it does anything else, before this we were omitting a few "export" commands in our trace output that happened before we enabled tracing.

---
 ci/lib.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
ci/lib.sh +6 -6
index 86e37da9bc5..57121dd664a 100755
@@ -1,5 +1,11 @@
 # Library of functions shared by all CI scripts
 
+# Set 'exit on error' for all CI scripts to let the caller know that
+# something went wrong.
+# Set tracing executed commands, primarily setting environment variables
+# and installing dependencies.
+set -ex
+
 skip_branch_tip_with_tag () {
 	# Sometimes, a branch is pushed at the same time the tag that points
 	# at the same commit as the tip of the branch is pushed, and building
@@ -85,12 +91,6 @@ export TERM=${TERM:-dumb}
 # Clear MAKEFLAGS that may come from the outside world.
 export MAKEFLAGS=
 
-# Set 'exit on error' for all CI scripts to let the caller know that
-# something went wrong.
-# Set tracing executed commands, primarily setting environment variables
-# and installing dependencies.
-set -ex
-
 if test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI"
 then
 	CI_TYPE=azure-pipelines
Signature · 1 line
2.36.1.1045.gf356b5617dd
#232 lore raw

PATCH v6 2/29 CI: make "$jobname" explicit, remove fallback

In reply to Ævar Arnfjörð Bjarmason

Remove the $jobname fallback last adjusted in b011fabd6e0 (ci/lib.sh:
encapsulate Travis-specific things, 2019-01-27).

This makes all of our CI jobs have an explicit $jobname, and removes the special-case of the Windows jobs implicitly setting one (that they don't currently use). In subsequent commits we'll use this new $jobname explicitly.

This is a follow-up to my 25715419bf4 (CI: don't run "make test" twice in one job, 2021-11-23), since then all the jobs except the Windows jobs have had an explicit $jobname.

In the case of the Windows jobs we were using a jobname of windows-gcc for both, due to picking "gcc" as a default fallback for "$CC" in ci/lib.sh.

Since we'll need this to run ci/lib.sh it makes sense to move the check to the top of the file. Subsequent commits will add more such assertions to it.

---
 .github/workflows/main.yml |  8 ++++++++
 ci/lib.sh                  | 12 +++++++-----
 2 files changed, 15 insertions(+), 5 deletions(-)
.github/workflows/main.yml +8 -0
index c35200defb9..309b9141249 100644
@@ -79,6 +79,8 @@ jobs:
 
   windows-build:
     name: win build
+    env:
+      jobname: windows-build
     needs: ci-config
     if: needs.ci-config.outputs.enabled == 'yes'
     runs-on: windows-latest
@@ -100,6 +102,8 @@ jobs:
         path: artifacts
   windows-test:
     name: win test
+    env:
+      jobname: windows-test
     runs-on: windows-latest
     needs: [windows-build]
     strategy:
@@ -131,6 +135,8 @@ jobs:
         path: ${{env.FAILED_TEST_ARTIFACTS}}
   vs-build:
     name: win+VS build
+    env:
+      jobname: vs-build
     needs: ci-config
     if: needs.ci-config.outputs.enabled == 'yes'
     env:
@@ -183,6 +189,8 @@ jobs:
         path: artifacts
   vs-test:
     name: win+VS test
+    env:
+      jobname: vs-test
     runs-on: windows-latest
     needs: vs-build
     strategy:
ci/lib.sh +7 -5
index 57121dd664a..3061de96d8f 100755
@@ -6,6 +6,13 @@
 # and installing dependencies.
 set -ex
 
+# Starting assertions
+if test -z "$jobname"
+then
+	echo "error: must set a CI jobname in the environment" >&2
+	exit 1
+fi
+
 skip_branch_tip_with_tag () {
 	# Sometimes, a branch is pushed at the same time the tag that points
 	# at the same commit as the tip of the branch is pushed, and building
@@ -146,11 +153,6 @@ test -n "${DONT_SKIP_TAGS-}" ||
 skip_branch_tip_with_tag
 skip_good_tree
 
-if test -z "$jobname"
-then
-	jobname="$CI_OS_NAME-$CC"
-fi
-
 export DEVELOPER=1
 export DEFAULT_TEST_TARGET=prove
 export GIT_TEST_CLONE_2GB=true
Signature · 1 line
2.36.1.1045.gf356b5617dd
#233 lore raw

PATCH v6 3/29 CI: remove more dead Travis CI support

In reply to Ævar Arnfjörð Bjarmason

Remove code that I missed in 4a6e4b96026 (CI: remove Travis CI support, 2021-11-23). This code was only called from or used by the now-removed .travis.yml, or needed by the Travis CI environment.

For the symlinking in ci/run-build-and-tests.sh: Back when 3c93b829205 (travis-ci: build Git during the 'script' phase, 2018-01-08) and 4b060a4d973 (ci: use a junction on Windows instead of a symlink, 2019-01-27) adjusted this "ln" command, the Windows build would use ci/run-build-and-tests.sh.

As seen in 889cacb6897 (ci: configure GitHub Actions for CI/PR, 2020-04-11) the current windows build uses a different entry point under the GitHub CI, which doesn't use this .prove caching. Namely "ci/run-test-slice.sh".

We can be certain that it's never used in "ci/run-test-slice.sh" because to have a ".prove" file we'd need to use "--state=save", which we only do in the dead Azure codepath in ci/lib.sh. If it were used it would do the wrong thing, because the different test slices would each try to clobber the same "t/.prove" file.

If a subsequent run then used the "--state=failed,slow,save" it would defeat the purpose of "ci/run-test-slice.sh", since all slices would then run all tests. I.e. behavior of prove's "--state" options is to select tests to run from the provided "--state" file, in addition to those specified on the command-line.

For ci/run-docker{,-build}.sh: It was likewise last referenced in the .travis.yml removed in my 4a6e4b96026. The current "dockerized" run in ".github/workflows/main.yml" calls the same entry points as the main "regular" job.

For "ci/print-test-failures.sh" the creation of the tarball added in aea8879a6ac (travis-ci: include the trash directories of failed tests in the trace log, 2018-08-01) has been dead since my 4a6e4b96026, which removed the fall-through case from the "case/esac" statement.

---
 ci/lib.sh                     | 10 ++----
 ci/print-test-failures.sh     | 39 ---------------------
 ci/run-build-and-tests.sh     |  5 ---
 ci/run-docker-build.sh        | 66 -----------------------------------
 ci/run-docker.sh              | 47 -------------------------
 ci/run-test-slice.sh          |  5 ---
 ci/util/extract-trash-dirs.sh | 50 --------------------------
 7 files changed, 2 insertions(+), 220 deletions(-)
 delete mode 100755 ci/run-docker-build.sh
 delete mode 100755 ci/run-docker.sh
 delete mode 100755 ci/util/extract-trash-dirs.sh
ci/lib.sh +2 -8
index 3061de96d8f..2e60c0a71b5 100755
@@ -106,9 +106,6 @@ then
 	CI_COMMIT="$BUILD_SOURCEVERSION"
 	CI_JOB_ID="$BUILD_BUILDID"
 	CI_JOB_NUMBER="$BUILD_BUILDNUMBER"
-	CI_OS_NAME="$(echo "$AGENT_OS" | tr A-Z a-z)"
-	test darwin != "$CI_OS_NAME" || CI_OS_NAME=osx
-	CI_REPO_SLUG="$(expr "$BUILD_REPOSITORY_URI" : '.*/\([^/]*/[^/]*\)$')"
 	CC="${CC:-gcc}"
 
 	# use a subdirectory of the cache dir (because the file share is shared
@@ -118,16 +115,13 @@ then
 	export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save"
 	export GIT_TEST_OPTS="--verbose-log -x --write-junit-xml"
 	MAKEFLAGS="$MAKEFLAGS --jobs=10"
-	test windows_nt != "$CI_OS_NAME" ||
+	test Windows_NT != "$AGENT_OS" ||
 	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
 elif test true = "$GITHUB_ACTIONS"
 then
 	CI_TYPE=github-actions
 	CI_BRANCH="$GITHUB_REF"
 	CI_COMMIT="$GITHUB_SHA"
-	CI_OS_NAME="$(echo "$RUNNER_OS" | tr A-Z a-z)"
-	test macos != "$CI_OS_NAME" || CI_OS_NAME=osx
-	CI_REPO_SLUG="$GITHUB_REPOSITORY"
 	CI_JOB_ID="$GITHUB_RUN_ID"
 	CC="${CC_PACKAGE:-${CC:-gcc}}"
 	DONT_SKIP_TAGS=t
@@ -137,7 +131,7 @@ then
 	export GIT_PROVE_OPTS="--timer --jobs 10"
 	export GIT_TEST_OPTS="--verbose-log -x"
 	MAKEFLAGS="$MAKEFLAGS --jobs=10"
-	test windows != "$CI_OS_NAME" ||
+	test Windows != "$RUNNER_OS" ||
 	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
 else
 	echo "Could not identify CI type" >&2
ci/print-test-failures.sh +0 -39
index 57277eefcd0..740ebb7c529 100755
@@ -16,15 +16,6 @@ then
 	exit
 fi
 
-case "$jobname" in
-osx-clang|osx-gcc)
-	# base64 in OSX doesn't wrap its output at 76 columns by
-	# default, but prints a single, very long line.
-	base64_opts="-b 76"
-	;;
-esac
-
-combined_trash_size=0
 for TEST_EXIT in test-results/*.exit
 do
 	if [ "$(cat "$TEST_EXIT")" != "0" ]
@@ -42,47 +33,17 @@ do
 		azure-pipelines)
 			mkdir -p failed-test-artifacts
 			mv "$trash_dir" failed-test-artifacts
-			continue
 			;;
 		github-actions)
 			mkdir -p failed-test-artifacts
 			echo "FAILED_TEST_ARTIFACTS=t/failed-test-artifacts" >>$GITHUB_ENV
 			cp "${TEST_EXIT%.exit}.out" failed-test-artifacts/
 			tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir"
-			continue
 			;;
 		*)
 			echo "Unhandled CI type: $CI_TYPE" >&2
 			exit 1
 			;;
 		esac
-		trash_tgz_b64="trash.$test_name.base64"
-		if [ -d "$trash_dir" ]
-		then
-			tar czp "$trash_dir" |base64 $base64_opts >"$trash_tgz_b64"
-
-			trash_size=$(wc -c <"$trash_tgz_b64")
-			if [ $trash_size -gt 1048576 ]
-			then
-				# larger than 1MB
-				echo "$(tput setaf 1)Didn't include the trash directory of '$test_name' in the trace log, it's too big$(tput sgr0)"
-				continue
-			fi
-
-			new_combined_trash_size=$(($combined_trash_size + $trash_size))
-			if [ $new_combined_trash_size -gt 1048576 ]
-			then
-				echo "$(tput setaf 1)Didn't include the trash directory of '$test_name' in the trace log, there is plenty of trash in there already.$(tput sgr0)"
-				continue
-			fi
-			combined_trash_size=$new_combined_trash_size
-
-			# DO NOT modify these two 'echo'-ed strings below
-			# without updating 'ci/util/extract-trash-dirs.sh'
-			# as well.
-			echo "$(tput setaf 1)Start of trash directory of '$test_name':$(tput sgr0)"
-			cat "$trash_tgz_b64"
-			echo "$(tput setaf 1)End of trash directory of '$test_name'$(tput sgr0)"
-		fi
 	fi
 done
ci/run-build-and-tests.sh +0 -5
index 280dda7d285..0a1ec8c2bae 100755
@@ -5,11 +5,6 @@
 
 . ${0%/*}/lib.sh
 
-case "$CI_OS_NAME" in
-windows*) cmd //c mklink //j t\\.prove "$(cygpath -aw "$cache_dir/.prove")";;
-*) ln -s "$cache_dir/.prove" t/.prove;;
-esac
-
 export MAKE_TARGETS="all test"
 
 case "$jobname" in
ci/run-docker-build.sh deleted +0 -66

66 lines not on this page · raw message

ci/run-docker.sh deleted +0 -47

47 lines not on this page · raw message

ci/run-test-slice.sh +0 -5
index f8c2c3106a2..b9a682b4bcd 100755
@@ -5,11 +5,6 @@
 
 . ${0%/*}/lib.sh
 
-case "$CI_OS_NAME" in
-windows*) cmd //c mklink //j t\\.prove "$(cygpath -aw "$cache_dir/.prove")";;
-*) ln -s "$cache_dir/.prove" t/.prove;;
-esac
-
 make --quiet -C t T="$(cd t &&
 	./helper/test-tool path-utils slice-tests "$1" "$2" t[0-9]*.sh |
 	tr '\n' ' ')"
ci/util/extract-trash-dirs.sh deleted +0 -50

50 lines not on this page · raw message

Signature · 1 line
2.36.1.1045.gf356b5617dd
#234 lore raw

PATCH v6 4/29 CI: remove dead "tree skipping" code

In reply to Ævar Arnfjörð Bjarmason

Remove the code related to skipping CI runs if the tree was successfully tested before. With the GitHub CI this is done with the "ci-config" step, see e76eec35540 (ci: allow per-branch config for GitHub Actions, 2020-05-07) and 7d78d5fc1a9 (ci: skip GitHub workflow runs for already-tested commits/trees, 2020-10-08).

This code hasn't been used since 4a6e4b96026 (CI: remove Travis CI support, 2021-11-23), and before that for the retired Azure pipeline support removed in 6081d3898fe (ci: retire the Azure Pipelines definition, 2020-04-11).

This change is needed because this and subsequent commits are turning "ci/lib.sh" into a dumber library that'll only be tasked with setting variables for CI jobs, or for specific steps within those jobs. Now we don't need to worry about it potentially skipping the run on its own.

This change also removes a subtle potential logic error introduced in 0e7696c64db (ci: disallow directional formatting, 2021-11-04). The "ci/check-directional-formatting.bash" script would have been made to run after the "save_good_tree" in invoked in "ci/run-static-analysis.sh". So if e.g. the "make coccicheck" failed we'd still mark the tree as good.

I.e. the addition of "ci/check-directional-formatting.bash" didn't take into account that the various users of the "save_good_tree" function in ci/*.sh made a hard assumption that they're the only ci/*.sh script that's being run (and they use "set -e").

This is entirely academic since we weren't actually running this code, but is something to be careful of if anyone ever needs to resurrect parts of this.

Finally, the cache_dir="$HOME/none" added in a3f2eec862b (ci/lib:
allow running in GitHub Actions, 2020-04-08) for the GitHub CI can be removed. It appears to have been added to appease the subsequent 'mkdir -p "$cache_dir"', which wasn't needed in GitHub CI.

1. https://lore.kernel.org/git/[email protected]/

---
 ci/lib.sh                 | 91 ---------------------------------------
 ci/run-build-and-tests.sh |  2 -
 ci/run-static-analysis.sh |  2 -
 ci/test-documentation.sh  |  2 -
 4 files changed, 97 deletions(-)
ci/lib.sh +0 -91
index 2e60c0a71b5..e04b53b428d 100755
@@ -13,75 +13,6 @@ then
 	exit 1
 fi
 
-skip_branch_tip_with_tag () {
-	# Sometimes, a branch is pushed at the same time the tag that points
-	# at the same commit as the tip of the branch is pushed, and building
-	# both at the same time is a waste.
-	#
-	# When the build is triggered by a push to a tag, $CI_BRANCH will
-	# have that tagname, e.g. v2.14.0.  Let's see if $CI_BRANCH is
-	# exactly at a tag, and if so, if it is different from $CI_BRANCH.
-	# That way, we can tell if we are building the tip of a branch that
-	# is tagged and we can skip the build because we won't be skipping a
-	# build of a tag.
-
-	if TAG=$(git describe --exact-match "$CI_BRANCH" 2>/dev/null) &&
-		test "$TAG" != "$CI_BRANCH"
-	then
-		echo "$(tput setaf 2)Tip of $CI_BRANCH is exactly at $TAG$(tput sgr0)"
-		exit 0
-	fi
-}
-
-# Save some info about the current commit's tree, so we can skip the build
-# job if we encounter the same tree again and can provide a useful info
-# message.
-save_good_tree () {
-	echo "$(git rev-parse $CI_COMMIT^{tree}) $CI_COMMIT $CI_JOB_NUMBER $CI_JOB_ID" >>"$good_trees_file"
-	# limit the file size
-	tail -1000 "$good_trees_file" >"$good_trees_file".tmp
-	mv "$good_trees_file".tmp "$good_trees_file"
-}
-
-# Skip the build job if the same tree has already been built and tested
-# successfully before (e.g. because the branch got rebased, changing only
-# the commit messages).
-skip_good_tree () {
-	if test true = "$GITHUB_ACTIONS"
-	then
-		return
-	fi
-
-	if ! good_tree_info="$(grep "^$(git rev-parse $CI_COMMIT^{tree}) " "$good_trees_file")"
-	then
-		# Haven't seen this tree yet, or no cached good trees file yet.
-		# Continue the build job.
-		return
-	fi
-
-	echo "$good_tree_info" | {
-		read tree prev_good_commit prev_good_job_number prev_good_job_id
-
-		if test "$CI_JOB_ID" = "$prev_good_job_id"
-		then
-			cat <<-EOF
-			$(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0)
-			This commit has already been built and tested successfully by this build job.
-			To force a re-build delete the branch's cache and then hit 'Restart job'.
-			EOF
-		else
-			cat <<-EOF
-			$(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0)
-			This commit's tree has already been built and tested successfully in build job $prev_good_job_number for commit $prev_good_commit.
-			The log of that build job is available at $SYSTEM_TASKDEFINITIONSURI$SYSTEM_TEAMPROJECT/_build/results?buildId=$prev_good_job_id
-			To force a re-build delete the branch's cache and then hit 'Restart job'.
-			EOF
-		fi
-	}
-
-	exit 0
-}
-
 check_unignored_build_artifacts ()
 {
 	! git ls-files --other --exclude-standard --error-unmatch \
@@ -102,16 +33,8 @@ if test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI"
 then
 	CI_TYPE=azure-pipelines
 	# We are running in Azure Pipelines
-	CI_BRANCH="$BUILD_SOURCEBRANCH"
-	CI_COMMIT="$BUILD_SOURCEVERSION"
-	CI_JOB_ID="$BUILD_BUILDID"
-	CI_JOB_NUMBER="$BUILD_BUILDNUMBER"
 	CC="${CC:-gcc}"
 
-	# use a subdirectory of the cache dir (because the file share is shared
-	# among *all* phases)
-	cache_dir="$HOME/test-cache/$SYSTEM_PHASENAME"
-
 	export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save"
 	export GIT_TEST_OPTS="--verbose-log -x --write-junit-xml"
 	MAKEFLAGS="$MAKEFLAGS --jobs=10"
@@ -120,13 +43,7 @@ then
 elif test true = "$GITHUB_ACTIONS"
 then
 	CI_TYPE=github-actions
-	CI_BRANCH="$GITHUB_REF"
-	CI_COMMIT="$GITHUB_SHA"
-	CI_JOB_ID="$GITHUB_RUN_ID"
 	CC="${CC_PACKAGE:-${CC:-gcc}}"
-	DONT_SKIP_TAGS=t
-
-	cache_dir="$HOME/none"
 
 	export GIT_PROVE_OPTS="--timer --jobs 10"
 	export GIT_TEST_OPTS="--verbose-log -x"
@@ -139,14 +56,6 @@ else
 	exit 1
 fi
 
-good_trees_file="$cache_dir/good-trees"
-
-mkdir -p "$cache_dir"
-
-test -n "${DONT_SKIP_TAGS-}" ||
-skip_branch_tip_with_tag
-skip_good_tree
-
 export DEVELOPER=1
 export DEFAULT_TEST_TARGET=prove
 export GIT_TEST_CLONE_2GB=true
ci/run-build-and-tests.sh +0 -2
index 0a1ec8c2bae..e5a21985b44 100755
@@ -45,5 +45,3 @@ esac
 # start running tests.
 make $MAKE_TARGETS
 check_unignored_build_artifacts
-
-save_good_tree
ci/run-static-analysis.sh +0 -2
index 65bcebda41a..5a87b1cac96 100755
@@ -28,5 +28,3 @@ fi
 
 make hdr-check ||
 exit 1
-
-save_good_tree
ci/test-documentation.sh +0 -2
index de41888430a..9e0652c30dd 100755
@@ -41,5 +41,3 @@ grep '<meta name="generator" content="Asciidoctor ' Documentation/git.html
 
 rm -f stdout.log stderr.log stderr.raw
 check_unignored_build_artifacts
-
-save_good_tree
Signature · 1 line
2.36.1.1045.gf356b5617dd
#235 lore raw

PATCH v6 5/29 CI: remove unused Azure ci/* code

In reply to Ævar Arnfjörð Bjarmason

Remove Azure-specific code that's been unused since 6081d3898fe (ci:
retire the Azure Pipelines definition, 2020-04-11). As noted in a larger removal of all of the Azure-supporting code in [1] (although that missed some of this) there is more of it in-tree, but let's focus on only the ci/* code for now.

This is needed because in subsequent commits this unused code would either need to be changed to accommodate changes in ci/*, or be removed.

As we'll see in those subsequent commits the end-state we're heading towards will actually make it easier to add new CI types in the future, even though the only one we're left with now is the GitHub CI. I.e. the "ci/*" framework will be made to do less, not more. We'll be offloading more of what it does to our generic build and test system.

While I'm at it (since the line needs to be touched anyway) change an odd 'if test true == "$GITHUB_ACTIONS"' to the more usual style used in other places of 'if test "$GITHUB_ACTIONS" = "true"'.

1. https://lore.kernel.org/git/[email protected]/

---
 ci/install-dependencies.sh |  3 ---
 ci/lib.sh                  | 13 +------------
 ci/mount-fileshare.sh      | 25 -------------------------
 ci/print-test-failures.sh  |  4 ----
 4 files changed, 1 insertion(+), 44 deletions(-)
 delete mode 100755 ci/mount-fileshare.sh
ci/install-dependencies.sh +0 -3
index 107757a1fea..4fa3e282dda 100755
@@ -34,8 +34,6 @@ macos-latest)
 	export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1
 	# Uncomment this if you want to run perf tests:
 	# brew install gnu-time
-	test -z "$BREW_INSTALL_PACKAGES" ||
-	brew install $BREW_INSTALL_PACKAGES
 	brew link --force gettext
 	mkdir -p $HOME/bin
 	(
@@ -71,7 +69,6 @@ Documentation)
 	sudo apt-get -q update
 	sudo apt-get -q -y install asciidoc xmlto docbook-xsl-ns make
 
-	test -n "$ALREADY_HAVE_ASCIIDOCTOR" ||
 	sudo gem install --version 1.5.8 asciidoctor
 	;;
 linux-gcc-default)
ci/lib.sh +1 -12
index e04b53b428d..ebb9461ff42 100755
@@ -29,18 +29,7 @@ export TERM=${TERM:-dumb}
 # Clear MAKEFLAGS that may come from the outside world.
 export MAKEFLAGS=
 
-if test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI"
-then
-	CI_TYPE=azure-pipelines
-	# We are running in Azure Pipelines
-	CC="${CC:-gcc}"
-
-	export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save"
-	export GIT_TEST_OPTS="--verbose-log -x --write-junit-xml"
-	MAKEFLAGS="$MAKEFLAGS --jobs=10"
-	test Windows_NT != "$AGENT_OS" ||
-	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
-elif test true = "$GITHUB_ACTIONS"
+if test "$GITHUB_ACTIONS" = "true"
 then
 	CI_TYPE=github-actions
 	CC="${CC_PACKAGE:-${CC:-gcc}}"
ci/mount-fileshare.sh deleted +0 -25
deleted file mode 100755
index 26b58a80960..00000000000
@@ -1,25 +0,0 @@
-#!/bin/sh
-
-die () {
-	echo "$*" >&2
-	exit 1
-}
-
-test $# = 4 ||
-die "Usage: $0 <share> <username> <password> <mountpoint>"
-
-mkdir -p "$4" || die "Could not create $4"
-
-case "$(uname -s)" in
-Linux)
-	sudo mount -t cifs -o vers=3.0,username="$2",password="$3",dir_mode=0777,file_mode=0777,serverino "$1" "$4"
-	;;
-Darwin)
-	pass="$(echo "$3" | sed -e 's/\//%2F/g' -e 's/+/%2B/g')" &&
-	mount -t smbfs,soft "smb://$2:$pass@${1#//}" "$4"
-	;;
-*)
-	die "No support for $(uname -s)"
-	;;
-esac ||
-die "Could not mount $4"
ci/print-test-failures.sh +0 -4
index 740ebb7c529..47e2ea1ad10 100755
@@ -30,10 +30,6 @@ do
 		test_name="${test_name##*/}"
 		trash_dir="trash directory.$test_name"
 		case "$CI_TYPE" in
-		azure-pipelines)
-			mkdir -p failed-test-artifacts
-			mv "$trash_dir" failed-test-artifacts
-			;;
 		github-actions)
 			mkdir -p failed-test-artifacts
 			echo "FAILED_TEST_ARTIFACTS=t/failed-test-artifacts" >>$GITHUB_ENV
Signature · 1 line
2.36.1.1045.gf356b5617dd
#236 lore raw

PATCH v6 6/29 CI/lib.sh: stop adding leading whitespace to $MAKEFLAGS

In reply to Ævar Arnfjörð Bjarmason

In a8c51f77d16 (ci: clear and mark MAKEFLAGS exported just once, 2019-02-07) the assignment of MAKEFLAGS was done up-front to future-proof this code for the bug fixed in 406f93ae488 (ci: make sure we build Git parallel, 2019-02-07).

But since 4a6e4b96026 (CI: remove Travis CI support, 2021-11-23) the --jobs=10 has been an invariant here, so let's assign --jobs=$NPROC to the MAKEFLAGS right away, this gets rid of the leading whitespace from the $MAKEFLAGS variable, as it would previously be empty on the first assignment.

---
 ci/lib.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)
ci/lib.sh +5 -3
index ebb9461ff42..5f95f55a147 100755
@@ -26,17 +26,19 @@ check_unignored_build_artifacts ()
 # GitHub Action doesn't set TERM, which is required by tput
 export TERM=${TERM:-dumb}
 
+# How many jobs to run in parallel?
+NPROC=10
+
 # Clear MAKEFLAGS that may come from the outside world.
-export MAKEFLAGS=
+export MAKEFLAGS=--jobs=$NPROC
 
 if test "$GITHUB_ACTIONS" = "true"
 then
 	CI_TYPE=github-actions
 	CC="${CC_PACKAGE:-${CC:-gcc}}"
 
-	export GIT_PROVE_OPTS="--timer --jobs 10"
+	export GIT_PROVE_OPTS="--timer --jobs $NPROC"
 	export GIT_TEST_OPTS="--verbose-log -x"
-	MAKEFLAGS="$MAKEFLAGS --jobs=10"
 	test Windows != "$RUNNER_OS" ||
 	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
 else
Signature · 1 line
2.36.1.1045.gf356b5617dd
#237 lore raw

PATCH v6 7/29 CI: don't have "git grep" invoke a pager in tree content check

In reply to Ævar Arnfjörð Bjarmason

Fix the omission of not adding "-P" (a.k.a. "--no-pager") to the "git" invocation that spaws "git grep", added in 0e7696c64db (ci: disallow directional formatting, 2021-11-04).

This didn't matter for the GitHub CI, but does when running the check locally. A subsequent commit will start invoking it via a new check from the top-level Makefile, and without this we'd visibly spawn a pager when doing so.

---
 ci/check-directional-formatting.bash | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
ci/check-directional-formatting.bash +1 -1
index e6211b141aa..de8e6e486f0 100755
@@ -23,5 +23,5 @@
 # U+2066..U+2069: LRI, RLI, FSI and PDI
 regex='(\u202a|\u202b|\u202c|\u202d|\u202e|\u2066|\u2067|\u2068|\u2069)'
 
-! LC_CTYPE=C git grep -El "$(LC_CTYPE=C.UTF-8 printf "$regex")" \
+! LC_CTYPE=C git -P grep -El "$(LC_CTYPE=C.UTF-8 printf "$regex")" \
 	-- ':(exclude,attr:binary)' ':(exclude)*.po'
Signature · 1 line
2.36.1.1045.gf356b5617dd
#238 lore raw

PATCH v6 8/29 CI: have "static-analysis" run a "make ci-static-analysis" target

In reply to Ævar Arnfjörð Bjarmason

Make it easier to run the CI logic locally by creating "ci-static-analysis" target.

Now the "check-coccicheck" logic added in 0860a7641ba (travis-ci: fail if Coccinelle static analysis found something to transform, 2018-07-23) can be easily tested outside of CI, and likewise the sanity check added in 0e7696c64db (ci: disallow directional formatting, 2021-11-04).

These targets could be improved, the "hdr-check" target will re-do all of its work every time it's run, and could with a change similar to my c234e8a0ecf (Makefile: make the "sparse" target non-.PHONY, 2021-09-23). The same goes for the new "ci-check-directional-formatting" target.

But without those improvements being able to easily run a 1=1 equivalent of the checks we do in CI during a local build is already an improvement. Subsequent commits will expand on this ability.

This change will also make the CI check faster, since we can take advantage of parallelism. The "make coccicheck" takes a very long time, and at the tail end we might only have one job pegging 100% of one CPU core for a long time. Now any extra cores will be free to run the rest of the targets under "ci-static-analysis".

---
 .github/workflows/main.yml |  3 +--
 Makefile                   | 28 ++++++++++++++++++++++++++++
 ci/run-static-analysis.sh  | 30 ------------------------------
 shared.mak                 |  1 +
 4 files changed, 30 insertions(+), 32 deletions(-)
 delete mode 100755 ci/run-static-analysis.sh
.github/workflows/main.yml +1 -2
index 309b9141249..ad752010102 100644
@@ -317,8 +317,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/run-static-analysis.sh
-    - run: ci/check-directional-formatting.bash
+    - run: make ci-static-analysis
   sparse:
     needs: ci-config
     if: needs.ci-config.outputs.enabled == 'yes'
Makefile +28 -0
index 61aadf3ce88..9ae7e1a373e 100644
@@ -3034,6 +3034,20 @@ coccicheck: $(addsuffix .patch,$(filter-out %.pending.cocci,$(wildcard contrib/c
 # See contrib/coccinelle/README
 coccicheck-pending: $(addsuffix .patch,$(wildcard contrib/coccinelle/*.pending.cocci))
 
+.PHONY: check-coccicheck
+check-coccicheck: coccicheck
+	$(QUIET_CHECK)for cocci_patch in contrib/coccinelle/*.patch; do \
+		if test -s "$$cocci_patch"; then \
+			echo "Coccinelle suggests the following changes in '$$cocci_patch':"; \
+			cat "$$cocci_patch"; \
+			fail=UnfortunatelyYes; \
+		fi \
+	done; \
+	if test -n "$$fail"; then \
+		echo "error: Coccinelle suggested some changes"; \
+		exit 1; \
+	fi
+
 .PHONY: coccicheck coccicheck-pending
 
 ### Installation rules
@@ -3469,3 +3483,17 @@ $(FUZZ_PROGRAMS): all
 		$(XDIFF_OBJS) $(EXTLIBS) git.o [email protected] $(LIB_FUZZING_ENGINE) -o $@
 
 fuzz-all: $(FUZZ_PROGRAMS)
+
+### CI "check" targets
+#
+# These targets are run from the CI, see .github/workflows/main.yml,
+# but can also be run manually to run the same assertions locally.
+
+.PHONY: ci-check-directional-formatting
+ci-check-directional-formatting:
+	$(QUIET_CHECK)ci/check-directional-formatting.bash
+
+.PHONY: ci-static-analysis
+ci-static-analysis: ci-check-directional-formatting
+ci-static-analysis: check-coccicheck
+ci-static-analysis: hdr-check
ci/run-static-analysis.sh deleted +0 -30
deleted file mode 100755
index 5a87b1cac96..00000000000
@@ -1,30 +0,0 @@
-#!/bin/sh
-#
-# Perform various static code analysis checks
-#
-
-. ${0%/*}/lib.sh
-
-make coccicheck
-
-set +x
-
-fail=
-for cocci_patch in contrib/coccinelle/*.patch
-do
-	if test -s "$cocci_patch"
-	then
-		echo "$(tput setaf 1)Coccinelle suggests the following changes in '$cocci_patch':$(tput sgr0)"
-		cat "$cocci_patch"
-		fail=UnfortunatelyYes
-	fi
-done
-
-if test -n "$fail"
-then
-	echo "$(tput setaf 1)error: Coccinelle suggested some changes$(tput sgr0)"
-	exit 1
-fi
-
-make hdr-check ||
-exit 1
shared.mak +1 -0
index 50d4596f0d6..83e0aa16e7d 100644
@@ -68,6 +68,7 @@ ifndef V
 	QUIET_HDR      = @echo '   ' HDR $(<:hcc=h);
 	QUIET_RC       = @echo '   ' RC $@;
 	QUIET_SPATCH   = @echo '   ' SPATCH $<;
+	QUIET_CHECK    = @echo '   ' CHECK $@;
 
 ## Used in "Documentation/Makefile"
 	QUIET_ASCIIDOC	= @echo '   ' ASCIIDOC $@;
Signature · 1 line
2.36.1.1045.gf356b5617dd
#239 lore raw

PATCH v6 9/29 CI: have "static-analysis" run "check-builtins", not "documentation"

In reply to Ævar Arnfjörð Bjarmason

Move the "make check-builtins" check from the "documentation" job to the "static-analysis" job.

The "check-builtins" target added in c74390e4a1d (cherry is built-in, do not ship git-cherry.sh, 2006-11-05) is unrelated to the documentation, so it's odd that b98712b9aa9 (travis-ci: build documentation, 2016-05-04) added it to the "documentation" job.

Let's just move it to the "ci-static-analysis" target, and while we're at it improve the output with $(QUIET_CHECK).

---
 Makefile                 | 3 ++-
 ci/test-documentation.sh | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)
Makefile +2 -1
index 9ae7e1a373e..3398d3f2e56 100644
@@ -3405,7 +3405,7 @@ check-docs::
 ### Make sure built-ins do not have dups and listed in git.c
 #
 check-builtins::
-	./check-builtins.sh
+	$(QUIET_CHECK)./check-builtins.sh
 
 ### Test suite coverage testing
 #
@@ -3495,5 +3495,6 @@ ci-check-directional-formatting:
 
 .PHONY: ci-static-analysis
 ci-static-analysis: ci-check-directional-formatting
+ci-static-analysis: check-builtins
 ci-static-analysis: check-coccicheck
 ci-static-analysis: hdr-check
ci/test-documentation.sh +0 -1
index 9e0652c30dd..41e2b126311 100755
@@ -14,7 +14,6 @@ filter_log () {
 	    "$1"
 }
 
-make check-builtins
 make check-docs
 
 # Build docs with AsciiDoc
Signature · 1 line
2.36.1.1045.gf356b5617dd
#240 lore raw

PATCH v6 10/29 CI: move p4 and git-lfs variables to ci/install-dependencies.sh

In reply to Ævar Arnfjörð Bjarmason

Move the declaration of variables that are only used by the "ubuntu-latest" block in "ci/install-dependencies.sh" there from "ci/lib.sh".

This makes the code easier to follow, and changes "ci/lib.sh" to a library that only exports CI variables for general use, we didn't need to export these $P4_PATH and $GIT_LFS_PATH variables.

---
 ci/install-dependencies.sh | 17 +++++++++++++++--
 ci/lib.sh                  | 12 ------------
 2 files changed, 15 insertions(+), 14 deletions(-)
ci/install-dependencies.sh +15 -2
index 4fa3e282dda..41a9185bb1e 100755
@@ -5,14 +5,27 @@
 
 . ${0%/*}/lib.sh
 
-P4WHENCE=https://cdist2.perforce.com/perforce/r$LINUX_P4_VERSION
-LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION
 UBUNTU_COMMON_PKGS="make libssl-dev libcurl4-openssl-dev libexpat-dev
  tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl
  libemail-valid-perl libio-socket-ssl-perl libnet-smtp-ssl-perl"
 
 case "$runs_on_pool" in
 ubuntu-latest)
+	# The Linux build installs the defined dependency versions below.
+	# The OS X build installs much more recent versions, whichever
+	# were recorded in the Homebrew database upon creating the OS X
+	# image.
+	# Keep that in mind when you encounter a broken OS X build!
+	LINUX_P4_VERSION="16.2"
+	LINUX_GIT_LFS_VERSION="1.5.2"
+
+	P4_PATH="$HOME/custom/p4"
+	GIT_LFS_PATH="$HOME/custom/git-lfs"
+	export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH"
+
+	P4WHENCE=https://cdist2.perforce.com/perforce/r$LINUX_P4_VERSION
+	LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION
+
 	sudo apt-get -q update
 	sudo apt-get -q -y install language-pack-is libsvn-perl apache2 \
 		$UBUNTU_COMMON_PKGS $CC_PACKAGE
ci/lib.sh +0 -12
index 5f95f55a147..e789cd2a1cd 100755
@@ -67,18 +67,6 @@ ubuntu-latest)
 	fi
 
 	export GIT_TEST_HTTPD=true
-
-	# The Linux build installs the defined dependency versions below.
-	# The OS X build installs much more recent versions, whichever
-	# were recorded in the Homebrew database upon creating the OS X
-	# image.
-	# Keep that in mind when you encounter a broken OS X build!
-	export LINUX_P4_VERSION="16.2"
-	export LINUX_GIT_LFS_VERSION="1.5.2"
-
-	P4_PATH="$HOME/custom/p4"
-	GIT_LFS_PATH="$HOME/custom/git-lfs"
-	export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH"
 	;;
 macos-latest)
 	if [ "$jobname" = osx-gcc ]
Signature · 1 line
2.36.1.1045.gf356b5617dd
#241 lore raw

PATCH v6 11/29 CI: consistently use "export" in ci/lib.sh

In reply to Ævar Arnfjörð Bjarmason

Change the "ci/lib.sh" script co consistently use "export", for e.g. MAKEFLAGS we were exporting it, and then assigning to it, let's do it the other way around.

Right now this doesn't matter, since we in
e.g. "ci/install-dependencies.sh" source this file, and don't use something like "env(1)" to retrieve these variables. But in a subsequent commit we'll "export" these variables through a wrapper (to additionally write them to a GitHub CI-specific $GITHUB_ENV file). This change makes that subsequent change easier to read, as it won't need to do any control flow refactoring.

---
 ci/lib.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)
ci/lib.sh +4 -3
index e789cd2a1cd..1b5b805c5ca 100755
@@ -30,7 +30,7 @@ export TERM=${TERM:-dumb}
 NPROC=10
 
 # Clear MAKEFLAGS that may come from the outside world.
-export MAKEFLAGS=--jobs=$NPROC
+MAKEFLAGS=--jobs=$NPROC
 
 if test "$GITHUB_ACTIONS" = "true"
 then
@@ -38,9 +38,10 @@ then
 	CC="${CC_PACKAGE:-${CC:-gcc}}"
 
 	export GIT_PROVE_OPTS="--timer --jobs $NPROC"
-	export GIT_TEST_OPTS="--verbose-log -x"
+	GIT_TEST_OPTS="--verbose-log -x"
 	test Windows != "$RUNNER_OS" ||
 	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
+	export GIT_TEST_OPTS
 else
 	echo "Could not identify CI type" >&2
 	env >&2
@@ -94,4 +95,4 @@ linux-leaks)
 	;;
 esac
 
-MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}"
+export MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}"
Signature · 1 line
2.36.1.1045.gf356b5617dd
#242 lore raw

PATCH v6 12/29 CI: export variables via a wrapper

In reply to Ævar Arnfjörð Bjarmason

Add and use a "setenv" wrapper to export the variables set in ci/lib.sh.

For now this merely adds debugging information about what variables are set, but in a subsequent commit we'll expand this to set variables in $GITHUB_ENV instead of the environment, and thus carry variable settings between steps in a job.

We'll also eventually be interested in what type of variable we're setting, and when it needs to be used, because we'll prominently expose what variables we set in what CI "step", so having only the ones we actually use will improve readability.

To that end call the "setenv" function with optional "--build", "--test" and "--all" parameters to indicate whether they're needed for building, testing or both.

We currently ignore the "varname" parameter, but will make use of it in a subsequent commit. Adding it here reduces the needless churn, as we won't need to change all these lines again for that subsequent change.

---
 ci/lib.sh                 | 51 ++++++++++++++++++++++++++++++---------
 ci/run-build-and-tests.sh | 34 +++++++++++++-------------
 2 files changed, 57 insertions(+), 28 deletions(-)
ci/lib.sh +40 -11
index 1b5b805c5ca..71e6d6e08e7 100755
@@ -13,6 +13,35 @@ then
 	exit 1
 fi
 
+# Helper functions
+setenv () {
+	while test $# != 0
+	do
+		case "$1" in
+		--build)
+			;;
+		--test)
+			;;
+		--all)
+			;;
+		-*)
+			echo "BUG: bad setenv() option '$1'" >&2
+			exit 1
+			;;
+		*)
+			break
+			;;
+		esac
+		shift
+	done
+
+	key=$1
+	val=$2
+	shift 2
+
+	eval "export $key=\"$val\""
+}
+
 check_unignored_build_artifacts ()
 {
 	! git ls-files --other --exclude-standard --error-unmatch \
@@ -24,7 +53,7 @@ check_unignored_build_artifacts ()
 }
 
 # GitHub Action doesn't set TERM, which is required by tput
-export TERM=${TERM:-dumb}
+setenv TERM ${TERM:-dumb}
 
 # How many jobs to run in parallel?
 NPROC=10
@@ -37,21 +66,21 @@ then
 	CI_TYPE=github-actions
 	CC="${CC_PACKAGE:-${CC:-gcc}}"
 
-	export GIT_PROVE_OPTS="--timer --jobs $NPROC"
+	setenv --test GIT_PROVE_OPTS "--timer --jobs $NPROC"
 	GIT_TEST_OPTS="--verbose-log -x"
 	test Windows != "$RUNNER_OS" ||
 	GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
-	export GIT_TEST_OPTS
+	setenv --test GIT_TEST_OPTS "$GIT_TEST_OPTS"
 else
 	echo "Could not identify CI type" >&2
 	env >&2
 	exit 1
 fi
 
-export DEVELOPER=1
-export DEFAULT_TEST_TARGET=prove
-export GIT_TEST_CLONE_2GB=true
-export SKIP_DASHED_BUILT_INS=YesPlease
+setenv --build DEVELOPER 1
+setenv --test DEFAULT_TEST_TARGET prove
+setenv --test GIT_TEST_CLONE_2GB true
+setenv --build SKIP_DASHED_BUILT_INS YesPlease
 
 case "$runs_on_pool" in
 ubuntu-latest)
@@ -67,7 +96,7 @@ ubuntu-latest)
 		MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python2"
 	fi
 
-	export GIT_TEST_HTTPD=true
+	setenv --test GIT_TEST_HTTPD true
 	;;
 macos-latest)
 	if [ "$jobname" = osx-gcc ]
@@ -90,9 +119,9 @@ linux-musl)
 	MAKEFLAGS="$MAKEFLAGS GIT_TEST_UTF8_LOCALE=C.UTF-8"
 	;;
 linux-leaks)
-	export SANITIZE=leak
-	export GIT_TEST_PASSING_SANITIZE_LEAK=true
+	setenv --build SANITIZE leak
+	setenv --test GIT_TEST_PASSING_SANITIZE_LEAK true
 	;;
 esac
 
-export MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}"
+setenv --all MAKEFLAGS "$MAKEFLAGS CC=${CC:-cc}"
ci/run-build-and-tests.sh +17 -17
index e5a21985b44..35d45a9373e 100755
@@ -9,33 +9,33 @@ export MAKE_TARGETS="all test"
 
 case "$jobname" in
 linux-gcc)
-	export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
+	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME main
 	;;
 linux-TEST-vars)
-	export GIT_TEST_SPLIT_INDEX=yes
-	export GIT_TEST_MERGE_ALGORITHM=recursive
-	export GIT_TEST_FULL_IN_PACK_ARRAY=true
-	export GIT_TEST_OE_SIZE=10
-	export GIT_TEST_OE_DELTA_SIZE=5
-	export GIT_TEST_COMMIT_GRAPH=1
-	export GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS=1
-	export GIT_TEST_MULTI_PACK_INDEX=1
-	export GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=1
-	export GIT_TEST_ADD_I_USE_BUILTIN=1
-	export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master
-	export GIT_TEST_WRITE_REV_INDEX=1
-	export GIT_TEST_CHECKOUT_WORKERS=2
+	setenv --test GIT_TEST_SPLIT_INDEX yes
+	setenv --test GIT_TEST_MERGE_ALGORITHM recursive
+	setenv --test GIT_TEST_FULL_IN_PACK_ARRAY true
+	setenv --test GIT_TEST_OE_SIZE 10
+	setenv --test GIT_TEST_OE_DELTA_SIZE 5
+	setenv --test GIT_TEST_COMMIT_GRAPH 1
+	setenv --test GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS 1
+	setenv --test GIT_TEST_MULTI_PACK_INDEX 1
+	setenv --test GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP 1
+	setenv --test GIT_TEST_ADD_I_USE_BUILTIN 1
+	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME master
+	setenv --test GIT_TEST_WRITE_REV_INDEX 1
+	setenv --test GIT_TEST_CHECKOUT_WORKERS 2
 	;;
 linux-clang)
-	export GIT_TEST_DEFAULT_HASH=sha1
+	setenv --test GIT_TEST_DEFAULT_HASH sha1
 	;;
 linux-sha256)
-	export GIT_TEST_DEFAULT_HASH=sha256
+	setenv --test GIT_TEST_DEFAULT_HASH sha256
 	;;
 pedantic)
 	# Don't run the tests; we only care about whether Git can be
 	# built.
-	export DEVOPTS=pedantic
+	setenv --build DEVOPTS pedantic
 	export MAKE_TARGETS=all
 	;;
 esac
Signature · 1 line
2.36.1.1045.gf356b5617dd
#243 lore raw

PATCH v6 13/29 CI: remove "run-build-and-tests.sh", run "make [test]" directly

In reply to Ævar Arnfjörð Bjarmason

Remove the already thin "ci/run-build-and-tests.sh" wrapper and instead make the CI run "make" or "make test" directly. By doing this we'll be able to easily see at a glance whether our failure was in the compilation or testing, whether that's via human eyes or improve machine readability.

We also need to run our new "ci/check-unignored-build-artifacts.sh" on success() in the CI now, just like we already had a step conditional on failure() running ci/print-test-failures.sh.

The reason we used a "ci/run-build-and-tests.sh" wrapper in the first place had to do with Travis CI-specific constraints that no longer apply to us, as the Travis CI support has been removed.

Instead we can configure the CI in an earlier step by running "ci/lib.sh", which under GitHub CI will write the environment variables we need to the "$GITHUB_ENV" file.

We'll then have access to them in subsequent steps, and crucially those variables will be prominently visible at the start of each step via an expandable drop-down in the UI.drop-do.

I.e. this changes the CI run from a top-down flow like (pseudocode):

- job:
- step1:
- use ci/lib.sh to set env vars
- run a script like ci/run-build-and-tests.sh
- step2:
- if: failure()
- use ci/lib.sh to set env vars
- run ci/print-test-failures.sh

To:

- job:
- step1:
- set variables in $GITHUB_ENV using ci/lib.sh
- step2:
- make
- step3:
- make test
- step4:
- if: failure()
- run ci/print-test-failures.sh
- step5:
- if: success()
- run ci/check-unignored-build-artifacts.sh

There is a proposal[2] to get some of the benefits of this approach by not re-arranging our variable setup in this way, but to instead use the GitHub CI grouping syntax to focus on the relevant parts of "make" or "make test" when we have failures.

Doing it this way makes for better looking GitHub CI UI, and lays much better ground work for our CI going forward. Because:

* The CI logic will be more portable to a future CI system, since a common feature of them is to run various commands in sequence, but a future system won't necessarily support the GitHub-specifics syntax of "grouping" output within a "step".

Even if those systems don't support a "$GITHUB_ENV" emulating will be much easier than to deal with some CI-specific grouping syntax.

* At the start of every step the GitHub CI presents an expandable list of environment variables from "$GITHUB_ENV". We'll now see exactly what variables affected that step (although we currently overshoot that a bit, and always define all variables).

* CI failures will be easier to reproduce locally, as this makes the relevant ci/* scripts something that sets up our environment, but leaves "make" and "make test" working as they do locally.

To reproduce a run the user only needs to set the variables discussed in the drop-down above, either manually or by running "ci/lib.sh".

* The output will be less verbose. The "ci/lib.sh" script uses "set -x", and before this e.g. "ci/static-analysis.sh" would start with 40 lines of trace output, culminating in using "export" to export the relevant environment variables.

Now that verbosity is in the earlier "ci/lib.sh" step, and not in any subsequent one. The "make" targets then start out with the relevant non-trace output right away.

* If we do want to use the grouping syntax within a "step" it'll now be easier to do so. It doesn't support nesting, so we'd have to make a choice between using it for e.g. "make" v.s. "make test", or individual test failures. See "sadly" in [3].

1. https://lore.kernel.org/git/[email protected]/
2. https://lore.kernel.org/git/[email protected]/
3. https://lore.kernel.org/git/9333ba781b8240f704e739b00d274f8c3d887e39.1643050574.git.gitgitgadget@gmail.com/

---
 .github/workflows/main.yml            | 14 ++++++--
 ci/check-unignored-build-artifacts.sh |  8 +++++
 ci/install-dependencies.sh            |  4 +++
 ci/lib.sh                             | 36 +++++++++++++++++++-
 ci/make-test-artifacts.sh             |  2 --
 ci/run-build-and-tests.sh             | 47 ---------------------------
 ci/run-test-slice.sh                  |  2 --
 ci/test-documentation.sh              |  1 -
 8 files changed, 58 insertions(+), 56 deletions(-)
 create mode 100755 ci/check-unignored-build-artifacts.sh
 delete mode 100755 ci/run-build-and-tests.sh
.github/workflows/main.yml +11 -3
index ad752010102..57b2db06923 100644
@@ -268,7 +268,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/run-build-and-tests.sh
+    - run: ci/lib.sh
+    - run: make
+    - run: make test
+      if: success()
     - run: ci/print-test-failures.sh
       if: failure()
     - name: Upload failed tests' directories
@@ -292,6 +295,7 @@ jobs:
           image: daald/ubuntu32:xenial
         - jobname: pedantic
           image: fedora
+          skip-tests: yes
     env:
       jobname: ${{matrix.vector.jobname}}
     runs-on: ubuntu-latest
@@ -299,9 +303,12 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - run: ci/install-docker-dependencies.sh
-    - run: ci/run-build-and-tests.sh
+    - run: ci/lib.sh
+    - run: make
+    - run: make test
+      if: success() && matrix.vector.skip-tests != 'yes'
     - run: ci/print-test-failures.sh
-      if: failure()
+      if: failure() && matrix.vector.skip-tests != 'yes'
     - name: Upload failed tests' directories
       if: failure() && env.FAILED_TEST_ARTIFACTS != ''
       uses: actions/upload-artifact@v1
@@ -317,6 +324,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
+    - run: ci/lib.sh
     - run: make ci-static-analysis
   sparse:
     needs: ci-config
ci/check-unignored-build-artifacts.sh new file +8 -0
new file mode 100755
index 00000000000..56d04b0db9a
@@ -0,0 +1,8 @@
+#!/bin/sh
+#
+# Check whether the build created anything not in our .gitignore
+#
+
+. ${0%/*}/lib.sh
+
+check_unignored_build_artifacts
ci/install-dependencies.sh +4 -0
index 41a9185bb1e..5d50604e724 100755
@@ -22,6 +22,10 @@ ubuntu-latest)
 	P4_PATH="$HOME/custom/p4"
 	GIT_LFS_PATH="$HOME/custom/git-lfs"
 	export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH"
+	if test -n "$GITHUB_PATH"
+	then
+		echo "$PATH" >>"$GITHUB_PATH"
+	fi
 
 	P4WHENCE=https://cdist2.perforce.com/perforce/r$LINUX_P4_VERSION
 	LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION
ci/lib.sh +35 -1
index 71e6d6e08e7..d18ee1a91e7 100755
@@ -1,3 +1,5 @@
+#!/bin/sh
+
 # Library of functions shared by all CI scripts
 
 # Set 'exit on error' for all CI scripts to let the caller know that
@@ -39,7 +41,10 @@ setenv () {
 	val=$2
 	shift 2
 
-	eval "export $key=\"$val\""
+	if test -n "$GITHUB_ENV"
+	then
+		echo "$key=$val" >>"$GITHUB_ENV"
+	fi
 }
 
 check_unignored_build_artifacts ()
@@ -109,6 +114,35 @@ macos-latest)
 esac
 
 case "$jobname" in
+linux-gcc)
+	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME main
+	;;
+linux-TEST-vars)
+	setenv --test GIT_TEST_SPLIT_INDEX yes
+	setenv --test GIT_TEST_MERGE_ALGORITHM recursive
+	setenv --test GIT_TEST_FULL_IN_PACK_ARRAY true
+	setenv --test GIT_TEST_OE_SIZE 10
+	setenv --test GIT_TEST_OE_DELTA_SIZE 5
+	setenv --test GIT_TEST_COMMIT_GRAPH 1
+	setenv --test GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS 1
+	setenv --test GIT_TEST_MULTI_PACK_INDEX 1
+	setenv --test GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP 1
+	setenv --test GIT_TEST_ADD_I_USE_BUILTIN 1
+	setenv --test GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME master
+	setenv --test GIT_TEST_WRITE_REV_INDEX 1
+	setenv --test GIT_TEST_CHECKOUT_WORKERS 2
+	;;
+linux-clang)
+	setenv --test GIT_TEST_DEFAULT_HASH sha1
+	;;
+linux-sha256)
+	setenv --test GIT_TEST_DEFAULT_HASH sha256
+	;;
+pedantic)
+	# Don't run the tests; we only care about whether Git can be
+	# built.
+	setenv --build DEVOPTS pedantic
+	;;
 linux32)
 	CC=gcc
 	;;
ci/make-test-artifacts.sh +0 -2
index 646967481f6..45298562982 100755
@@ -8,5 +8,3 @@ mkdir -p "$1" # in case ci/lib.sh decides to quit early
 . ${0%/*}/lib.sh
 
 make artifacts-tar ARTIFACTS_DIRECTORY="$1"
-
-check_unignored_build_artifacts
ci/run-build-and-tests.sh deleted +0 -47

47 lines not on this page · raw message

ci/run-test-slice.sh +0 -2
index b9a682b4bcd..70326961454 100755
@@ -8,5 +8,3 @@
 make --quiet -C t T="$(cd t &&
 	./helper/test-tool path-utils slice-tests "$1" "$2" t[0-9]*.sh |
 	tr '\n' ' ')"
-
-check_unignored_build_artifacts
ci/test-documentation.sh +0 -1
index 41e2b126311..6b5cce03bd7 100755
@@ -39,4 +39,3 @@ test -s Documentation/git.html
 grep '<meta name="generator" content="Asciidoctor ' Documentation/git.html
 
 rm -f stdout.log stderr.log stderr.raw
-check_unignored_build_artifacts
Signature · 1 line
2.36.1.1045.gf356b5617dd
#244 lore raw

PATCH v6 14/29 CI: make ci/{lib,install-dependencies}.sh POSIX-compatible

In reply to Ævar Arnfjörð Bjarmason

Change the unportable constructs in the "ci/install-dependencies.sh" scripts to use POSIX replacements, and have it run under "/bin/sh" instead of "/usr/bin/env bash". We needed to:

* Replace pushd/popd with a sub-shell.
* Change "mkdir --parents" to "mkdir -p".
* Use POSIX "export" syntax.

This is in preparation for combining this script and "ci/install-docker-dependencies.sh", which doesn't have access to "bash".

While we're at it change "if [" to "if test" in ci/lib.sh. This isn't strictly speaking a case of POSIX-incompatibility, but that syntax was being used there because this code was using idiomatic bash constructs. Let's have it use our usual style of "if test" instead.

---
 ci/install-dependencies.sh | 27 +++++++++++++++++----------
 ci/lib.sh                  |  4 ++--
 2 files changed, 19 insertions(+), 12 deletions(-)
ci/install-dependencies.sh +17 -10
index 5d50604e724..3e05d38d4ab 100755
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh
 #
 # Install dependencies required to build and test Git on Linux and macOS
 #
@@ -21,7 +21,8 @@ ubuntu-latest)
 
 	P4_PATH="$HOME/custom/p4"
 	GIT_LFS_PATH="$HOME/custom/git-lfs"
-	export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH"
+	PATH="$GIT_LFS_PATH:$P4_PATH:$PATH"
+	export PATH
 	if test -n "$GITHUB_PATH"
 	then
 		echo "$PATH" >>"$GITHUB_PATH"
@@ -33,22 +34,28 @@ ubuntu-latest)
 	sudo apt-get -q update
 	sudo apt-get -q -y install language-pack-is libsvn-perl apache2 \
 		$UBUNTU_COMMON_PKGS $CC_PACKAGE
-	mkdir --parents "$P4_PATH"
-	pushd "$P4_PATH"
+	mkdir -p "$P4_PATH"
+	(
+		cd "$P4_PATH"
 		wget --quiet "$P4WHENCE/bin.linux26x86_64/p4d"
 		wget --quiet "$P4WHENCE/bin.linux26x86_64/p4"
 		chmod u+x p4d
 		chmod u+x p4
-	popd
-	mkdir --parents "$GIT_LFS_PATH"
-	pushd "$GIT_LFS_PATH"
+	)
+	mkdir -p "$GIT_LFS_PATH"
+	(
+		cd "$GIT_LFS_PATH"
 		wget --quiet "$LFSWHENCE/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz"
 		tar --extract --gunzip --file "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz"
 		cp git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs .
-	popd
+	)
 	;;
 macos-latest)
-	export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1
+	HOMEBREW_NO_AUTO_UPDATE=1
+	export HOMEBREW_NO_AUTO_UPDATE
+	HOMEBREW_NO_INSTALL_CLEANUP=1
+	export HOMEBREW_NO_INSTALL_CLEANUP
+
 	# Uncomment this if you want to run perf tests:
 	# brew install gnu-time
 	brew link --force gettext
@@ -64,7 +71,7 @@ macos-latest)
 
 	if test -n "$CC_PACKAGE"
 	then
-		BREW_PACKAGE=${CC_PACKAGE/-/@}
+		BREW_PACKAGE=$(echo $CC_PACKAGE | tr '-' '@')
 		brew install "$BREW_PACKAGE"
 		brew link "$BREW_PACKAGE"
 	fi
ci/lib.sh +2 -2
index d18ee1a91e7..8df133b840b 100755
@@ -94,7 +94,7 @@ ubuntu-latest)
 		break
 	fi
 
-	if [ "$jobname" = linux-gcc ]
+	if test "$jobname" = linux-gcc
 	then
 		MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python3"
 	else
@@ -104,7 +104,7 @@ ubuntu-latest)
 	setenv --test GIT_TEST_HTTPD true
 	;;
 macos-latest)
-	if [ "$jobname" = osx-gcc ]
+	if test "$jobname" = osx-gcc
 	then
 		MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
 	else
Signature · 1 line
2.36.1.1045.gf356b5617dd
#245 lore raw

PATCH v6 15/29 CI: check ignored unignored build artifacts in "win[+VS] build" too

In reply to Ævar Arnfjörð Bjarmason

Use the newly created "ci/check-unignored-build-artifacts.sh" script in the "win build" and "win+VS build" steps too. These steps weren't doing this sanity check of untracked files against .gitignore before, but as the check passes for them let's keep doing that extra sanity check.

---
 .github/workflows/main.yml | 6 ++++++
 1 file changed, 6 insertions(+)
.github/workflows/main.yml +6 -0
index 57b2db06923..c97fbd0f0b3 100644
@@ -93,6 +93,9 @@ jobs:
         HOME: ${{runner.workspace}}
         NO_PERL: 1
       run: . /etc/profile && ci/make-test-artifacts.sh artifacts
+    - run: ci/check-unignored-build-artifacts.sh
+      if: success()
+      shell: bash
     - name: zip up tracked files
       run: git archive -o artifacts/tracked.tar.gz HEAD
     - name: upload tracked files and build artifacts
@@ -180,6 +183,9 @@ jobs:
       run: |
         mkdir -p artifacts &&
         eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts NO_GETTEXT=YesPlease 2>&1 | grep ^tar)"
+    - run: ci/check-unignored-build-artifacts.sh
+      if: success()
+      shell: bash
     - name: zip up tracked files
       run: git archive -o artifacts/tracked.tar.gz HEAD
     - name: upload tracked files and build artifacts
Signature · 1 line
2.36.1.1045.gf356b5617dd
#246 lore raw

PATCH v6 16/29 CI: invoke "make artifacts-tar" directly in windows-build

In reply to Ævar Arnfjörð Bjarmason

Change the windows-build job to invoke the "make artifacts-tar" step directly, instead of calling a "ci/make-test-artifacts.sh" script.

The script was needed because "ci/lib.sh" would set up various environment variables for us, but now we can instead use the "ci/lib.sh" in its script mode.

The "mkdir -p" added in b819f1d2cec (ci: parallelize testing on Windows, 2019-01-29) isn't needed, the same commit added that "mkdir -p" to the "artifacts-tar" rule itself, so we can have "make" create the directory for us.

This also has the benefit of making the "build" step less chatty, since it won't start with the verbose "set -x" output, that's now contained in the "ci/lib.sh" step.

---
 .github/workflows/main.yml |  5 ++++-
 ci/make-test-artifacts.sh  | 10 ----------
 2 files changed, 4 insertions(+), 11 deletions(-)
 delete mode 100755 ci/make-test-artifacts.sh
.github/workflows/main.yml +4 -1
index c97fbd0f0b3..ca993afd098 100644
@@ -87,12 +87,15 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
+    - run: ci/lib.sh
+      shell: bash
     - name: build
       shell: bash
       env:
         HOME: ${{runner.workspace}}
         NO_PERL: 1
-      run: . /etc/profile && ci/make-test-artifacts.sh artifacts
+      run: . /etc/profile && make artifacts-tar ARTIFACTS_DIRECTORY=artifacts
+      if: success()
     - run: ci/check-unignored-build-artifacts.sh
       if: success()
       shell: bash
ci/make-test-artifacts.sh deleted +0 -10
deleted file mode 100755
index 45298562982..00000000000
@@ -1,10 +0,0 @@
-#!/bin/sh
-#
-# Build Git and store artifacts for testing
-#
-
-mkdir -p "$1" # in case ci/lib.sh decides to quit early
-
-. ${0%/*}/lib.sh
-
-make artifacts-tar ARTIFACTS_DIRECTORY="$1"
Signature · 1 line
2.36.1.1045.gf356b5617dd
#247 lore raw

PATCH v6 17/29 CI: split up and reduce "ci/test-documentation.sh"

In reply to Ævar Arnfjörð Bjarmason

Change the "ci/test-documentation.sh" script to run the bash-specific parts in as one command in the CI job itself, and to run the two "make doc" commands at the top-level.

It'll now be obvious from the title of the step if we failed in the asciidoc or asciidoctor step.

Since the "check_unignored_build_artifacts()" function is now only used in "ci/check-unignored-build-artifacts.sh" move that function there.

The recipe for the job in ".github/workflows/main.yml" is now a bit verbose because it's effectively the same job twice, with a "make clean" in-between. It would be better for the verbosity to run it via a matrix as done in the alternate approach in [1] does, but then we'd sacrifice overall CPU time for the brevity. It might still be worth doing, but let's go for this simpler approach for now.

1. https://lore.kernel.org/git/[email protected]/

---
 .github/workflows/main.yml            | 16 +++++++++++++-
 ci/check-unignored-build-artifacts.sh | 10 +++++++++
 ci/lib.sh                             | 10 ---------
 ci/test-documentation.sh              | 31 +++++++++------------------
 4 files changed, 35 insertions(+), 32 deletions(-)
.github/workflows/main.yml +15 -1
index ca993afd098..88eb2ef2dad 100644
@@ -365,4 +365,18 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
-    - run: ci/test-documentation.sh
+    - run: ci/lib.sh
+    - run: make check-docs
+    - run: "make doc > >(tee stdout.log) 2> >(tee stderr.raw >&2)"
+      shell: bash
+    - run: ci/test-documentation.sh AsciiDoc
+      if: success()
+    - run: ci/check-unignored-build-artifacts.sh
+      if: success()
+    - run: make clean
+    - run: "make USE_ASCIIDOCTOR=1 doc > >(tee stdout.log) 2> >(tee stderr.raw >&2)"
+      shell: bash
+    - run: ci/test-documentation.sh Asciidoctor
+      if: success()
+    - run: ci/check-unignored-build-artifacts.sh
+      if: success()
ci/check-unignored-build-artifacts.sh +10 -0
index 56d04b0db9a..0bc04f32804 100755
@@ -5,4 +5,14 @@
 
 . ${0%/*}/lib.sh
 
+check_unignored_build_artifacts ()
+{
+	! git ls-files --other --exclude-standard --error-unmatch \
+		-- ':/*' 2>/dev/null ||
+	{
+		echo "$(tput setaf 1)error: found unignored build artifacts$(tput sgr0)"
+		false
+	}
+}
+
 check_unignored_build_artifacts
ci/lib.sh +0 -10
index 8df133b840b..5141de85024 100755
@@ -47,16 +47,6 @@ setenv () {
 	fi
 }
 
-check_unignored_build_artifacts ()
-{
-	! git ls-files --other --exclude-standard --error-unmatch \
-		-- ':/*' 2>/dev/null ||
-	{
-		echo "$(tput setaf 1)error: found unignored build artifacts$(tput sgr0)"
-		false
-	}
-}
-
 # GitHub Action doesn't set TERM, which is required by tput
 setenv TERM ${TERM:-dumb}
 
ci/test-documentation.sh +10 -21
index 6b5cce03bd7..b8a6a6f664e 100755
@@ -1,10 +1,12 @@
-#!/usr/bin/env bash
+#!/bin/sh
 #
-# Perform sanity checks on documentation and build it.
+# Perform sanity checks on "make doc" output and built documentation
 #
 
 . ${0%/*}/lib.sh
 
+generator=$1
+
 filter_log () {
 	sed -e '/^GIT_VERSION = /d' \
 	    -e "/constant Gem::ConfigMap is deprecated/d" \
@@ -14,28 +16,15 @@ filter_log () {
 	    "$1"
 }
 
-make check-docs
-
-# Build docs with AsciiDoc
-make doc > >(tee stdout.log) 2> >(tee stderr.raw >&2)
-cat stderr.raw
-filter_log stderr.raw >stderr.log
-test ! -s stderr.log
-test -s Documentation/git.html
-test -s Documentation/git.xml
-test -s Documentation/git.1
-grep '<meta name="generator" content="AsciiDoc ' Documentation/git.html
-
-rm -f stdout.log stderr.log stderr.raw
-check_unignored_build_artifacts
-
-# Build docs with AsciiDoctor
-make clean
-make USE_ASCIIDOCTOR=1 doc > >(tee stdout.log) 2> >(tee stderr.raw >&2)
 cat stderr.raw
 filter_log stderr.raw >stderr.log
 test ! -s stderr.log
 test -s Documentation/git.html
-grep '<meta name="generator" content="Asciidoctor ' Documentation/git.html
+if test "$generator" = "Asciidoctor"
+then
+	test -s Documentation/git.xml
+	test -s Documentation/git.1
+fi
+grep "<meta name=\"generator\" content=\"$generator " Documentation/git.html
 
 rm -f stdout.log stderr.log stderr.raw
Signature · 1 line
2.36.1.1045.gf356b5617dd