utils/test-pkg: add concurrency parameter

Builds run by test-pkg are often not CPU limited, and so running multiple builds concurrently has the potential to speed things up quite a lot.

Add a concurrency parameter to the script that allows for multiple builds to run concurrently. The default is the current behavior, running a single build at a time. If -C|--concurrent=0 is specified, the number of concurrent builds is set to the output of $(nproc) to match the number of logical CPUs.

$ time bash utils/test-pkg -c sdl2.config -p sdl2 -C1

                    bootlin-armv5-uclibc [1/6]: OK
                     bootlin-armv7-glibc [2/6]: OK
                   bootlin-armv7m-uclibc [3/6]: SKIPPED
                     bootlin-x86-64-musl [4/6]: OK
                      br-arm-full-static [5/6]: SKIPPED
                             arm-aarch64 [6/6]: OK

6 builds, 2 skipped, 0 build failed, 0 legal-info failed, 0 show-info failed

real    2m32.979s
user    6m15.356s
sys     0m30.878s

$ time bash utils/test-pkg -c sdl2.config -p sdl2 -C0

                    bootlin-armv5-uclibc [1/6]: OK
                     bootlin-armv7-glibc [2/6]: OK
                   bootlin-armv7m-uclibc [3/6]: SKIPPED
                     bootlin-x86-64-musl [4/6]: OK
                      br-arm-full-static [5/6]: SKIPPED
                             arm-aarch64 [6/6]: OK

6 builds, 2 skipped, 0 build failed, 0 legal-info failed, 0 show-info failed

real    0m41.704s
user    6m40.802s
sys     0m29.976s

Signed-off-by Joseph Kogut <[email protected]>

---
I'm labeling this an RFC for now, as this script may need some more testing to ensure there aren't any bugs with rendering, job cleanup, or problems with terminfo dependencies that aren't widely supported.

The patch appears to behave as intended in my testing, and it speeds up package testing quite a lot. The commit description shows the results testing sdl2, and I've pasted another result below from building the llama-cpp package I've recently submitted.

$ time utils/test-pkg -c llama-cpp.config -a -p llama-cpp
                             arm-aarch64 [ 1/40]: OK
                   bootlin-aarch64-glibc [ 2/40]: OK
               bootlin-arcle-hs38-uclibc [ 3/40]: SKIPPED
                    bootlin-armv5-uclibc [ 4/40]: SKIPPED
                     bootlin-armv7-glibc [ 5/40]: OK
                      bootlin-armv7-musl [ 6/40]: OK
                   bootlin-armv7m-uclibc [ 7/40]: SKIPPED
                bootlin-m68k-5208-uclibc [ 8/40]: SKIPPED
               bootlin-m68k-68040-uclibc [ 9/40]: SKIPPED
             bootlin-microblazeel-uclibc [10/40]: SKIPPED
                   bootlin-mipsel-uclibc [11/40]: SKIPPED
                bootlin-mipsel32r6-glibc [12/40]: OK
                 bootlin-openrisc-uclibc [13/40]: SKIPPED
           bootlin-powerpc-e500mc-uclibc [14/40]: SKIPPED
        bootlin-powerpc64le-power8-glibc [15/40]: OK
                   bootlin-riscv32-glibc [16/40]: SKIPPED
                   bootlin-riscv64-glibc [17/40]: SKIPPED
                    bootlin-riscv64-musl [18/40]: SKIPPED
                 bootlin-s390x-z13-glibc [19/40]: SKIPPED
                      bootlin-sh4-uclibc [20/40]: SKIPPED
                    bootlin-sparc-uclibc [21/40]: SKIPPED
                   bootlin-sparc64-glibc [22/40]: OK
                    bootlin-x86-64-glibc [23/40]: OK
                     bootlin-x86-64-musl [24/40]: OK
                   bootlin-x86-64-uclibc [25/40]: SKIPPED
                   bootlin-x86-i686-musl [26/40]: OK
                   bootlin-xtensa-uclibc [27/40]: SKIPPED
                            br-arm-basic [28/40]: SKIPPED
                    br-arm-full-nothread [29/40]: SKIPPED
                      br-arm-full-static [30/40]: SKIPPED
                   br-i386-pentium4-full [31/40]: SKIPPED
                      br-mips64-n64-full [32/40]: SKIPPED
                 br-mips64r6-el-hf-glibc [33/40]: OK
               br-powerpc-603e-basic-cpp [34/40]: SKIPPED
               br-powerpc64-power7-glibc [35/40]: OK
                       linaro-aarch64-be [36/40]: SKIPPED
                          linaro-aarch64 [37/40]: SKIPPED
                              linaro-arm [38/40]: SKIPPED
                           sourcery-mips [39/40]: SKIPPED
                         sourcery-mips64 [40/40]: SKIPPED
40 builds, 28 skipped, 0 build failed, 0 legal-info failed, 0 show-info failed

real    15m30.117s
user    37m47.514s
sys     1m24.012s

$ time utils/test-pkg -c llama-cpp.config -a -p llama-cpp -C0

                             arm-aarch64 [ 1/40]: OK
                   bootlin-aarch64-glibc [ 2/40]: OK
               bootlin-arcle-hs38-uclibc [ 3/40]: SKIPPED
                    bootlin-armv5-uclibc [ 4/40]: SKIPPED
                     bootlin-armv7-glibc [ 5/40]: OK
                      bootlin-armv7-musl [ 6/40]: OK
                   bootlin-armv7m-uclibc [ 7/40]: SKIPPED
                bootlin-m68k-5208-uclibc [ 8/40]: SKIPPED
               bootlin-m68k-68040-uclibc [ 9/40]: SKIPPED
             bootlin-microblazeel-uclibc [10/40]: SKIPPED
                   bootlin-mipsel-uclibc [11/40]: SKIPPED
                bootlin-mipsel32r6-glibc [12/40]: OK
                 bootlin-openrisc-uclibc [13/40]: SKIPPED
           bootlin-powerpc-e500mc-uclibc [14/40]: SKIPPED
        bootlin-powerpc64le-power8-glibc [15/40]: OK
                   bootlin-riscv32-glibc [16/40]: SKIPPED
                   bootlin-riscv64-glibc [17/40]: SKIPPED
                    bootlin-riscv64-musl [18/40]: SKIPPED
                 bootlin-s390x-z13-glibc [19/40]: SKIPPED
                      bootlin-sh4-uclibc [20/40]: SKIPPED
                    bootlin-sparc-uclibc [21/40]: SKIPPED
                   bootlin-sparc64-glibc [22/40]: OK
                    bootlin-x86-64-glibc [23/40]: OK
                     bootlin-x86-64-musl [24/40]: OK
                   bootlin-x86-64-uclibc [25/40]: SKIPPED
                   bootlin-x86-i686-musl [26/40]: OK
                   bootlin-xtensa-uclibc [27/40]: SKIPPED
                            br-arm-basic [28/40]: SKIPPED
                    br-arm-full-nothread [29/40]: SKIPPED
                      br-arm-full-static [30/40]: SKIPPED
                   br-i386-pentium4-full [31/40]: SKIPPED
                      br-mips64-n64-full [32/40]: SKIPPED
                 br-mips64r6-el-hf-glibc [33/40]: OK
               br-powerpc-603e-basic-cpp [34/40]: SKIPPED
               br-powerpc64-power7-glibc [35/40]: OK
                       linaro-aarch64-be [36/40]: SKIPPED
                          linaro-aarch64 [37/40]: SKIPPED
                              linaro-arm [38/40]: SKIPPED
                           sourcery-mips [39/40]: SKIPPED
                         sourcery-mips64 [40/40]: SKIPPED

40 builds, 28 skipped, 0 build failed, 0 legal-info failed, 0 show-info failed

real    2m32.778s
user    54m8.452s
sys     1m44.897s

This test was performed on a 16-core Ryzen 9 9950X, and the all-toolchain build was ~6x faster compared to the same serial run. The improvement would be greater with a build that launches more jobs (fewer skips).

As always, feedback is very welcome.

---
 utils/test-pkg | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 126 insertions(+), 15 deletions(-)

utils/test-pkg +126 -16

index 19d8713d6a..345fc4832d 100755
@@ -4,20 +4,34 @@ set -e
 TOOLCHAINS_CSV='support/config-fragments/autobuild/toolchain-configs.csv'
 TEMP_CONF=""
 
+declare -a running
+
 do_clean() {
+    # restore cursor
+    tput cnorm
+    printf '\n'
+
     if [ -n "${TEMP_CONF}" ]; then
         rm -f "${TEMP_CONF}"
     fi
+
+    # Terminate any running jobs
+    for pid in "${!running[@]}"; do
+        if kill -0 "${running[i]}" 2>/dev/null; then
+            kill -TERM "$pid" 2>/dev/null || true
+        fi
+    done
 }
 
 main() {
     local o O opts
-    local cfg dir pkg random toolchains_csv toolchain all number mode prepare_only
+    local cfg dir pkg random toolchains_csv toolchain all number mode prepare_only \
+           concurrency
     local ret nb nb_skip nb_fail nb_legal nb_show nb_tc build_dir keep
     local -a toolchains
     local pkg_br_name
 
-    o='hakc:d:n:p:r:t:'
+    o='hakc:d:n:p:r:t:C:'
     O='help,all,keep,prepare-only,config-snippet:,build-dir:,number:,package:,random:,toolchains-csv:'
     opts="$(getopt -n "${my_name}" -o "${o}" -l "${O}" -- "${@}")"
     eval set -- "${opts}"

@@ -28,6 +42,7 @@ main() {
     number=0
     mode=0
     prepare_only=0
+    concurrency=1
     toolchains_csv="${TOOLCHAINS_CSV}"
     while [ ${#} -gt 0 ]; do
         case "${1}" in

@@ -61,6 +76,9 @@ main() {
         (-t|--toolchains-csv)
             toolchains_csv="${2}"; shift 2
             ;;
+        (-C|--concurrency)
+            concurrency="${2}"; shift 2
+            ;;
         (--)
             shift; break
             ;;

@@ -79,6 +97,13 @@ main() {
     if [ ! -e "${cfg}" ]; then
         printf "error: %s: no such file\n" "${cfg}" >&2; exit 1
     fi
+    if [ "${concurrency}" -eq 0 ] 2>/dev/null; then
+        concurrency=$(nproc)
+    fi
+
+    if ! [ "${concurrency}" -gt 0 ] 2>/dev/null; then
+        printf "error: concurrency must be an integer\n" >&2; exit 1
+    fi
     if [ -z "${dir}" ]; then
         dir="${HOME}/br-test-pkg"
     fi

@@ -95,6 +120,13 @@ main() {
         mode=$((mode+1))
     fi
 
+    tput civis
+    tput sc
+
+    declare -A pid_to_idx
+    declare -a running
+    declare -a display_order
+
     # Default mode is to test the N first toolchains, which have been
     # chosen to be a good selection of toolchains.
     if [ ${mode} -eq 0 ] ; then

@@ -126,27 +158,103 @@ main() {
     nb_fail=0
     nb_legal=0
     nb_show=0
-    for toolchainconfig in "${toolchains[@]}"; do
-        : $((nb++))
-        toolchain="$(basename "${toolchainconfig}" .config)"
-        build_dir="${dir}/${toolchain}"
-        printf "%40s [%*d/%d]: " "${toolchain}" ${#nb_tc} "${nb}" "${nb_tc}"
-        build_one "${build_dir}" "${toolchainconfig}" "${cfg}" "${pkg}" "${prepare_only}" && ret=0 || ret=${?}
-        case ${ret} in
-        (0) printf "OK\n";;
-        (1) : $((nb_skip++)); printf "SKIPPED\n";;
-        (2) : $((nb_fail++)); printf "FAILED\n";;
-        (3) : $((nb_legal++)); printf "FAILED\n";;
-        (4) : $((nb_show++)); printf "FAILED\n";;
-        esac
+
+    tput civis
+
+    # Allocate lines for all toolchains up front to avoid scroll-invalidating
+    # the saved cursor
+    for ((i = 0; i < nb_tc; i++)); do printf '\n'; done
+    tput cuu "${nb_tc}"
+    tput sc
+
+    declare -A pid_to_idx
+    declare -a display_order
+
+    while (( nb < nb_tc || ${#running[@]} > 0)); do
+        while (( nb < nb_tc && ${#running[@]} < concurrency )); do
+            toolchainconfig=${toolchains[$nb]}
+            toolchain="$(basename "${toolchainconfig}" .config)"
+            build_dir="${dir}/${toolchain}"
+            build_one \
+                "${build_dir}" \
+                "${toolchainconfig}" \
+                "${cfg}" \
+                "${pkg}" \
+                "${prepare_only}" &
+
+            pid=$!; pid_to_idx[${pid}]=${nb}
+            running+=( "$pid" )
+            slot=${#display_order[@]}
+            display_order+=( "$nb" )
+
+           # Draw placeholder on its preallocated line
+            update_line "$slot" "%40s [%*d/%d]:" \
+                "${toolchain}" \
+                "${#nb_tc}" \
+                $((nb + 1)) \
+                "${nb_tc}"
+
+                nb=$((nb + 1))
+        done
+
+        for i in "${!running[@]}"; do
+            pid="${running[${i}]}"
+            if ! kill -0 "${pid}" 2>/dev/null; then
+                wait "${pid}" && ret=0 || ret=${?}
+                idx=${pid_to_idx[$pid]}
+                toolchainconfig=${toolchains[$idx]}
+                toolchain="$(basename "${toolchainconfig}" .config)"
+                case "${ret}" in
+                (0) stat="OK";;
+                (1) : $((nb_skip++)); stat="SKIPPED";;
+                (2) : $((nb_fail++)); stat="FAILED";;
+                (3) : $((nb_legal++)); stat="FAILED";;
+                (4) : $((nb_show++)); stat="FAILED";;
+                esac
+
+                for slot in "${!display_order[@]}"; do
+                    if [[ ${display_order[$slot]} -eq ${idx} ]]; then
+                        break
+                    fi
+                done
+
+                update_line "$slot" "%40s [%*d/%d]: %s" \
+                    "${toolchain}" \
+                    "${#nb_tc}" \
+                    "$((idx + 1))" \
+                    "${nb_tc}" \
+                    "$stat"
+
+                unset 'running[i]'
+            fi
+        done
+
+        running=( "${running[@]}" )
+        sleep 0.1
     done
 
+    tput rc
+    tput cud "${#display_order[@]}"
+    tput cnorm
+
+    printf '\n\n'
     printf "%d builds, %d skipped, %d build failed, %d legal-info failed, %d show-info failed\n" \
         "${nb}" "${nb_skip}" "${nb_fail}" "${nb_legal}" "${nb_show}"
 
     return $((nb_fail + nb_legal))
 }
 
+update_line() {
+    local slot=$1; shift
+    tput rc
+    tput cud "$((slot + 1))"
+    fmt=$1; shift
+
+    # shellcheck disable=SC2059
+    printf -- "${fmt}" "$@"
+    printf '\033[K'
+}
+
 build_one() {
     local dir="${1}"
     local toolchainconfig="${2}"

@@ -273,6 +381,9 @@ Options:
     -r N, --random N
         Limit the tests to the N randomly selected toolchains.
 
+    -C N, --concurrency N
+        Run N builds concurrently. If N is 0, match the number of logical CPUs.
+
     -t CSVFILE, --toolchains-csv CSVFILE
         CSV file containing the paths to config fragments of toolchains to
         try. If not specified, the toolchains in ${TOOLCHAINS_CSV} will be
 
---

base-commit: f23c810176305604c54b79611d885547f4e45c23
change-id: 20250619-concurrent-test-pkg-f3ad6d3c01b4

Best regards,
--
Joseph Kogut <[email protected]>