Skip to contentSkip to conversation

ci: Use FDO_CI_CONCURRENT to limit build/test concurrency

PATCH Superseded Daniel Stone 2 participants 4 messages Jul 29, 2021

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 ci: Use FDO_CI_CONCURRENT to limit build/test concurrency

${FDO_CI_CONCURRENT} is set to an appropriate concurrency on our shared runners. Using all available CPUs will just DoS the world and would explain a lot about the timeouts.

---
 .gitlab-ci.yml | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)
.gitlab-ci.yml +19 -19
index 3b0e98ae..b5edfee8 100644
@@ -64,9 +64,9 @@ build:tests-fedora:
   stage: build
   script:
     - meson --prefix=/opt/igt $MESON_OPTIONS build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
     - DESTDIR="$PWD/installdir" ninja -C build install
-    - ninja -C build igt-gpu-tools-doc
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} igt-gpu-tools-doc
   artifacts:
     paths:
       - build
@@ -77,7 +77,7 @@ build:tests-fedora-no-libunwind:
   script:
     - dnf remove -y libunwind-devel
     - meson -Dlibunwind=disabled build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
 
 # documentation building is different with meson<0.47
 build:tests-fedora-oldest-meson:
@@ -86,15 +86,15 @@ build:tests-fedora-oldest-meson:
     - dnf remove -y meson
     - pip install /usr/src/meson-*.tar.gz
     - meson $MESON_OPTIONS build
-    - ninja -C build || ninja -C build -j 1
-    - ninja -C build igt-gpu-tools-doc
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} igt-gpu-tools-doc
 
 # ensure igt_nouveau stubs build when nouveau support for libdrm is disabled
 build:tests-fedora-no-libdrm-nouveau:
   stage: build
   script:
     - meson -Dlibdrm_drivers= build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
 
 build:tests-fedora-clang:
   stage: build
@@ -102,7 +102,7 @@ build:tests-fedora-clang:
     CC: clang
   script:
     - meson $MESON_OPTIONS build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
   artifacts:
     paths:
       - build
@@ -112,7 +112,7 @@ build:tests-debian-meson:
   stage: build
   script:
     - meson $MESON_OPTIONS build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
     - cp build/tests/test-list.txt meson-test-list.txt
   artifacts:
     paths:
@@ -123,7 +123,7 @@ build:tests-debian-minimal:
   stage: build
   script:
     - meson -Dtests=disabled -Dlibdrm_drivers="" build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
   artifacts:
     paths:
       - build
@@ -134,7 +134,7 @@ build:tests-debian-meson-armhf:
   script:
     - export PKG_CONFIG_PATH=/usr/lib/arm-linux-gnueabihf/pkgconfig/
     - meson --cross-file meson-cross-armhf.txt build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
   artifacts:
     paths:
       - build
@@ -145,7 +145,7 @@ build:tests-debian-meson-arm64:
   script:
     - export PKG_CONFIG_PATH=/usr/lib/aarch64-linux-gnu/pkgconfig/
     - meson --cross-file meson-cross-arm64.txt build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
   artifacts:
     paths:
       - build
@@ -156,7 +156,7 @@ build:tests-debian-meson-mips:
   script:
     - export PKG_CONFIG_PATH=/usr/lib/mips-linux-gnu/pkgconfig/
     - meson --cross-file meson-cross-mips.txt build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
   artifacts:
     paths:
       - build
@@ -168,7 +168,7 @@ test:ninja-test:
   dependencies:
     - build:tests-fedora
   stage: test
-  script: ninja -C build test
+  script: meson -C build test --num-processes ${FDO_CI_CONCURRENT:-4}
 
 test:ninja-test-clang:
   retry: 2
@@ -177,7 +177,7 @@ test:ninja-test-clang:
   variables:
     CC: clang
   stage: test
-  script: ninja -C build test
+  script: meson -C build test --num-processes ${FDO_CI_CONCURRENT:-4}
 
 test:ninja-test-minimal:
   retry: 2
@@ -185,7 +185,7 @@ test:ninja-test-minimal:
   dependencies:
     - build:tests-debian-minimal
   stage: test
-  script: ninja -C build test
+  script: meson -C build test --num-processes ${FDO_CI_CONCURRENT:-4}
 
 test:ninja-test-arm64:
   retry: 2
@@ -196,7 +196,7 @@ test:ninja-test-arm64:
   script:
     - export PKG_CONFIG_PATH=/usr/lib/aarch64-linux-gnu/pkgconfig/
     - env > build/envdump.txt
-    - ninja -C build test
+    - meson -C build test --num-processes ${FDO_CI_CONCURRENT:-4}
   artifacts:
     paths:
       - build
@@ -211,7 +211,7 @@ test:ninja-test-armhf:
   script:
     - export PKG_CONFIG_PATH=/usr/lib/arm-linux-gnueabihf/pkgconfig/
     - env > build/envdump.txt
-    - ninja -C build test
+    - meson -C build test --num-processes ${FDO_CI_CONCURRENT:-4}
   artifacts:
     paths:
       - build
@@ -226,7 +226,7 @@ test:ninja-test-mips:
   script:
     - export PKG_CONFIG_PATH=/usr/lib/mips-linux-gnu/pkgconfig/
     - env > build/envdump.txt
-    - ninja -C build test
+    - meson -C build test --num-processes ${FDO_CI_CONCURRENT:-4}
   artifacts:
     paths:
       - build
@@ -255,7 +255,7 @@ pages:
     - build:tests-fedora
   stage: deploy
   script:
-    - ninja -C build igt-gpu-tools-doc
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} igt-gpu-tools-doc
     - cp -r build/docs/reference/igt-gpu-tools/html public
   artifacts:
     paths:
Signature · 6 lines
2.31.1

_______________________________________________
igt-dev mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/igt-dev
#1 lore raw
Quoted 108 lines · Daniel Stone

${FDO_CI_CONCURRENT} is set to an appropriate concurrency on our shared runners. Using all available CPUs will just DoS the world and would explain a lot about the timeouts.

---
 .gitlab-ci.yml | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)
.gitlab-ci.yml +12 -12
index 3b0e98ae..b5edfee8 100644
@@ -64,9 +64,9 @@ build:tests-fedora:
   stage: build
   script:
     - meson --prefix=/opt/igt $MESON_OPTIONS build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
     - DESTDIR="$PWD/installdir" ninja -C build install
-    - ninja -C build igt-gpu-tools-doc
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} igt-gpu-tools-doc
   artifacts:
     paths:
       - build
@@ -77,7 +77,7 @@ build:tests-fedora-no-libunwind:
   script:
     - dnf remove -y libunwind-devel
     - meson -Dlibunwind=disabled build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
 
 # documentation building is different with meson<0.47
 build:tests-fedora-oldest-meson:
@@ -86,15 +86,15 @@ build:tests-fedora-oldest-meson:
     - dnf remove -y meson
     - pip install /usr/src/meson-*.tar.gz
     - meson $MESON_OPTIONS build
-    - ninja -C build || ninja -C build -j 1
-    - ninja -C build igt-gpu-tools-doc
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} igt-gpu-tools-doc
 
 # ensure igt_nouveau stubs build when nouveau support for libdrm is disabled
 build:tests-fedora-no-libdrm-nouveau:
   stage: build
   script:
     - meson -Dlibdrm_drivers= build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
 
 build:tests-fedora-clang:
   stage: build
@@ -102,7 +102,7 @@ build:tests-fedora-clang:
     CC: clang
   script:
     - meson $MESON_OPTIONS build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
   artifacts:
     paths:
       - build
@@ -112,7 +112,7 @@ build:tests-debian-meson:
   stage: build
   script:
     - meson $MESON_OPTIONS build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
     - cp build/tests/test-list.txt meson-test-list.txt
   artifacts:
     paths:
@@ -123,7 +123,7 @@ build:tests-debian-minimal:
   stage: build
   script:
     - meson -Dtests=disabled -Dlibdrm_drivers="" build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
   artifacts:
     paths:
       - build
@@ -134,7 +134,7 @@ build:tests-debian-meson-armhf:
   script:
     - export PKG_CONFIG_PATH=/usr/lib/arm-linux-gnueabihf/pkgconfig/
     - meson --cross-file meson-cross-armhf.txt build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
   artifacts:
     paths:
       - build
@@ -145,7 +145,7 @@ build:tests-debian-meson-arm64:
   script:
     - export PKG_CONFIG_PATH=/usr/lib/aarch64-linux-gnu/pkgconfig/
     - meson --cross-file meson-cross-arm64.txt build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
   artifacts:
     paths:
       - build
@@ -156,7 +156,7 @@ build:tests-debian-meson-mips:
   script:
     - export PKG_CONFIG_PATH=/usr/lib/mips-linux-gnu/pkgconfig/
     - meson --cross-file meson-cross-mips.txt build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
   artifacts:
     paths:
       - build

.gitlab-ci.yml test:ninja-test: · line 171

Quoted 5 lines
.gitlab-ci.yml +1 -1
@@ -168,7 +168,7 @@ test:ninja-test:
   dependencies:
     - build:tests-fedora
   stage: test
-  script: ninja -C build test
+  script: meson -C build test --num-processes ${FDO_CI_CONCURRENT:-4}

usage: meson [-h] {setup,configure,dist,install,introspect,init,test,wrap,subprojects,help,rewrite,compile} ...
meson: error: unrecognized arguments: -C --num-processes 4

Doesn't ninja -C build -j $... test work for this?

Signature · 1 line
Petri Latvala
Quoted 64 lines
.gitlab-ci.yml +6 -6
 test:ninja-test-clang:
   retry: 2
@@ -177,7 +177,7 @@ test:ninja-test-clang:
   variables:
     CC: clang
   stage: test
-  script: ninja -C build test
+  script: meson -C build test --num-processes ${FDO_CI_CONCURRENT:-4}
 
 test:ninja-test-minimal:
   retry: 2
@@ -185,7 +185,7 @@ test:ninja-test-minimal:
   dependencies:
     - build:tests-debian-minimal
   stage: test
-  script: ninja -C build test
+  script: meson -C build test --num-processes ${FDO_CI_CONCURRENT:-4}
 
 test:ninja-test-arm64:
   retry: 2
@@ -196,7 +196,7 @@ test:ninja-test-arm64:
   script:
     - export PKG_CONFIG_PATH=/usr/lib/aarch64-linux-gnu/pkgconfig/
     - env > build/envdump.txt
-    - ninja -C build test
+    - meson -C build test --num-processes ${FDO_CI_CONCURRENT:-4}
   artifacts:
     paths:
       - build
@@ -211,7 +211,7 @@ test:ninja-test-armhf:
   script:
     - export PKG_CONFIG_PATH=/usr/lib/arm-linux-gnueabihf/pkgconfig/
     - env > build/envdump.txt
-    - ninja -C build test
+    - meson -C build test --num-processes ${FDO_CI_CONCURRENT:-4}
   artifacts:
     paths:
       - build
@@ -226,7 +226,7 @@ test:ninja-test-mips:
   script:
     - export PKG_CONFIG_PATH=/usr/lib/mips-linux-gnu/pkgconfig/
     - env > build/envdump.txt
-    - ninja -C build test
+    - meson -C build test --num-processes ${FDO_CI_CONCURRENT:-4}
   artifacts:
     paths:
       - build
@@ -255,7 +255,7 @@ pages:
     - build:tests-fedora
   stage: deploy
   script:
-    - ninja -C build igt-gpu-tools-doc
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} igt-gpu-tools-doc
     - cp -r build/docs/reference/igt-gpu-tools/html public
   artifacts:
     paths:
Signature · 6 lines
2.31.1

_______________________________________________
igt-dev mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/igt-dev

_______________________________________________
igt-dev mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/igt-dev

#2 lore raw
Quoted 119 lines · Petri Latvala
Quoted 108 lines · Daniel Stone

${FDO_CI_CONCURRENT} is set to an appropriate concurrency on our shared runners. Using all available CPUs will just DoS the world and would explain a lot about the timeouts.

---
 .gitlab-ci.yml | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)
.gitlab-ci.yml +12 -12
index 3b0e98ae..b5edfee8 100644
@@ -64,9 +64,9 @@ build:tests-fedora:
   stage: build
   script:
     - meson --prefix=/opt/igt $MESON_OPTIONS build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
     - DESTDIR="$PWD/installdir" ninja -C build install
-    - ninja -C build igt-gpu-tools-doc
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} igt-gpu-tools-doc
   artifacts:
     paths:
       - build
@@ -77,7 +77,7 @@ build:tests-fedora-no-libunwind:
   script:
     - dnf remove -y libunwind-devel
     - meson -Dlibunwind=disabled build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
 
 # documentation building is different with meson<0.47
 build:tests-fedora-oldest-meson:
@@ -86,15 +86,15 @@ build:tests-fedora-oldest-meson:
     - dnf remove -y meson
     - pip install /usr/src/meson-*.tar.gz
     - meson $MESON_OPTIONS build
-    - ninja -C build || ninja -C build -j 1
-    - ninja -C build igt-gpu-tools-doc
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} igt-gpu-tools-doc
 
 # ensure igt_nouveau stubs build when nouveau support for libdrm is disabled
 build:tests-fedora-no-libdrm-nouveau:
   stage: build
   script:
     - meson -Dlibdrm_drivers= build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
 
 build:tests-fedora-clang:
   stage: build
@@ -102,7 +102,7 @@ build:tests-fedora-clang:
     CC: clang
   script:
     - meson $MESON_OPTIONS build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
   artifacts:
     paths:
       - build
@@ -112,7 +112,7 @@ build:tests-debian-meson:
   stage: build
   script:
     - meson $MESON_OPTIONS build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
     - cp build/tests/test-list.txt meson-test-list.txt
   artifacts:
     paths:
@@ -123,7 +123,7 @@ build:tests-debian-minimal:
   stage: build
   script:
     - meson -Dtests=disabled -Dlibdrm_drivers="" build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
   artifacts:
     paths:
       - build
@@ -134,7 +134,7 @@ build:tests-debian-meson-armhf:
   script:
     - export PKG_CONFIG_PATH=/usr/lib/arm-linux-gnueabihf/pkgconfig/
     - meson --cross-file meson-cross-armhf.txt build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
   artifacts:
     paths:
       - build
@@ -145,7 +145,7 @@ build:tests-debian-meson-arm64:
   script:
     - export PKG_CONFIG_PATH=/usr/lib/aarch64-linux-gnu/pkgconfig/
     - meson --cross-file meson-cross-arm64.txt build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
   artifacts:
     paths:
       - build
@@ -156,7 +156,7 @@ build:tests-debian-meson-mips:
   script:
     - export PKG_CONFIG_PATH=/usr/lib/mips-linux-gnu/pkgconfig/
     - meson --cross-file meson-cross-mips.txt build
-    - ninja -C build || ninja -C build -j 1
+    - ninja -C build -j${FDO_CI_CONCURRENT:-4} || ninja -C build -j 1
   artifacts:
     paths:
       - build

.gitlab-ci.yml test:ninja-test: · line 171

Quoted 5 lines
.gitlab-ci.yml +1 -1
@@ -168,7 +168,7 @@ test:ninja-test:
   dependencies:
     - build:tests-fedora
   stage: test
-  script: ninja -C build test
+  script: meson -C build test --num-processes ${FDO_CI_CONCURRENT:-4}

usage: meson [-h] {setup,configure,dist,install,introspect,init,test,wrap,subprojects,help,rewrite,compile} ...
meson: error: unrecognized arguments: -C --num-processes 4

Quoted 1 line

Doesn't ninja -C build -j $... test work for this?

Based on my tests, no.

Signature · 5 lines
Petri Latvala
_______________________________________________
igt-dev mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/igt-dev
#3 lore raw
Quoted 13 lines · Petri Latvala
Quoted 4 lines · Petri Latvala
Quoted 4 lines
(quoted diff) +0 -0
@@ -168,7 +168,7 @@ test:ninja-test:

   dependencies:
     - build:tests-fedora
   stage: test

Quoted 8 lines
Quoted 3 lines

-  script: ninja -C build test
+  script: meson -C build test --num-processes
${FDO_CI_CONCURRENT:-4}

usage: meson [-h] {setup,configure,dist,install,introspect,init,test,wrap,subprojects ,help,rewrite,compile} ...
meson: error: unrecognized arguments: -C --num-processes 4

Yeah, just spotted that locally. v2 sent.

Quoted 3 lines
Quoted 1 line

Doesn't ninja -C build -j $... test   work for this?

Based on my tests, no.

Indeed not. meson test is recommend to use over the ninja target anyway.

Anyway, I've sent v2 off now which actually works.

Cheers,
Daniel

_______________________________________________
igt-dev mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/igt-dev