Skip to contentSkip to conversation

Some improvement for gitlab CI jobs

PATCH Superseded Anthony PERARD 1 participant 5 messages Feb 28, 2022

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

cover Some improvement for gitlab CI jobs

Patch series available in this git branch:
https://xenbits.xen.org/git-http/people/aperard/xen-unstable.git br.gitlab-ci-improvement-v1

Allow build and test jobs to run concurently.

Avoid running "test artifact" jobs on branch "master" and other, when test jobs aren't runned.

Anthony PERARD (4):
automation: fix typo in .gcc-tmpl
automation: add a templates for test jobs
automation: only run test artifact jobs when needed
automation: use "needs" instead of "dependencies" for test jobs

 automation/gitlab-ci/build.yaml | 18 ++++---
 automation/gitlab-ci/test.yaml  | 89 +++++++++------------------------
 2 files changed, 37 insertions(+), 70 deletions(-)
Signature · 1 line
Anthony PERARD
#1 lore raw

PATCH 1/4 automation: fix typo in .gcc-tmpl

The name of the field doesn't matter because it's use as a YAML achor, but it's nicer to have the proper spelling.

---
 automation/gitlab-ci/build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
automation/gitlab-ci/build.yaml +1 -1
index cc36428cf5..26dcfb1c25 100644
@@ -17,7 +17,7 @@
     - /^stable-.*/
 
 .gcc-tmpl:
-  variabes: &gcc
+  variables: &gcc
     CC: gcc
     CXX: g++
 
Signature · 1 line
Anthony PERARD
#2 lore raw

PATCH 2/4 automation: add a templates for test jobs

In reply to Anthony PERARD

Allow to set common configuration from a single place for all tests jobs.

---
 automation/gitlab-ci/test.yaml | 73 ++++++++--------------------------
 1 file changed, 17 insertions(+), 56 deletions(-)
automation/gitlab-ci/test.yaml +17 -56
index 47e8704df3..7cf21b7493 100644
@@ -1,7 +1,15 @@
-# Test jobs
-build-each-commit-gcc:
+.test-jobs-common:
   stage: test
   image: registry.gitlab.com/xen-project/xen/${CONTAINER}
+  except:
+    - master
+    - smoke
+    - /^coverity-tested\/.*/
+    - /^stable-.*/
+
+# Test jobs
+build-each-commit-gcc:
+  extends: .test-jobs-common
   variables:
     CONTAINER: debian:stretch
     XEN_TARGET_ARCH: x86_64
@@ -16,15 +24,9 @@ build-each-commit-gcc:
   dependencies: []
   tags:
     - x86_64
-  except:
-    - master
-    - smoke
-    - /^coverity-tested\/.*/
-    - /^stable-.*/
 
 qemu-alpine-arm64-gcc:
-  stage: test
-  image: registry.gitlab.com/xen-project/xen/${CONTAINER}
+  extends: .test-jobs-common
   variables:
     CONTAINER: debian:unstable-arm64v8
   script:
@@ -41,15 +43,9 @@ qemu-alpine-arm64-gcc:
     when: always
   tags:
     - arm64
-  except:
-    - master
-    - smoke
-    - /^coverity-tested\/.*/
-    - /^stable-.*/
 
 qemu-alpine-x86_64-gcc:
-  stage: test
-  image: registry.gitlab.com/xen-project/xen/${CONTAINER}
+  extends: .test-jobs-common
   variables:
     CONTAINER: debian:stretch
   script:
@@ -65,15 +61,9 @@ qemu-alpine-x86_64-gcc:
     when: always
   tags:
     - x86_64
-  except:
-    - master
-    - smoke
-    - /^coverity-tested\/.*/
-    - /^stable-.*/
 
 qemu-smoke-arm64-gcc:
-  stage: test
-  image: registry.gitlab.com/xen-project/xen/${CONTAINER}
+  extends: .test-jobs-common
   variables:
     CONTAINER: debian:unstable-arm64v8
   script:
@@ -89,15 +79,9 @@ qemu-smoke-arm64-gcc:
     when: always
   tags:
     - arm64
-  except:
-    - master
-    - smoke
-    - /^coverity-tested\/.*/
-    - /^stable-.*/
 
 qemu-smoke-x86-64-gcc:
-  stage: test
-  image: registry.gitlab.com/xen-project/xen/${CONTAINER}
+  extends: .test-jobs-common
   variables:
     CONTAINER: debian:stretch
   script:
@@ -111,15 +95,9 @@ qemu-smoke-x86-64-gcc:
     - debian-stretch-gcc-debug
   tags:
     - x86_64
-  except:
-    - master
-    - smoke
-    - /^coverity-tested\/.*/
-    - /^stable-.*/
 
 qemu-smoke-x86-64-clang:
-  stage: test
-  image: registry.gitlab.com/xen-project/xen/${CONTAINER}
+  extends: .test-jobs-common
   variables:
     CONTAINER: debian:stretch
   script:
@@ -133,15 +111,9 @@ qemu-smoke-x86-64-clang:
     - debian-unstable-clang-debug
   tags:
     - x86_64
-  except:
-    - master
-    - smoke
-    - /^coverity-tested\/.*/
-    - /^stable-.*/
 
 qemu-smoke-x86-64-gcc-pvh:
-  stage: test
-  image: registry.gitlab.com/xen-project/xen/${CONTAINER}
+  extends: .test-jobs-common
   variables:
     CONTAINER: debian:stretch
   script:
@@ -155,15 +127,9 @@ qemu-smoke-x86-64-gcc-pvh:
     - debian-stretch-gcc-debug
   tags:
     - x86_64
-  except:
-    - master
-    - smoke
-    - /^coverity-tested\/.*/
-    - /^stable-.*/
 
 qemu-smoke-x86-64-clang-pvh:
-  stage: test
-  image: registry.gitlab.com/xen-project/xen/${CONTAINER}
+  extends: .test-jobs-common
   variables:
     CONTAINER: debian:stretch
   script:
@@ -177,8 +143,3 @@ qemu-smoke-x86-64-clang-pvh:
     - debian-unstable-clang-debug
   tags:
     - x86_64
-  except:
-    - master
-    - smoke
-    - /^coverity-tested\/.*/
-    - /^stable-.*/
Signature · 1 line
Anthony PERARD
#3 lore raw

PATCH 3/4 automation: only run test artifact jobs when needed

In reply to Anthony PERARD

Share the same "except" as the one used for tests.

---
 automation/gitlab-ci/build.yaml | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)
automation/gitlab-ci/build.yaml +11 -5
index 26dcfb1c25..cbbe0b8346 100644
@@ -567,10 +567,16 @@ alpine-3.12-gcc-debug-arm64:
     CONTAINER: alpine:3.12-arm64v8
 
 
+## Test artifacts common
+
+.test-jobs-artifact-common:
+  stage: build
+  except: !reference [.test-jobs-common, except]
+
 # Arm test artifacts
 
 alpine-3.12-arm64-rootfs-export:
-  stage: build
+  extends: .test-jobs-artifact-common
   image: registry.gitlab.com/xen-project/xen/tests-artifacts/alpine:3.12-arm64v8
   script:
     - mkdir binaries && cp /initrd.tar.gz binaries/initrd.tar.gz
@@ -581,7 +587,7 @@ alpine-3.12-arm64-rootfs-export:
     - arm64
 
 kernel-5.9.9-arm64-export:
-  stage: build
+  extends: .test-jobs-artifact-common
   image: registry.gitlab.com/xen-project/xen/tests-artifacts/kernel:5.9.9-arm64v8
   script:
     - mkdir binaries && cp /Image binaries/Image
@@ -592,7 +598,7 @@ kernel-5.9.9-arm64-export:
     - arm64
 
 qemu-system-aarch64-5.2.0-arm64-export:
-  stage: build
+  extends: .test-jobs-artifact-common
   image: registry.gitlab.com/xen-project/xen/tests-artifacts/qemu-system-aarch64:5.2.0-arm64v8
   script:
     - mkdir binaries && cp /qemu-system-aarch64 binaries/qemu-system-aarch64
@@ -606,7 +612,7 @@ qemu-system-aarch64-5.2.0-arm64-export:
 # x86_64 test artifacts
 
 alpine-3.12-rootfs-export:
-  stage: build
+  extends: .test-jobs-artifact-common
   image: registry.gitlab.com/xen-project/xen/tests-artifacts/alpine:3.12
   script:
     - mkdir binaries && cp /initrd.tar.gz binaries/initrd.tar.gz
@@ -617,7 +623,7 @@ alpine-3.12-rootfs-export:
     - x86_64
 
 kernel-5.10.74-export:
-  stage: build
+  extends: .test-jobs-artifact-common
   image: registry.gitlab.com/xen-project/xen/tests-artifacts/kernel:5.10.74
   script:
     - mkdir binaries && cp /bzImage binaries/bzImage
Signature · 1 line
Anthony PERARD
#4 lore raw

PATCH 4/4 automation: use "needs" instead of "dependencies" for test jobs

In reply to Anthony PERARD

Like with "dependencies", the jobs will get artifacts from the jobs listed in "needs". But the test jobs can run as soon as the build jobs listed have finished.

---
 automation/gitlab-ci/test.yaml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)
automation/gitlab-ci/test.yaml +8 -8
index 7cf21b7493..56747fb335 100644
@@ -21,7 +21,7 @@ build-each-commit-gcc:
     paths:
       - '*.log'
     when: always
-  dependencies: []
+  needs: []
   tags:
     - x86_64
 
@@ -31,7 +31,7 @@ qemu-alpine-arm64-gcc:
     CONTAINER: debian:unstable-arm64v8
   script:
     - ./automation/scripts/qemu-alpine-arm64.sh 2>&1 | tee qemu-smoke-arm64.log
-  dependencies:
+  needs:
     - alpine-3.12-gcc-arm64
     - alpine-3.12-arm64-rootfs-export
     - kernel-5.9.9-arm64-export
@@ -50,7 +50,7 @@ qemu-alpine-x86_64-gcc:
     CONTAINER: debian:stretch
   script:
     - ./automation/scripts/qemu-alpine-x86_64.sh 2>&1 | tee qemu-smoke-x86_64.log
-  dependencies:
+  needs:
     - alpine-3.12-gcc
     - alpine-3.12-rootfs-export
     - kernel-5.10.74-export
@@ -68,7 +68,7 @@ qemu-smoke-arm64-gcc:
     CONTAINER: debian:unstable-arm64v8
   script:
     - ./automation/scripts/qemu-smoke-arm64.sh 2>&1 | tee qemu-smoke-arm64.log
-  dependencies:
+  needs:
     - debian-unstable-gcc-arm64
     - kernel-5.9.9-arm64-export
     - qemu-system-aarch64-5.2.0-arm64-export
@@ -91,7 +91,7 @@ qemu-smoke-x86-64-gcc:
       - smoke.serial
       - '*.log'
     when: always
-  dependencies:
+  needs:
     - debian-stretch-gcc-debug
   tags:
     - x86_64
@@ -107,7 +107,7 @@ qemu-smoke-x86-64-clang:
       - smoke.serial
       - '*.log'
     when: always
-  dependencies:
+  needs:
     - debian-unstable-clang-debug
   tags:
     - x86_64
@@ -123,7 +123,7 @@ qemu-smoke-x86-64-gcc-pvh:
       - smoke.serial
       - '*.log'
     when: always
-  dependencies:
+  needs:
     - debian-stretch-gcc-debug
   tags:
     - x86_64
@@ -139,7 +139,7 @@ qemu-smoke-x86-64-clang-pvh:
       - smoke.serial
       - '*.log'
     when: always
-  dependencies:
+  needs:
     - debian-unstable-clang-debug
   tags:
     - x86_64
Signature · 1 line
Anthony PERARD