Skip to contentSkip to conversation

CI: report disk usage before/after pruning sstate

PATCH New Jon Mason 1 participant 3 messages Jun 29, 2021

lore
Share & tools
lore

Jon Mason

author
#0 lore raw

PATCH 1/3 CI: report disk usage before/after pruning sstate

From: Ross Burton <[email protected]>

Change-Id I66853ce1bf5e33c37094cc8fff04cbd4daaadf91
---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)
.gitlab-ci.yml +2 -0
index 6e7f083..bfcaa4a 100644
@@ -178,7 +178,9 @@ prune-sstate:
   stage: prep
   when: manual
   script:
+  - du -h -s $SSTATE_DIR
   - find $SSTATE_DIR -type f -atime +30 -delete
+  - du -h -s $SSTATE_DIR
 
 # Report on disk usage
 usage:
Signature · 1 line
2.17.1
author
#1 lore raw

PATCH 2/3 CI: mark build jobs as interruptible

From: Ross Burton <[email protected]>

Mark the build jobs as interruptible, as they don't write to shared resources (unlike update-repos or get-binary-toolchains). This means that if the same branch is pushed again GitLab is likely to be able to abort the obsolete pipeline to start the new build sooner.

Change-Id I9284273e9b3118b616d3cb062cb957d98fc5e37e
---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)
.gitlab-ci.yml +1 -0
index bfcaa4a..2720a68 100644
@@ -26,6 +26,7 @@ stages:
 # Generalised fragment to do a Kas build
 .build:
   extends: .setup
+  interruptible: true
   script:
   - KASFILES=$(ci/jobs-to-kas $CI_JOB_NAME)
   - kas shell --update --force-checkout $KASFILES -c 'cat conf/*.conf'
Signature · 1 line
2.17.1
author
#2 lore raw

PATCH 3/3 CI: add testing and testimage

In reply to Jon Mason

Add a testing task to the gitlab CI and run testimage in that job. Currently, it only runs against some of the qemu based machines. Using slirp, there is no need for a privileged container or tun/tap devices.

Change-Id Ia85a3d0089f7d4dc7595c3a45d328c79d8e675f1
---
 .gitlab-ci.yml   | 94 +++++++++++++++++++++++++++++++++++++++++-------
 ci/testimage.yml | 14 ++++++++
 2 files changed, 96 insertions(+), 12 deletions(-)
 create mode 100644 ci/testimage.yml
.gitlab-ci.yml +82 -12
index 2720a68..641a5f1 100644
@@ -5,6 +5,7 @@ stages:
   - prep
   - bootstrap
   - build
+  - test
 
 # Common job fragment to get a worker ready
 .setup:
@@ -16,6 +17,7 @@ stages:
     DL_DIR: $CI_BUILDS_DIR/persist/downloads
     BB_LOGCONFIG: $CI_PROJECT_DIR/ci/logging.yml
     TOOLCHAIN_DIR: $CI_BUILDS_DIR/persist/toolchains
+    IMAGE_DIR: $CI_PROJECT_DIR/work/build/tmp/deploy/images
   before_script:
     - echo KAS_WORK_DIR = $KAS_WORK_DIR
     - echo SSTATE_DIR = $SSTATE_DIR
@@ -28,10 +30,29 @@ stages:
   extends: .setup
   interruptible: true
   script:
-  - KASFILES=$(ci/jobs-to-kas $CI_JOB_NAME)
-  - kas shell --update --force-checkout $KASFILES -c 'cat conf/*.conf'
-  - kas build $KASFILES
-  - ./ci/check-warnings $KAS_WORK_DIR/build/warnings.log
+    - KASFILES=$(./ci/jobs-to-kas $CI_JOB_NAME)
+    - kas shell --update --force-checkout $KASFILES -c 'cat conf/*.conf'
+    - kas build $KASFILES
+    - ./ci/check-warnings $KAS_WORK_DIR/build/warnings.log
+
+.build_artifacts:
+  artifacts:
+    paths:
+      - $IMAGE_DIR/*
+    expire_in: 1 day
+
+.test:
+  extends: .setup
+  stage: test
+  # Unique before_script here to avoid the rm from the one in .setup
+  before_script:
+    - mkdir --verbose --parents $KAS_WORK_DIR $KAS_REPO_REF_DIR $SSTATE_DIR $DL_DIR $TOOLCHAIN_DIR
+  script:
+    - KASFILES=$(./ci/jobs-to-kas $CI_JOB_NAME)
+    # These are needed to avoid warnings about file permissions from the artifacts
+    - sudo chown -R builder $KAS_WORK_DIR
+    - sudo chgrp -R builder $KAS_WORK_DIR
+    - kas build $KASFILES -c testimage
 
 
 #
@@ -116,22 +137,34 @@ n1sdp/armgcc:
   allow_failure: true
 
 qemuarm:
-  extends: .build
-
-qemuarmv5:
-  extends: .build
+  extends:
+    - .build
+    - .build_artifacts
 
 qemuarm64-secureboot:
-  extends: .build
+  extends:
+    - .build
+    - .build_artifacts
 
 qemuarm64-secureboot/clang:
-  extends: .build
+  extends:
+    - .build
+    - .build_artifacts
 
 qemuarm64-secureboot/clang/musl:
-  extends: .build
+  extends:
+    - .build
+    - .build_artifacts
 
 qemuarm64-secureboot/musl:
-  extends: .build
+  extends:
+    - .build
+    - .build_artifacts
+
+qemuarmv5:
+  extends:
+    - .build
+    - .build_artifacts
 
 sgi575:
   extends: .build
@@ -140,6 +173,43 @@ tc0:
   extends: .build
 
 
+#
+# Third phase, the test jobs
+#
+
+# QEMU based machines can use testimage, others will need something else (i.e., LAVA)
+
+qemuarm/testimage:
+  extends: .test
+  needs:
+    - job: qemuarm
+
+qemuarm64-secureboot/testimage:
+  extends: .test
+  needs:
+    - job: qemuarm64-secureboot
+
+qemuarm64-secureboot/clang/testimage:
+  extends: .test
+  needs:
+    - job: qemuarm64-secureboot/clang
+
+qemuarm64-secureboot/clang/musl/testimage:
+  extends: .test
+  needs:
+    - job: qemuarm64-secureboot/clang/musl
+
+qemuarm64-secureboot/musl/testimage:
+  extends: .test
+  needs:
+    - job: qemuarm64-secureboot/musl
+
+qemuarmv5/testimage:
+  extends: .test
+  needs:
+    - job: qemuarmv5
+
+
 #
 # Utility tasks, not executed automatically
 #
ci/testimage.yml new file +14 -0
new file mode 100644
index 0000000..df2c22e
@@ -0,0 +1,14 @@
+header:
+  version: 9
+
+local_conf_header:
+  testimage: |
+    IMAGE_CLASSES += "testimage"
+  slirp: |
+    TEST_RUNQEMUPARAMS = "slirp"
+    TEST_SERVER_IP = "127.0.0.1"
+    QEMU_USE_SLIRP = "1"
+
+# Multiple targets are available, put it down to just one
+target:
+  - core-image-base
Signature · 1 line
2.17.1