Add cip-core-image-kernelci building and uploading system
v3 cover Add cip-core-image-kernelci building and uploading system
Changes in this version
- added missing dmesg.sh installation
- consolidated customizations recipes
Jan
Alice Ferrazzi (4):
Add cip-core-image-kernelci
Add dmesg filter needed for lava test result
Add script deploy-kernelci.py for upload the cip-core-image-kernelci enable cip-core-image-kernelci
Jan Kiszka (2):
customizations: Factor out a common.inc
kernelci-customizations: Reuse common customizations
.gitlab-ci.yml | 38 +++++++++++++
kas/opt/kernelci.yml | 16 ++++++
.../{customizations.bb => common.inc} | 4 +-
recipes-core/customizations/customizations.bb | 27 +--------
.../images/cip-core-image-kernelci.bb | 16 ++++++
.../kernelci-customizations/files/dmesg.sh | 23 ++++++++
.../kernelci-customizations/files/postinst | 34 ++++++++++++
.../kernelci-customizations.bb | 23 ++++++++
scripts/deploy-kernelci.py | 55 +++++++++++++++++++
9 files changed, 209 insertions(+), 27 deletions(-)
create mode 100644 kas/opt/kernelci.yml
copy recipes-core/customizations/{customizations.bb => common.inc} (90%)
create mode 100644 recipes-core/images/cip-core-image-kernelci.bb
create mode 100644 recipes-core/kernelci-customizations/files/dmesg.sh
create mode 100644 recipes-core/kernelci-customizations/files/postinst
create mode 100644 recipes-core/kernelci-customizations/kernelci-customizations.bb
create mode 100755 scripts/deploy-kernelci.pySignature · 1 line
2.31.1
PATCH v3 1/6 Add cip-core-image-kernelci
From: Alice Ferrazzi <[email protected]>
This image is currently used by KernelCI production for testing purpose. The purpose of this image is that KernelCI need to be able to autologin without password and to detect the login shell using special characters. Currently added are the default settings used by KernelCI images.
--- kas/opt/kernelci.yml | 16 +++++++++ .../images/cip-core-image-kernelci.bb | 16 +++++++++ .../files/99-silent-printk.conf | 1 + .../kernelci-customizations/files/ethernet | 23 +++++++++++++ .../kernelci-customizations/files/postinst | 34 +++++++++++++++++++ .../kernelci-customizations.bb | 34 +++++++++++++++++++ 6 files changed, 124 insertions(+) create mode 100644 kas/opt/kernelci.yml create mode 100644 recipes-core/images/cip-core-image-kernelci.bb create mode 100644 recipes-core/kernelci-customizations/files/99-silent-printk.conf create mode 100644 recipes-core/kernelci-customizations/files/ethernet create mode 100644 recipes-core/kernelci-customizations/files/postinst create mode 100644 recipes-core/kernelci-customizations/kernelci-customizations.bb
kas/opt/kernelci.yml new file +16 -0
@@ -0,0 +1,16 @@ +# +# CIP Core, generic profile +# +# Copyright (c) Toshiba Corporation, 2020 +# Copyright (c) Cybertrust Japan Co., Ltd., 2021 +# +# Authors: +# Venkata Pyla <[email protected]> +# Alice Ferrazzi <[email protected]> +# +# SPDX-License-Identifier: MIT +# +header: + version: 10 + +target: cip-core-image-kernelci
recipes-core/images/cip-core-image-kernelci.bb new file +16 -0
@@ -0,0 +1,16 @@ +# +# A reference image for KernelCI +# +# Copyright (c) Cybertrust Japan Co., Ltd., 2021 +# +# Authors: +# Alice Ferrazzi <[email protected]> +# +# SPDX-License-Identifier: MIT +# + +inherit image + +DESCRIPTION = "CIP Core image for KernelCI" + +IMAGE_INSTALL += "kernelci-customizations"
recipes-core/kernelci-customizations/files/99-silent-printk.conf new file +1 -0
@@ -0,0 +1 @@ +kernel.printk = 3 4 1 3
recipes-core/kernelci-customizations/files/ethernet new file +23 -0
@@ -0,0 +1,23 @@ +# +# CIP Core, generic profile +# +# Copyright (c) Siemens AG, 2019 +# +# Authors: +# Jan Kiszka <[email protected]> +# +# SPDX-License-Identifier: MIT +# + +allow-hotplug eth0 +allow-hotplug enp0s2 +allow-hotplug enp2s0 + +# used on BBB +iface eth0 inet dhcp + +# used on qemu-amd64 +iface enp0s2 inet dhcp + +# used on simatic-ipc227e +iface enp2s0 inet dhcp
recipes-core/kernelci-customizations/files/postinst new file +34 -0
@@ -0,0 +1,34 @@ +#!/bin/sh +# +# CIP Core, generic profile +# +# Copyright (c) Siemens AG, 2019 +# Copyright (c) Cybertrust Japan Co., Ltd., 2021 +# +# Authors: +# Jan Kiszka <[email protected]> +# Alice Ferrazzi <[email protected]> +# +# SPDX-License-Identifier: MIT +# + +echo "CIP Core Demo & Test Image" > /etc/issue + +# permit root login without password +echo "PermitRootLogin yes" >> /etc/ssh/sshd_config +echo "PermitEmptyPasswords yes" >> /etc/ssh/sshd_config +passwd root -d + +# serial getty service for autologin +mkdir -p /etc/systemd/system/[email protected]/ +echo "[Service]" > /etc/systemd/system/[email protected]/override.conf +echo "ExecStart=" >> /etc/systemd/system/[email protected]/override.conf +echo "ExecStart=-/sbin/agetty --autologin root --keep-baud 115200,38400,9600 %I $TERM" >> /etc/systemd/system/[email protected]/override.conf + +# set the profile for KernelCI +echo "PS1='\$(pwd) # '" > /root/.profile +echo "cd /" >> /root/.profile + +HOSTNAME=demo +echo "$HOSTNAME" > /etc/hostname +echo "127.0.0.1 $HOSTNAME" >> /etc/hosts
recipes-core/kernelci-customizations/kernelci-customizations.bb new file +34 -0
@@ -0,0 +1,34 @@ +# +# CIP Core, generic profile +# +# Copyright (c) Siemens AG, 2019 +# Copyright (c) Cybertrust Japan Co., Ltd., 2021 +# +# Authors: +# Jan Kiszka <[email protected]> +# Alice Ferrazzi <[email protected]> +# +# SPDX-License-Identifier: MIT +# + +inherit dpkg-raw + +DESCRIPTION = "CIP Core image demo & customizations" + +SRC_URI = " \ + file://postinst \ + file://ethernet \ + file://99-silent-printk.conf" + +DEPENDS += "sshd-regen-keys" + +DEBIAN_DEPENDS = " \ + ifupdown, isc-dhcp-client, net-tools, iputils-ping, ssh, sshd-regen-keys" + +do_install() { + install -v -d ${D}/etc/network/interfaces.d + install -v -m 644 ${WORKDIR}/ethernet ${D}/etc/network/interfaces.d/ + + install -v -d ${D}/etc/sysctl.d + install -v -m 644 ${WORKDIR}/99-silent-printk.conf ${D}/etc/sysctl.d/ +}
Signature · 1 line
2.31.1
PATCH v3 2/6 Add dmesg filter needed for lava test result
In reply to Jan KiszkaFrom: Alice Ferrazzi <[email protected]>
KernelCI is using a dmesg filter script for checking dmesg logs result with lava.
Adding the script to the cip-core-image-kernelci.
--- .../kernelci-customizations/files/dmesg.sh | 23 +++++++++++++++++++ .../kernelci-customizations.bb | 4 ++++ 2 files changed, 27 insertions(+) create mode 100644 recipes-core/kernelci-customizations/files/dmesg.sh
recipes-core/kernelci-customizations/files/dmesg.sh new file +23 -0
@@ -0,0 +1,23 @@ +#!/bin/sh + +set -e + +if [ "$KERNELCI_LAVA" = "y" ]; then + alias test-result='lava-test-case' +else + alias test-result='echo' +fi + +for level in crit alert emerg; do + dmesg --level=$level --notime -x -k > dmesg.$level + test -s dmesg.$level && res=fail || res=pass + count=$(cat dmesg.$level | wc -l) + cat dmesg.$level + test-result \ + $level \ + --result $res \ + --measurement $count \ + --units lines +done + +exit 0
recipes-core/kernelci-customizations/kernelci-customizations.bb +4 -0
@@ -18,6 +18,7 @@ DESCRIPTION = "CIP Core image demo & customizations" SRC_URI = " \ file://postinst \ file://ethernet \ + file://dmesg.sh \ file://99-silent-printk.conf" DEPENDS += "sshd-regen-keys"
@@ -31,4 +32,7 @@ do_install() { install -v -d ${D}/etc/sysctl.d install -v -m 644 ${WORKDIR}/99-silent-printk.conf ${D}/etc/sysctl.d/ + + install -v -d ${D}/opt/kernelci + install -v -m 744 ${WORKDIR}/dmesg.sh ${D}/opt/kernelci/ }
Signature · 1 line
2.31.1
PATCH v3 3/6 Add script deploy-kernelci.py for upload the cip-core-image-kernelci
In reply to Jan KiszkaFrom: Alice Ferrazzi <[email protected]>
The cip-core-image-kernelci need to be uploaded to the KernelCI production storage for been used by KernelCI.
This script use the KernelCI API for uploading the cip-core-image-kernelci to the production storage. The images are uploaded in the following link:
https://storage.kernelci.org/images/rootfs/cip/
--- scripts/deploy-kernelci.py | 55 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100755 scripts/deploy-kernelci.py
scripts/deploy-kernelci.py new file +55 -0
@@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import subprocess +import requests +import os +import sys +import time +from urllib.parse import urljoin + +cdate=time.strftime("%Y%m%d") +api="https://api.kernelci.org/upload" +token=os.getenv("KERNELCI_TOKEN") + +release=sys.argv[1] +target=sys.argv[2] +extension=sys.argv[3] + +rootfs_filename="cip-core-image-kernelci-cip-core-"+release+"-"+target+".tar.gz" +initrd_filename="cip-core-image-kernelci-cip-core-"+release+"-"+target+"-initrd.img" +initrd_gz_filename="cip-core-image-kernelci-cip-core-"+release+"-"+target+"-initrd.img.gz" + +input_dir="build/tmp/deploy/images/"+target +upload_path="/images/rootfs/cip/"+cdate+"/"+target+"/" +upload_path_latest="/images/rootfs/cip/latest/"+target+"/" +rootfs=input_dir+"/"+rootfs_filename +initrd=input_dir+"/"+initrd_filename + +def upload_file(api, token, path, input_file, input_filename): + headers = { + 'Authorization': token, + } + data = { + 'path': path, + } + files = { + 'file': (input_filename, open(input_file, 'rb').read()), + } + url = urljoin(api, 'upload') + resp = requests.post(url, headers=headers, data=data, files=files) + resp.raise_for_status() + +if os.path.exists(rootfs) and os.path.exists(initrd): + print("uploading rootfs to KernelCI") + upload_file(api, token, upload_path, rootfs, rootfs_filename) + print("uploading initrd to KernelCI") + upload_file(api, token, upload_path, initrd, initrd_gz_filename) + print("uploaded to: https://storage.kernelci.org"+upload_path) + + # Upload latest + print("uploading rootfs to KernelCI CIP latest") + upload_file(api, token, upload_path_latest, rootfs, rootfs_filename) + print("uploading initrd to KernelCI CIP latest") + upload_file(api, token, upload_path_latest, initrd, initrd_gz_filename) + print("uploaded to: https://storage.kernelci.org"+upload_path_latest)
Signature · 1 line
2.31.1
Re: [isar-cip-core][PATCH v3 3/6] Add script deploy-kernelci.py for upload the cip-core-image-kernelci
Quoted 29 lines · Jan Kiszka
From: Alice Ferrazzi <[email protected]>
The cip-core-image-kernelci need to be uploaded to the KernelCI production storage for been used by KernelCI.
This script use the KernelCI API for uploading the cip-core-image-kernelci to the production storage. The images are uploaded in the following link:
https://storage.kernelci.org/images/rootfs/cip/Signed-off-by Alice Ferrazzi <[email protected]>Signed-off-by Jan Kiszka <[email protected]>--- scripts/deploy-kernelci.py | 55 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100755 scripts/deploy-kernelci.pyscripts/deploy-kernelci.py new file +7
-0@@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import subprocess +import requests +import os +import sys
scripts/deploy-kernelci.py line 17
Quoted 10 lines
scripts/deploy-kernelci.py new file +10
-0+import time +from urllib.parse import urljoin + +cdate=time.strftime("%Y%m%d") +api="https://api.kernelci.org/upload" +token=os.getenv("KERNELCI_TOKEN") + +release=sys.argv[1] +target=sys.argv[2] +extension=sys.argv[3]
Just stumbled over these:
We do not use "extension" (it's always "kernelci"), that's an easy cleanup. But more important, we also pass the dtb while we do not push it to kernelci - by intention?
Jan
Signature · 2 lines
Siemens AG, Technology Competence Center Embedded Linux
Re: [isar-cip-core][PATCH v3 3/6] Add script deploy-kernelci.py for upload the cip-core-image-kernelci
Quoted 3 lines
We do not use "extension" (it's always "kernelci"), that's an easy cleanup. But more important, we also pass the dtb while we do not push it to kernelci - by intention?
I don't think changing extension to static name "kernelci" is useful. about dtb you are right. currently I don't remember if it was intentional. probably would be useful to upload also the dtb and we can than try to find a way of make KernelCI work with the generated dtb.
Thanks,
Alicef
--
======================================
Cybertrust Japan Co.,Ltd.
Alice Ferrazzi
[email protected]
======================================
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [cip-dev] [isar-cip-core][PATCH v3 3/6] Add script deploy-kernelci.py for upload the cip-core-image-kernelci
Quoted 10 lines · Alice Ferrazzi via lists.cip-project.org
Quoted 4 lines
We do not use "extension" (it's always "kernelci"), that's an easy cleanup. But more important, we also pass the dtb while we do not push it to kernelci - by intention?
I don't think changing extension to static name "kernelci" is useful. about dtb you are right. currently I don't remember if it was intentional. probably would be useful to upload also the dtb and we can than try to find a way of make KernelCI work with the generated dtb.
I was looking again the script and extension variable is actually not used anymore,
I thought from your text to just replace it with a static "kernelci" variable. Works for me to remove it.
Thanks,
Alicef
Signature · 5 lines
====================================== Cybertrust Japan Co.,Ltd. Alice Ferrazzi [email protected] ======================================
PATCH v3 4/6 enable cip-core-image-kernelci
In reply to Jan KiszkaFrom: Alice Ferrazzi <[email protected]>
enable CIP core image KernelCI to be built with gitlab pipeline and uploaded to the KernelCI storage using KernelCI API. enable cip-core-image-kernelci to be built for amd64, arm and arm64.
--- .gitlab-ci.yml | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+)
.gitlab-ci.yml +38 -0
@@ -9,6 +9,7 @@ variables: targz: disable dtb: none deploy: enable + deploy_kernelci: disable stages: - build
@@ -38,6 +39,7 @@ default: - echo "Building ${base_yaml}" - kas build ${base_yaml} - if [ "${deploy}" = "enable" ]; then scripts/deploy-cip-core.sh ${release} ${target} ${extension} ${dtb} ${CI_COMMIT_REF_SLUG}; fi + - if [ "${deploy_kernelci}" = "enable" ]; then scripts/deploy-kernelci.py ${release} ${target} ${extension} ${dtb}; fi # base image build:simatic-ipc227e-base:
@@ -77,6 +79,18 @@ build:qemu-amd64-base: wic_targz: disable targz: enable +build:qemu-amd64-base-kernelci: + extends: + - .build_base + variables: + target: qemu-amd64 + extension: kernelci + use_rt: disable + wic_targz: disable + targz: enable + deploy: disable + deploy_kernelci: enable + build:qemu-arm64-base: extends: - .build_base
@@ -87,6 +101,18 @@ build:qemu-arm64-base: wic_targz: disable targz: enable +build:qemu-arm64-base-kernelci: + extends: + - .build_base + variables: + target: qemu-arm64 + extension: kernelci + use_rt: disable + wic_targz: disable + targz: enable + deploy: disable + deploy_kernelci: enable + build:qemu-arm-base: extends: - .build_base
@@ -97,6 +123,18 @@ build:qemu-arm-base: wic_targz: disable targz: enable +build:qemu-arm-base-kernelci: + extends: + - .build_base + variables: + target: qemu-arm + extension: kernelci + use_rt: disable + wic_targz: disable + targz: enable + deploy: disable + deploy_kernelci: enable + # test build:simatic-ipc227e-test: extends:
Signature · 1 line
2.31.1
PATCH v3 5/6 customizations: Factor out a common.inc
In reply to Jan KiszkaFrom: Jan Kiszka <[email protected]>
This can be re-used by other customization recipes.
---
.../{customizations.bb => common.inc} | 4 +--
recipes-core/customizations/customizations.bb | 27 ++-----------------
2 files changed, 4 insertions(+), 27 deletions(-)
copy recipes-core/customizations/{customizations.bb => common.inc} (90%)recipes-core/customizations/common.inc renamed from recipes-core/customizations/customizations.bb +2 -2
@@ -1,7 +1,7 @@ # # CIP Core, generic profile # -# Copyright (c) Siemens AG, 2019 +# Copyright (c) Siemens AG, 2019-2022 # # Authors: # Jan Kiszka <[email protected]>
@@ -11,7 +11,7 @@ inherit dpkg-raw -DESCRIPTION = "CIP Core image demo & customizations" +FILESPATH_append := ":${FILE_DIRNAME}/files" SRC_URI = " \ file://postinst \
recipes-core/customizations/customizations.bb +2 -25
@@ -1,7 +1,7 @@ # # CIP Core, generic profile # -# Copyright (c) Siemens AG, 2019 +# Copyright (c) Siemens AG, 2019-2022 # # Authors: # Jan Kiszka <[email protected]>
@@ -9,29 +9,6 @@ # SPDX-License-Identifier: MIT # -inherit dpkg-raw +require common.inc DESCRIPTION = "CIP Core image demo & customizations" - -SRC_URI = " \ - file://postinst \ - file://ethernet \ - file://99-silent-printk.conf" - -WIRELESS_FIRMWARE_PACKAGE ?= "" -INSTALL_WIRELESS_TOOLS ??= "0" - -DEPENDS += "sshd-regen-keys" - -DEBIAN_DEPENDS = " \ - ifupdown, isc-dhcp-client, net-tools, iputils-ping, ssh, sshd-regen-keys \ - ${@(', iw, wireless-regdb, ' + d.getVar('WIRELESS_FIRMWARE_PACKAGE')) \ - if d.getVar('INSTALL_WIRELESS_TOOLS') == '1' else ''}" - -do_install() { - install -v -d ${D}/etc/network/interfaces.d - install -v -m 644 ${WORKDIR}/ethernet ${D}/etc/network/interfaces.d/ - - install -v -d ${D}/etc/sysctl.d - install -v -m 644 ${WORKDIR}/99-silent-printk.conf ${D}/etc/sysctl.d/ -}
Signature · 1 line
2.31.1
PATCH v3 6/6 kernelci-customizations: Reuse common customizations
In reply to Jan KiszkaFrom: Jan Kiszka <[email protected]>
We can share most of the customization steps for kernelci with the common recipe. This comes with the theoretical risk that changes to the latter will break the former but has the larger benefit of avoiding duplications and gaining support for new boards etc. automatically.
--- .../files/99-silent-printk.conf | 1 - .../kernelci-customizations/files/ethernet | 23 ------------------- .../kernelci-customizations.bb | 23 ++++--------------- 3 files changed, 4 insertions(+), 43 deletions(-) delete mode 100644 recipes-core/kernelci-customizations/files/99-silent-printk.conf delete mode 100644 recipes-core/kernelci-customizations/files/ethernet
recipes-core/kernelci-customizations/files/99-silent-printk.conf deleted +0 -1
@@ -1 +0,0 @@ -kernel.printk = 3 4 1 3
recipes-core/kernelci-customizations/files/ethernet deleted +0 -23
@@ -1,23 +0,0 @@ -# -# CIP Core, generic profile -# -# Copyright (c) Siemens AG, 2019 -# -# Authors: -# Jan Kiszka <[email protected]> -# -# SPDX-License-Identifier: MIT -# - -allow-hotplug eth0 -allow-hotplug enp0s2 -allow-hotplug enp2s0 - -# used on BBB -iface eth0 inet dhcp - -# used on qemu-amd64 -iface enp0s2 inet dhcp - -# used on simatic-ipc227e -iface enp2s0 inet dhcp
recipes-core/kernelci-customizations/kernelci-customizations.bb +4 -19
@@ -11,28 +11,13 @@ # SPDX-License-Identifier: MIT # -inherit dpkg-raw +require recipes-core/customizations/common.inc -DESCRIPTION = "CIP Core image demo & customizations" +DESCRIPTION = "CIP Core KernelCI image customizations" -SRC_URI = " \ - file://postinst \ - file://ethernet \ - file://dmesg.sh \ - file://99-silent-printk.conf" - -DEPENDS += "sshd-regen-keys" - -DEBIAN_DEPENDS = " \ - ifupdown, isc-dhcp-client, net-tools, iputils-ping, ssh, sshd-regen-keys" - -do_install() { - install -v -d ${D}/etc/network/interfaces.d - install -v -m 644 ${WORKDIR}/ethernet ${D}/etc/network/interfaces.d/ - - install -v -d ${D}/etc/sysctl.d - install -v -m 644 ${WORKDIR}/99-silent-printk.conf ${D}/etc/sysctl.d/ +SRC_URI += "file://dmesg.sh" +do_install_append() { install -v -d ${D}/opt/kernelci install -v -m 744 ${WORKDIR}/dmesg.sh ${D}/opt/kernelci/ }
Signature · 1 line
2.31.1