busybox: Remove PN from rdeps of PN-dev
PATCH 1/3 busybox: Remove PN from rdeps of PN-dev
PN-dev is empty and causes dependency loops during SDK build
--- meta/recipes-core/busybox/busybox.inc | 1 + 1 file changed, 1 insertion(+)
meta/recipes-core/busybox/busybox.inc +1 -0
@@ -21,6 +21,7 @@ export EXTRA_LDFLAGS = "${LDFLAGS}" EXTRA_OEMAKE = "CC='${CC}' LD='${CCLD}' V=1 ARCH=${TARGET_ARCH} CROSS_COMPILE=${TARGET_PREFIX} SKIP_STRIP=y HOSTCC='${BUILD_CC}' HOSTCPP='${BUILD_CPP}'" PACKAGES =+ "${PN}-httpd ${PN}-udhcpd ${PN}-udhcpc ${PN}-syslog ${PN}-mdev ${PN}-hwclock" +RDEPENDS_${PN}-dev = "" FILES_${PN}-httpd = "${sysconfdir}/init.d/busybox-httpd /srv/www" FILES_${PN}-syslog = "${sysconfdir}/init.d/syslog* ${sysconfdir}/syslog-startup.conf* ${sysconfdir}/syslog.conf* ${systemd_unitdir}/system/syslog.service ${sysconfdir}/default/busybox-syslog"
Signature · 1 line
2.20.1
PATCH 2/3 libc-package.bbclass: Use install utility to create directories with right permissions
Could fix issues where dirs with wrong permission is created with mkdir -p
--- meta/classes/libc-package.bbclass | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
meta/classes/libc-package.bbclass +1 -1
@@ -61,7 +61,7 @@ LOCALETREESRC ?= "${PKGD}" do_prep_locale_tree() { treedir=${WORKDIR}/locale-tree rm -rf $treedir - mkdir -p $treedir/${base_bindir} $treedir/${base_libdir} $treedir/${datadir} $treedir/${localedir} + install -d $treedir/${base_bindir} $treedir/${base_libdir} $treedir/${datadir} $treedir/${localedir} tar -cf - -C ${LOCALETREESRC}${datadir} -p i18n | tar -xf - -C $treedir/${datadir} # unzip to avoid parsing errors for i in $treedir/${datadir}/i18n/charmaps/*gz; do
Signature · 1 line
2.20.1
PATCH v3 3/3 image_types.bbclass: Set memory usage limit and CPU threads for xz
In reply to Khem Rajwhen building with opkg backend and huge packages e.g. chromium/llvm all going in parallel, memory pressure causes xz to catapult with
do_package_write_ipk: Failed to create package, opkg-build failed with: xz: (stdin): Cannot allocate memory
since there are many tasks going on in parallel, xz adds to memory pressure and it wants it all, put an upper limit for memory xz can use
We add a variable XZ_MAXRAM with 30% of RAM limit and can be customized if builders have more memory one can set it like
XZ_DEFAULTS = "-M 0 -T 0"
---
V2: Consider all locations of xz use
v3: Use ${@oe.utils.cpu_count()} in place of BB_NUMBER_THREADS
meta/classes/image_types.bbclass | 9 ++++----- meta/classes/package_ipk.bbclass | 2 +- meta/classes/populate_sdk_base.bbclass | 2 +- meta/conf/bitbake.conf | 3 +++ 4 files changed, 9 insertions(+), 7 deletions(-)
meta/classes/image_types.bbclass +4 -5
@@ -56,7 +56,6 @@ def imagetypes_getdepends(d): XZ_COMPRESSION_LEVEL ?= "-3" XZ_INTEGRITY_CHECK ?= "crc32" -XZ_THREADS ?= "-T 0" ZIP_COMPRESSION_LEVEL ?= "-9"
@@ -152,12 +151,12 @@ UBI_VOLNAME ?= "${MACHINE}-rootfs" multiubi_mkfs() { local mkubifs_args="$1" local ubinize_args="$2" - + # Added prompt error message for ubi and ubifs image creation. if [ -z "$mkubifs_args" ] || [ -z "$ubinize_args" ]; then bbfatal "MKUBIFS_ARGS and UBINIZE_ARGS have to be set, see http://www.linux-mtd.infradead.org/faq/ubifs.html for details" fi - + if [ -z "$3" ]; then local vname="" else
@@ -284,7 +283,7 @@ CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip sum md5sum sha1sum sha224sum sha25 CONVERSION_CMD_lzma = "lzma -k -f -7 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" CONVERSION_CMD_gz = "pigz -f -9 -n -c ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.gz" CONVERSION_CMD_bz2 = "pbzip2 -f -k ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" -CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_THREADS} --check=${XZ_INTEGRITY_CHECK} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.xz" +CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_DEFAULTS} --check=${XZ_INTEGRITY_CHECK} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.xz" CONVERSION_CMD_lz4 = "lz4 -9 -z -l ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.lz4" CONVERSION_CMD_lzo = "lzop -9 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" CONVERSION_CMD_zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zip ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
@@ -319,7 +318,7 @@ CONVERSION_DEPENDS_base64 = "coreutils-native" RUNNABLE_IMAGE_TYPES ?= "ext2 ext3 ext4" RUNNABLE_MACHINE_PATTERNS ?= "qemu" -DEPLOYABLE_IMAGE_TYPES ?= "hddimg iso" +DEPLOYABLE_IMAGE_TYPES ?= "hddimg iso" # The IMAGE_TYPES_MASKED variable will be used to mask out from the IMAGE_FSTYPES, # images that will not be built at do_rootfs time: vmdk, vdi, qcow2, hdddirect, hddimg, iso, etc.
meta/classes/package_ipk.bbclass +1 -1
@@ -8,7 +8,7 @@ IPKGCONF_SDK = "${WORKDIR}/opkg-sdk.conf" PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks" # Program to be used to build opkg packages -OPKGBUILDCMD ??= 'opkg-build -Z xz -a "--threads 0"' +OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"' OPKG_ARGS += "--force_postinstall --prefer-arch-to-version" OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}"
meta/classes/populate_sdk_base.bbclass +1 -1
@@ -226,7 +226,7 @@ fakeroot tar_sdk() { # Package it up mkdir -p ${SDKDEPLOYDIR} cd ${SDK_OUTPUT}/${SDKPATH} - tar ${SDKTAROPTS} -cf - . | xz -T 0 > ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.tar.xz + tar ${SDKTAROPTS} -cf - . | xz ${XZ_DEFAULTS} > ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.tar.xz } TOOLCHAIN_SHAR_EXT_TMPL ?= "${COREBASE}/meta/files/toolchain-shar-extract.sh"
meta/conf/bitbake.conf +3 -0
@@ -791,6 +791,9 @@ BB_NUMBER_THREADS ?= "${@oe.utils.cpu_count()}" # Default to setting automatically based on cpu count PARALLEL_MAKE ?= "-j ${@oe.utils.cpu_count()}" +# Default parallelism and resource usage for xz +XZ_DEFAULTS ?= "--memlimit=50% --threads=${@oe.utils.cpu_count()}" + ################################################################## # Magic Cookie for SANITY CHECK ##################################################################
--
2.20.1
Re: [PATCH v3 3/3] image_types.bbclass: Set memory usage limit and CPU threads for xz
Quoted 11 lines · Khem Raj
when building with opkg backend and huge packages e.g. chromium/llvm all going in parallel, memory pressure causes xz to catapult with
do_package_write_ipk: Failed to create package, opkg-build failed with: xz: (stdin): Cannot allocate memorysince there are many tasks going on in parallel, xz adds to memory pressure and it wants it all, put an upper limit for memory xz can use
We add a variable
XZ_MAXRAMwith 30% of RAM limit and can be customized if builders have more memory one can set it like
The commit log refers to the wrong variable name, it seems.
Signature · 3 lines
Otavio Salvador O.S. Systems http://www.ossystems.com.br http://code.ossystems.com.br Mobile: +55 (53) 9 9981-7854 Mobile: +1 (347) 903-9750
Re: [PATCH v3 3/3] image_types.bbclass: Set memory usage limit and CPU threads for xz
Quoted 8 lines · Otavio Salvador
Quoted 6 lines · Khem Raj
when building with opkg backend and huge packages e.g. chromium/llvm all going in parallel, memory pressure causes xz to catapult with
do_package_write_ipk: Failed to create package, opkg-build failed with: xz: (stdin): Cannot allocate memory
Quoted 8 lines
Quoted 5 lines
since there are many tasks going on in parallel, xz adds to memory pressure and it wants it all, put an upper limit for memory xz can use
We add a variable
XZ_MAXRAMwith 30% of RAM limit and can be customized if builders have more memory one can set it likeThe commit log refers to the wrong variable name, it seems.
I think thats a good point. but we should submit right names for documentation
Quoted 5 lines
--
Otavio SalvadorO.S. Systems
http://www.ossystems.com.br http://code.ossystems.com.br
Mobile: +55 (53) 9 9981-7854 Mobile: +1 (347) 903-9750
Would a neater solution be to PACKAGES_remove = ${PN}-dev?
Ross
Quoted 28 lines · Khem Raj
PN-dev is empty and causes dependency loops during SDK build
Signed-off-by Khem Raj <[email protected]>--- meta/recipes-core/busybox/busybox.inc | 1 + 1 file changed, 1 insertion(+)meta/recipes-core/busybox/busybox.inc +1
-1@@ -21,6 +21,7 @@ export EXTRA_LDFLAGS = "${LDFLAGS}" EXTRA_OEMAKE = "CC='${CC}' LD='${CCLD}' V=1 ARCH=${TARGET_ARCH} CROSS_COMPILE=${TARGET_PREFIX} SKIP_STRIP=y HOSTCC='${BUILD_CC}' HOSTCPP='${BUILD_CPP}'" PACKAGES =+ "${PN}-httpd ${PN}-udhcpd ${PN}-udhcpc ${PN}-syslog ${PN}-mdev ${PN}-hwclock" +RDEPENDS_${PN}-dev = "" FILES_${PN}-httpd = "${sysconfdir}/init.d/busybox-httpd /srv/www" FILES_${PN}-syslog = "${sysconfdir}/init.d/syslog* ${sysconfdir}/syslog-startup.conf* ${sysconfdir}/syslog.conf* ${systemd_unitdir}/system/syslog.service ${sysconfdir}/default/busybox-syslog" --2.20.1
--
_______________________________________________
Openembedded-core mailing list
[email protected]
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Quoted 3 lines · Ross
Would a neater solution be to
PACKAGES_remove= ${PN}-dev?
That was first thing I tried too. It does not fix the problem
Quoted 31 lines
Ross
Quoted 28 lines · Khem Raj
PN-dev is empty and causes dependency loops during SDK build
Signed-off-by Khem Raj <[email protected]>--- meta/recipes-core/busybox/busybox.inc | 1 + 1 file changed, 1 insertion(+)meta/recipes-core/busybox/busybox.inc +1
-1@@ -21,6 +21,7 @@ export EXTRA_LDFLAGS = "${LDFLAGS}" EXTRA_OEMAKE = "CC='${CC}' LD='${CCLD}' V=1 ARCH=${TARGET_ARCH} CROSS_COMPILE=${TARGET_PREFIX} SKIP_STRIP=y HOSTCC='${BUILD_CC}' HOSTCPP='${BUILD_CPP}'" PACKAGES =+ "${PN}-httpd ${PN}-udhcpd ${PN}-udhcpc ${PN}-syslog ${PN}-mdev ${PN}-hwclock" +RDEPENDS_${PN}-dev = "" FILES_${PN}-httpd = "${sysconfdir}/init.d/busybox-httpd /srv/www" FILES_${PN}-syslog = "${sysconfdir}/init.d/syslog* ${sysconfdir}/syslog-startup.conf* ${sysconfdir}/syslog.conf* ${systemd_unitdir}/system/syslog.service ${sysconfdir}/default/busybox-syslog" --2.20.1
--
_______________________________________________
Openembedded-core mailing list
[email protected]
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Richard/Ross
Are you able to reproduce the problem and have
alternative solution?
Quoted 39 lines · Khem Raj
Quoted 3 lines · Ross
Would a neater solution be to
PACKAGES_remove= ${PN}-dev?That was first thing I tried too. It does not fix the problem
Quoted 31 lines
Ross
Quoted 28 lines · Khem Raj
PN-dev is empty and causes dependency loops during SDK build
Signed-off-by Khem Raj <[email protected]>--- meta/recipes-core/busybox/busybox.inc | 1 + 1 file changed, 1 insertion(+)meta/recipes-core/busybox/busybox.inc +1
-1@@ -21,6 +21,7 @@ export EXTRA_LDFLAGS = "${LDFLAGS}" EXTRA_OEMAKE = "CC='${CC}' LD='${CCLD}' V=1 ARCH=${TARGET_ARCH} CROSS_COMPILE=${TARGET_PREFIX} SKIP_STRIP=y HOSTCC='${BUILD_CC}' HOSTCPP='${BUILD_CPP}'" PACKAGES =+ "${PN}-httpd ${PN}-udhcpd ${PN}-udhcpc ${PN}-syslog ${PN}-mdev ${PN}-hwclock" +RDEPENDS_${PN}-dev = "" FILES_${PN}-httpd = "${sysconfdir}/init.d/busybox-httpd /srv/www" FILES_${PN}-syslog = "${sysconfdir}/init.d/syslog* ${sysconfdir}/syslog-startup.conf* ${sysconfdir}/syslog.conf* ${systemd_unitdir}/system/syslog.service ${sysconfdir}/default/busybox-syslog" --2.20.1
--
_______________________________________________
Openembedded-core mailing list
[email protected]
http://lists.openembedded.org/mailman/listinfo/openembedded-core
Quoted 4 lines · Khem Raj
Richard/Ross
Are you able to reproduce the problem and have
alternative solution?
Your reproduction steps needed "use ipk", then yes, I could. I've sent out a patch which addresses the problem.
Cheers,
Richard
Quoted 9 lines · Richard Purdie
Quoted 4 lines · Khem Raj
Richard/Ross
Are you able to reproduce the problem and have
alternative solution?Your reproduction steps needed "use ipk", then yes, I could. I've sent out a patch which addresses the problem.
Thanks yes ipk was implied with oe-core+bitbake :) I will test https://patchwork.openembedded.org/patch/158751/
and see if my SDK is good with out busybox hack
Quoted 7 lines
Cheers,
Richard
Quoted 6 lines · Khem Raj
PN-dev is empty and causes dependency loops during SDK build
Signed-off-by Khem Raj <[email protected]>--- meta/recipes-core/busybox/busybox.inc | 1 + 1 file changed, 1 insertion(+)
I commented on this elsewhere but for completeness, this is a workaround, not a fix. The problem is that the PROVIDES in target-sdk- provides-dummy.bb are clearly missing something.
This means that the SDK tries to exclude "/bin/sh" and similar target dependencies from the SDK rootfs which leads to excluding busybox and then fails to do so. If busybox is installed, busybox-dev gets pulled in and the problems start.
The real fix is to figure out the missing target-sdk-provides-dummy provides.
If someone could share a way to reproduce the failure that would be a good start.
Cheers,
Richard
Quoted 18 lines · Richard Purdie
Quoted 6 lines · Khem Raj
PN-dev is empty and causes dependency loops during SDK build
Signed-off-by Khem Raj <[email protected]>--- meta/recipes-core/busybox/busybox.inc | 1 + 1 file changed, 1 insertion(+)I commented on this elsewhere but for completeness, this is a workaround, not a fix. The problem is that the PROVIDES in target-sdk- provides-dummy.bb are clearly missing something.
This means that the SDK tries to exclude "
/bin/sh" and similar target dependencies from the SDK rootfs which leads to excluding busybox and then fails to do so. If busybox is installed, busybox-dev gets pulled in and the problems start.
Quoted 6 lines
The real fix is to figure out the missing target-sdk-provides-dummy provides.
If someone could share a way to reproduce the failure that would be a good start.
Use oe-core+bitbake ( nodisro ) settings and then
bitbake -cpopulate_sdk core-image-minimal
and you see
WARNING: core-image-minimal-1.0-r0 do_populate_sdk: Unable to install
packages. Command
'/mnt/a/yoe/build/tmp/work/sama5d27_som1_ek_sd-yoe-linux-musleabi/core-image-minimal/1.0-r0/recipe-sysroot-native/usr/bin/opkg
--volatile-cache -f/mnt/a/yoe/build/tmp/work/sama5d27_som1_ek_sd-yoe-linux-musleabi/core-image-minimal/1.0-r0/opkg.conf
-t /mnt/a/yoe/build/tmp/work/sama5d27_som1_ek_sd-yoe-linux-musleabi/core-image-minimal/1.0-r0/temp/ipktemp/
-o /mnt/a/yoe/build/tmp/work/sama5d27_som1_ek_sd-yoe-linux-musleabi/core-image-minimal/1.0-r0/sdk/image/opt/yoe-musl-sysvinit-wayland/2.6/sysroots/cortexa5t2hf-neon-vfpv4-yoe-linux-musleabi
--force_postinstall --prefer-arch-to-version install libgcc-s-src
eudev-dbg update-rc.d-dev libcxx-dbg base-passwd-dev
sysvinit-inittab-dbg libz-dev modutils-initscripts-dbg initscripts-dev
opkg-utils-dbg musl-utils-dev libgcc-s-dbg init-ifupdown-dbg
packagegroup-core-standalone-sdk-target-dbg sysvinit-inittab-dev
busybox-dev gcc-runtime-dbg netbase-dbg kmod-dbg base-passwd-src
base-passwd-dbg initscripts-dbg eudev-dev sysvinit-dbg
run-postinsts-dbg target-sdk-provides-dummy-dbg base-files-dbg
musl-utils-dbg libz-dbg base-files-dev
packagegroup-core-standalone-sdk-target-dev musl-utils-src
compiler-rt-dbg dt-overlay-at91-dev busybox-dbg util-linux-dbg
kmod-dev util-linux-dev dt-overlay-at91-dbg packagegroup-core-boot-dev
sysvinit-src target-sdk-provides-dummy-dev sysvinit-dev
init-ifupdown-dev packagegroup-core-boot-dbg update-rc.d-dbg musl-src
modutils-initscripts-dev netbase-dev run-postinsts-dev' returned 1:
Collected errors:
* Solver encountered 1 problem(s):
* Problem 1/1:
* - package busybox-dev-1.29.3-r0.4.cortexa5t2hf-neon-vfpv4
requires busybox = 1.29.3-r0.4, but none of the providers can be
installed
*
* Solution 1:
* - allow deinstallation of
target-sdk-provides-dummy-1.0-r0.3.sdk-provides-dummy-target
* - do not ask to install a package providing target-sdk-provides-dummy-dev
* Solution 2:
* - do not ask to install a package providing busybox-dev
Quoted 4 lines
Cheers,
Richard