buildhistory: Fix intermittent package file list creation
PATCH v3 1/3 buildhistory: Fix intermittent package file list creation
From: Pedro Ferreira <[email protected]>
The directory that buildhistory_list_pkg_files writes to during do_package is created by do_packagedata so a clean buildhistory doesn't have files-in-package written during the first build since packagedata happens after do_package.
Ensure the output package folder is created to avoid missing files-in-package.txt files.
Also it ensures that in case of find fails we leave with a hard error instead of hiding the error on the for loop.
--- meta/classes/buildhistory.bbclass | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-)
meta/classes/buildhistory.bbclass +3 -6
@@ -597,15 +597,12 @@ buildhistory_list_files_no_owners() { buildhistory_list_pkg_files() { # Create individual files-in-package for each recipe's package - for pkgdir in $(find ${PKGDEST}/* -maxdepth 0 -type d); do + pkgdirlist=$(find ${PKGDEST}/* -maxdepth 0 -type d) + for pkgdir in $pkgdirlist; do pkgname=$(basename $pkgdir) outfolder="${BUILDHISTORY_DIR_PACKAGE}/$pkgname" outfile="$outfolder/files-in-package.txt" - # Make sure the output folder exists so we can create the file - if [ ! -d $outfolder ] ; then - bbdebug 2 "Folder $outfolder does not exist, file $outfile not created" - continue - fi + mkdir -p $outfolder buildhistory_list_files $pkgdir $outfile fakeroot done }
Signature · 1 line
2.34.1
PATCH v3 2/3 buildhistory: Restoring files from preserve list
From: Pedro Ferreira <[email protected]>
This fix will ensure that, when we activate feature BUILDHISTORY_RESET, files marked to keep on feature BUILDHISTORY_PRESERVE will indeed exist is buildhistory final path since they are moved to buildhistory/old but not restored at any point.
--- meta/classes/buildhistory.bbclass | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)
meta/classes/buildhistory.bbclass +23 -0
@@ -108,6 +108,7 @@ python buildhistory_emit_pkghistory() { import json import shlex import errno + import shutil pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') oldpkghistdir = d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE')
@@ -221,6 +222,20 @@ python buildhistory_emit_pkghistory() { items.sort() return ' '.join(items) + def preservebuildhistoryfiles(pkg, preserve): + if os.path.exists(os.path.join(oldpkghistdir, pkg)): + listofobjs = os.listdir(os.path.join(oldpkghistdir, pkg)) + for obj in listofobjs: + if obj not in preserve: + continue + try: + bb.utils.mkdirhier(os.path.join(pkghistdir, pkg)) + shutil.copyfile(os.path.join(oldpkghistdir, pkg, obj), os.path.join(pkghistdir, pkg, obj)) + except IOError as e: + bb.note("Unable to copy file. %s" % e) + except EnvironmentError as e: + bb.note("Unable to copy file. %s" % e) + pn = d.getVar('PN') pe = d.getVar('PE') or "0" pv = d.getVar('PV')
@@ -248,6 +263,14 @@ python buildhistory_emit_pkghistory() { if not os.path.exists(pkghistdir): bb.utils.mkdirhier(pkghistdir) else: + # We need to make sure that all files kept in + # buildhistory/old are restored successfully + # otherwise next block of code wont have files to + # check and purge + if d.getVar("BUILDHISTORY_RESET"): + for pkg in packagelist: + preservebuildhistoryfiles(pkg, preserve) + # Remove files for packages that no longer exist for item in os.listdir(pkghistdir): if item not in preserve:
Signature · 1 line
2.34.1
Re: [OE-core] [kirkstone][PATCH v3 2/3] buildhistory: Restoring files from preserve list
Hi,
you should request backporting:
https://lists.openembedded.org/g/openembedded-core/message/203338
as well as it now fails in kirkstone (and possibly in scarthgap as well) due to these buildhistory changes being backported (libdevmapper in meta-oe has the same issue, backport was already sent there, but I haven't seen one for meta-world-pkgdata).
Quoted 76 lines · Pedro Ferreira via lists.openembedded.org
From: Pedro Ferreira <[email protected]>
This fix will ensure that, when we activate feature
BUILDHISTORY_RESET, files marked to keep on featureBUILDHISTORY_PRESERVEwill indeed exist is buildhistory final path since they are moved to buildhistory/old but not restored at any point.Signed-off-by Pedro Ferreira <[email protected]>Signed-off-by Richard Purdie <[email protected]>(cherry picked from commit 9f68a45aa238ae5fcdfaca71ba0e7015e9cb720e)--- meta/classes/buildhistory.bbclass | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+)meta/classes/buildhistory.bbclass +23
-1@@ -108,6 +108,7 @@ python buildhistory_emit_pkghistory() { import json import shlex import errno + import shutil pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') oldpkghistdir = d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE')@@ -221,6 +222,20 @@ python buildhistory_emit_pkghistory() { items.sort() return ' '.join(items) + def preservebuildhistoryfiles(pkg, preserve): + if os.path.exists(os.path.join(oldpkghistdir, pkg)): + listofobjs = os.listdir(os.path.join(oldpkghistdir, pkg)) + for obj in listofobjs: + if obj not in preserve: + continue + try: + bb.utils.mkdirhier(os.path.join(pkghistdir, pkg)) + shutil.copyfile(os.path.join(oldpkghistdir, pkg, obj), os.path.join(pkghistdir, pkg, obj)) + except IOError as e: + bb.note("Unable to copy file. %s" % e) + except EnvironmentError as e: + bb.note("Unable to copy file. %s" % e) + pn = d.getVar('PN') pe = d.getVar('PE') or "0" pv = d.getVar('PV')@@ -248,6 +263,14 @@ python buildhistory_emit_pkghistory() { if not os.path.exists(pkghistdir): bb.utils.mkdirhier(pkghistdir) else: + # We need to make sure that all files kept in + # buildhistory/old are restored successfully + # otherwise next block of code wont have files to + # check and purge + if d.getVar("BUILDHISTORY_RESET"): + for pkg in packagelist: + preservebuildhistoryfiles(pkg, preserve) + # Remove files for packages that no longer exist for item in os.listdir(pkghistdir): if item not in preserve: --2.34.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#204663): https://lists.openembedded.org/g/openembedded-core/message/204663
Mute This Topic: https://lists.openembedded.org/mt/108536777/3617156
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
PATCH v3 3/3 buildhistory: Simplify intercept call sites and drop SSTATEPOSTINSTFUNC usage
In reply to Pedro FerreiraFrom: Richard Purdie <[email protected]>
We planned to drop SSTATEPOSTINSTFUNC some time ago with the introduction of postfuncs. Finally get around to doing that which should make the buildhistory code a little more readable.
Unfortunately ordering the buildhistory function calls after the sstate ones is difficult without coding that into the sstate class. This patch does that to ensure everything functions as expected until we can find a better way. This is still likely preferable than the generic sstate postfuncs support since the function flow is much more readable.
--- meta/classes/buildhistory.bbclass | 39 +++++++++++++++---------------- meta/classes/sstate.bbclass | 5 +++- 2 files changed, 23 insertions(+), 21 deletions(-)
meta/classes/buildhistory.bbclass +19 -20
@@ -45,11 +45,18 @@ BUILDHISTORY_PUSH_REPO ?= "" BUILDHISTORY_TAG ?= "build" BUILDHISTORY_PATH_PREFIX_STRIP ?= "" -SSTATEPOSTINSTFUNCS:append = " buildhistory_emit_pkghistory" -# We want to avoid influencing the signatures of sstate tasks - first the function itself: -sstate_install[vardepsexclude] += "buildhistory_emit_pkghistory" -# then the value added to SSTATEPOSTINSTFUNCS: -SSTATEPOSTINSTFUNCS[vardepvalueexclude] .= "| buildhistory_emit_pkghistory" +# We want to avoid influencing the signatures of the task so use vardepsexclude +do_populate_sysroot[postfuncs] += "buildhistory_emit_sysroot" +do_populate_sysroot_setscene[postfuncs] += "buildhistory_emit_sysroot" +do_populate_sysroot[vardepsexclude] += "buildhistory_emit_sysroot" + +do_package[postfuncs] += "buildhistory_list_pkg_files" +do_package_setscene[postfuncs] += "buildhistory_list_pkg_files" +do_package[vardepsexclude] += "buildhistory_list_pkg_files" + +do_packagedata[postfuncs] += "buildhistory_emit_pkghistory" +do_packagedata_setscene[postfuncs] += "buildhistory_emit_pkghistory" +do_packagedata[vardepsexclude] += "buildhistory_emit_pkghistory" # Similarly for our function that gets the output signatures SSTATEPOSTUNPACKFUNCS:append = " buildhistory_emit_outputsigs"
@@ -89,27 +96,15 @@ buildhistory_emit_sysroot() { # Write out metadata about this package for comparison when writing future packages # python buildhistory_emit_pkghistory() { - if d.getVar('BB_CURRENTTASK') in ['populate_sysroot', 'populate_sysroot_setscene']: - bb.build.exec_func("buildhistory_emit_sysroot", d) - return 0 - - if not "package" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): - return 0 - - if d.getVar('BB_CURRENTTASK') in ['package', 'package_setscene']: - # Create files-in-<package-name>.txt files containing a list of files of each recipe's package - bb.build.exec_func("buildhistory_list_pkg_files", d) - return 0 - - if not d.getVar('BB_CURRENTTASK') in ['packagedata', 'packagedata_setscene']: - return 0 - import re import json import shlex import errno import shutil + if not "package" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): + return 0 + pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') oldpkghistdir = d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE')
@@ -619,6 +614,10 @@ buildhistory_list_files_no_owners() { } buildhistory_list_pkg_files() { + if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'package', '1', '0', d)}" = "0" ] ; then + return + fi + # Create individual files-in-package for each recipe's package pkgdirlist=$(find ${PKGDEST}/* -maxdepth 0 -type d) for pkgdir in $pkgdirlist; do
meta/classes/sstate.bbclass +4 -1
@@ -156,7 +156,10 @@ python () { d.setVar('SSTATETASKS', " ".join(unique_tasks)) for task in unique_tasks: d.prependVarFlag(task, 'prefuncs', "sstate_task_prefunc ") - d.appendVarFlag(task, 'postfuncs', " sstate_task_postfunc") + # Generally sstate should be last, execpt for buildhistory functions + postfuncs = (d.getVarFlag(task, 'postfuncs') or "").split() + newpostfuncs = [p for p in postfuncs if "buildhistory" not in p] + ["sstate_task_postfunc"] + [p for p in postfuncs if "buildhistory" in p] + d.setVarFlag(task, 'postfuncs', " ".join(newpostfuncs)) d.setVarFlag(task, 'network', '1') d.setVarFlag(task + "_setscene", 'network', '1') }
Signature · 1 line
2.34.1