pkgdata and buildhistory improvements
cover pkgdata and buildhistory improvements
Fix up pkgdata, and change buildhistory to read from it rather than collecting package information directly; also make some performance and content improvements to buildhistory at the same time.
Note: there are minor changes necessary for Hob and Toaster UI code in bitbake to match up with these; they have been sent separately to the bitbake-devel list.
The following changes since commit 9cf1ac73c4e35101a4f5c01a5e1c53f9d567bc58:
oe-init-build-env-memres: Add auto port functionality (2013-12-02 11:27:46 +0000)
are available in the git repository at:
git://git.openembedded.org/openembedded-core-contrib paule/buildhistory
http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=paule/buildhistory
Paul Eggleton (8):
classes/package: fix FILES_INFO serialisation in pkgdata classes/package: record PKGSIZE as total file size in pkgdata classes/package: write PE and PKGE out to pkgdata classes/buildhistory: drop cruft from old SRCREV tracking
implementation
classes/buildhistory: add additional variables to image information classes/buildhistory: improve collection of package info classes/buildhistory: reduce parsing time classes/buildhistory: do git garbage collection after committing
meta/classes/buildhistory.bbclass | 118 +++++++++++++++++------------------ meta/classes/package.bbclass | 33 +++++----- meta/classes/toaster.bbclass | 28 ++------- meta/lib/oe/buildhistory_analysis.py | 4 +- scripts/oe-pkgdata-util | 25 ++++---- 5 files changed, 92 insertions(+), 116 deletions(-)
Signature · 1 line
1.8.1.2
PATCH 1/8 classes/package: fix FILES_INFO serialisation in pkgdata
The FILES_INFO entry in each pkgdata file stores the list of files for each package. Make the following improvements to how this is stored:
* Store paths as they would be seen on the target rather than erroneously including the full path to PKGDEST (which is specific to the build host the package was built on)
* For simplicity when loading the data, store complete paths for each entry instead of trying to break off the first part and use it as the dict key
* Record sizes for each file (as needed by Toaster)
* Serialise the value explicitly using json rather than just passing it through str().
Fixes [YOCTO #5443].
--- meta/classes/package.bbclass | 19 +++++++++---------- meta/classes/toaster.bbclass | 28 +++++----------------------- scripts/oe-pkgdata-util | 19 +++++++------------ 3 files changed, 21 insertions(+), 45 deletions(-)
meta/classes/package.bbclass +9 -10
@@ -1110,6 +1110,7 @@ PKGDESTWORK = "${WORKDIR}/pkgdata" python emit_pkgdata() { from glob import glob + import json def write_if_exists(f, pkg, var): def encode(str):
@@ -1173,22 +1174,20 @@ python emit_pkgdata() { workdir = d.getVar('WORKDIR', True) for pkg in packages.split(): - items = {} - for files_list in pkgfiles[pkg]: - item_name = os.path.basename(files_list) - item_path = os.path.dirname(files_list) - if item_path not in items: - items[item_path] = [] - items[item_path].append(item_name) - subdata_file = pkgdatadir + "/runtime/%s" % pkg - pkgval = d.getVar('PKG_%s' % pkg, True) if pkgval is None: pkgval = pkg d.setVar('PKG_%s' % pkg, pkg) - d.setVar('FILES_INFO', str(items)) + pkgdestpkg = os.path.join(pkgdest, pkg) + files = {} + for f in pkgfiles[pkg]: + relpth = os.path.relpath(f, pkgdestpkg) + fstat = os.lstat(f) + files[os.sep + relpth] = fstat.st_size + d.setVar('FILES_INFO', json.dumps(files)) + subdata_file = pkgdatadir + "/runtime/%s" % pkg sf = open(subdata_file, 'w') write_if_exists(sf, pkg, 'PN') write_if_exists(sf, pkg, 'PV')
meta/classes/toaster.bbclass +5 -23
@@ -39,8 +39,7 @@ python toaster_package_dumpdata() { # scan and send data for each package - import ast - import fnmatch + import json lpkgdata = {} for pkg in packages.split():
@@ -54,28 +53,11 @@ python toaster_package_dumpdata() { (n, v) = line.rstrip().split(":", 1) if pkg in n: n = n.replace("_" + pkg, "") - lpkgdata[n] = v.strip() - line = sf.readline() - pkgsplitname = os.path.join(pkgdest, pkg) - # replace FILES_INFO data with a dictionary of file name - file size if n == 'FILES_INFO': - filesizedata = {} - val = v.strip().replace('\\\'', '\'') - dictval = ast.literal_eval(val) - for parent, dirlist in dictval.items(): - idx = parent.find(pkgsplitname) - if idx > -1: - parent = parent[idx+len(pkgsplitname):] - else: - bb.error("Invalid path while looking for file ", parent) - for basename in dirlist: - fullpath = os.path.join(parent, basename) - try: - filesizedata[fullpath] = os.stat(pkgsplitname + fullpath).st_size - except OSError: - # we may hit a symlink that is not pointing correctly over package-split - filesizedata[fullpath] = 0 - lpkgdata[n] = filesizedata + lpkgdata[n] = json.loads(v) + else: + lpkgdata[n] = v.strip() + line = sf.readline() # Fire an event containing the pkg data bb.event.fire(bb.event.MetadataEvent("SinglePackageInfo", lpkgdata), d)
scripts/oe-pkgdata-util +7 -12
@@ -262,25 +262,20 @@ def find_path(args, usage, debug=False): print('ERROR: Unable to find pkgdata directory %s' % pkgdata_dir) sys.exit(1) - import ast + import json import fnmatch for root, dirs, files in os.walk(os.path.join(pkgdata_dir, 'runtime')): for fn in files: - pkgsplitname = '/packages-split/%s' % fn with open(os.path.join(root,fn)) as f: for line in f: if line.startswith('FILES_INFO:'): - val = line.split(':', 1)[1].strip().replace('\\\'', '\'') - dictval = ast.literal_eval(val) - for parent, dirlist in dictval.items(): - idx = parent.find(pkgsplitname) - if idx > -1: - parent = parent[idx+len(pkgsplitname):] - for basename in dirlist: - fullpth = os.path.join(parent, basename) - if fnmatch.fnmatchcase(fullpth, targetpath): - print("%s: %s" % (fn, fullpth)) + val = line.split(':', 1)[1].strip() + dictval = json.loads(val) + for fullpth in dictval.keys(): + if fnmatch.fnmatchcase(fullpth, targetpath): + print("%s: %s" % (fn, fullpth)) + break def main():
Signature · 1 line
1.8.1.2
PATCH 2/8 classes/package: record PKGSIZE as total file size in pkgdata
In reply to Paul EggletonWe were using "du -sk" to collect the total size of all files in each package for writing out to PKGSIZE in each pkgdata file; however this reports the total space used on disk not the total size of all files, which means it is dependent on the block size and filesystem being used for TMPDIR on the build host. Instead, take the total of the size reported by lstat() for each packaged file, which we are already collecting for FILES_INFO in any case.
Note: this changes PKGSIZE to be reported in bytes instead of kilobytes since this is what lstat reports, but this is really what we should be storing anyway so that we have the precision if we need it.
Fixes [YOCTO #5334]
--- meta/classes/package.bbclass | 12 +++--------- scripts/oe-pkgdata-util | 6 +++++- 2 files changed, 8 insertions(+), 10 deletions(-)
meta/classes/package.bbclass +3 -9
@@ -1127,14 +1127,6 @@ python emit_pkgdata() { f.write('%s: %s\n' % (var, encode(val))) return - def get_directory_size(dir): - if os.listdir(dir): - with os.popen('du -sk %s' % dir) as f: - size = int(f.readlines()[0].split('\t')[0]) - else: - size = 0 - return size - def write_extra_pkgs(variants, pn, packages, pkgdatadir): for variant in variants: with open("%s/%s-%s" % (pkgdatadir, variant, pn), 'w') as fd:
@@ -1181,9 +1173,11 @@ python emit_pkgdata() { pkgdestpkg = os.path.join(pkgdest, pkg) files = {} + total_size = 0 for f in pkgfiles[pkg]: relpth = os.path.relpath(f, pkgdestpkg) fstat = os.lstat(f) + total_size += fstat.st_size files[os.sep + relpth] = fstat.st_size d.setVar('FILES_INFO', json.dumps(files))
@@ -1220,7 +1214,7 @@ python emit_pkgdata() { for dfile in (d.getVar('FILERDEPENDSFLIST_' + pkg, True) or "").split(): write_if_exists(sf, pkg, 'FILERDEPENDS_' + dfile) - sf.write('%s_%s: %s\n' % ('PKGSIZE', pkg, get_directory_size(pkgdest + "/%s" % pkg))) + sf.write('%s_%s: %d\n' % ('PKGSIZE', pkg, total_size)) sf.close() # Symlinks needed for reverse lookups (from the final package name)
scripts/oe-pkgdata-util +5 -1
@@ -184,7 +184,11 @@ def read_value(args, usage, debug=False): if qvar == "PKGSIZE": # append packagename qvar = "%s_%s" % (var, mappedpkg) - print(readvar(revlink, qvar)) + # PKGSIZE is now in bytes, but we we want it in KB + pkgsize = (int(readvar(revlink, qvar)) + 1024 // 2) // 1024 + print("%d" % pkgsize) + else: + print(readvar(revlink, qvar)) def lookup_pkg(args, usage, debug=False): if len(args) < 2:
Signature · 1 line
1.8.1.2
PATCH 3/8 classes/package: write PE and PKGE out to pkgdata
In reply to Paul EggletonThese are important parts of the version for every package, so we should include them in PKGDATA just as we include PV/PR/PKGV/PKGR.
--- meta/classes/package.bbclass | 2 ++ 1 file changed, 2 insertions(+)
meta/classes/package.bbclass +2 -0
@@ -1184,8 +1184,10 @@ python emit_pkgdata() { subdata_file = pkgdatadir + "/runtime/%s" % pkg sf = open(subdata_file, 'w') write_if_exists(sf, pkg, 'PN') + write_if_exists(sf, pkg, 'PE') write_if_exists(sf, pkg, 'PV') write_if_exists(sf, pkg, 'PR') + write_if_exists(sf, pkg, 'PKGE') write_if_exists(sf, pkg, 'PKGV') write_if_exists(sf, pkg, 'PKGR') write_if_exists(sf, pkg, 'LICENSE')
Signature · 1 line
1.8.1.2
PATCH 4/8 classes/buildhistory: drop cruft from old SRCREV tracking implementation
In reply to Paul EggletonThis should have been removed when the implementation was rewritten in OE-Core commit 2179db89436d719635f858c87d1e098696bead2a. The collected values weren't being used anywhere since then.
--- meta/classes/buildhistory.bbclass | 12 ------------ 1 file changed, 12 deletions(-)
meta/classes/buildhistory.bbclass +0 -12
@@ -46,10 +46,7 @@ python buildhistory_emit_pkghistory() { self.pr = "r0" self.depends = "" self.packages = "" - self.bbfile = "" - self.src_uri = "" self.srcrev = "" - self.srcrev_autorev = "" class PackageInfo:
@@ -159,11 +156,6 @@ python buildhistory_emit_pkghistory() { pv = d.getVar('PV', True) pr = d.getVar('PR', True) - bbfile = d.getVar('BB_FILENAME', True) - src_uri = d.getVar('SRC_URI', True) - srcrev = d.getVar('SRCREV', True) - srcrev_autorev = 'yes' if d.getVar('SRCREV', False) == 'AUTOINC' else 'no' - packages = squashspaces(d.getVar('PACKAGES', True)) packagelist = packages.split()
@@ -184,10 +176,6 @@ python buildhistory_emit_pkghistory() { rcpinfo.pv = pv rcpinfo.pr = pr rcpinfo.depends = sortlist(squashspaces(d.getVar('DEPENDS', True) or "")) - rcpinfo.bbfile = bbfile - rcpinfo.src_uri = src_uri - rcpinfo.srcrev = srcrev - rcpinfo.srcrev_autorev = srcrev_autorev rcpinfo.packages = packages write_recipehistory(rcpinfo, d)
Signature · 1 line
1.8.1.2
PATCH 5/8 classes/buildhistory: add additional variables to image information
In reply to Paul EggletonAdd PACKAGE_EXCLUDE and NO_RECOMMENDATIONS to the info we track for images, since these can change what ends up in the image.
--- meta/classes/buildhistory.bbclass | 8 ++++---- meta/lib/oe/buildhistory_analysis.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-)
meta/classes/buildhistory.bbclass +4 -4
@@ -473,13 +473,13 @@ def outputvars(vars, listvars, d): return ret.rstrip('\n') def buildhistory_get_imagevars(d): - imagevars = "DISTRO DISTRO_VERSION USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS ROOTFS_POSTPROCESS_COMMAND IMAGE_POSTPROCESS_COMMAND" - listvars = "USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS" + imagevars = "DISTRO DISTRO_VERSION USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE ROOTFS_POSTPROCESS_COMMAND IMAGE_POSTPROCESS_COMMAND" + listvars = "USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS PACKAGE_EXCLUDE" return outputvars(imagevars, listvars, d) def buildhistory_get_sdkvars(d): - sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE SDKIMAGE_FEATURES BAD_RECOMMENDATIONS" - listvars = "SDKIMAGE_FEATURES BAD_RECOMMENDATIONS" + sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE SDKIMAGE_FEATURES BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE" + listvars = "SDKIMAGE_FEATURES BAD_RECOMMENDATIONS PACKAGE_EXCLUDE" return outputvars(sdkvars, listvars, d)
meta/lib/oe/buildhistory_analysis.py +2 -2
@@ -17,7 +17,7 @@ import bb.utils # How to display fields -list_fields = ['DEPENDS', 'RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS', 'FILES', 'FILELIST', 'USER_CLASSES', 'IMAGE_CLASSES', 'IMAGE_FEATURES', 'IMAGE_LINGUAS', 'IMAGE_INSTALL', 'BAD_RECOMMENDATIONS'] +list_fields = ['DEPENDS', 'RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS', 'FILES', 'FILELIST', 'USER_CLASSES', 'IMAGE_CLASSES', 'IMAGE_FEATURES', 'IMAGE_LINGUAS', 'IMAGE_INSTALL', 'BAD_RECOMMENDATIONS', 'PACKAGE_EXCLUDE'] list_order_fields = ['PACKAGES'] defaultval_map = {'PKG': 'PKG', 'PKGE': 'PE', 'PKGV': 'PV', 'PKGR': 'PR'} numeric_fields = ['PKGSIZE', 'IMAGESIZE']
@@ -35,7 +35,7 @@ related_fields['RRECOMMENDS'] = ['DEPENDS'] related_fields['FILELIST'] = ['FILES'] related_fields['PKGSIZE'] = ['FILELIST'] related_fields['files-in-image.txt'] = ['installed-package-names.txt', 'USER_CLASSES', 'IMAGE_CLASSES', 'ROOTFS_POSTPROCESS_COMMAND', 'IMAGE_POSTPROCESS_COMMAND'] -related_fields['installed-package-names.txt'] = ['IMAGE_FEATURES', 'IMAGE_LINGUAS', 'IMAGE_INSTALL', 'BAD_RECOMMENDATIONS'] +related_fields['installed-package-names.txt'] = ['IMAGE_FEATURES', 'IMAGE_LINGUAS', 'IMAGE_INSTALL', 'BAD_RECOMMENDATIONS', 'NO_RECOMMENDATIONS', 'PACKAGE_EXCLUDE'] class ChangeRecord:
Signature · 1 line
1.8.1.2
PATCH 6/8 classes/buildhistory: improve collection of package info
In reply to Paul EggletonUse a function added to SSTATEPOSTINSTFUNCS and read the necessary information out of pkgdata, instead of using a function executed during do_package that reads the data directly. This has two benefits:
* The package info collection will now work when the package content is restored from shared state
* Adding/removing the inherit of buildhistory will no longer change the do_package signatures and force re-execution of that function for every recipe.
Fixes [YOCTO #5358]
--- meta/classes/buildhistory.bbclass | 91 +++++++++++++++++++++------------------ 1 file changed, 48 insertions(+), 43 deletions(-)
meta/classes/buildhistory.bbclass +48 -43
@@ -17,25 +17,22 @@ BUILDHISTORY_COMMIT ?= "0" BUILDHISTORY_COMMIT_AUTHOR ?= "buildhistory <buildhistory@${DISTRO}>" BUILDHISTORY_PUSH_REPO ?= "" -# Must inherit package first before changing PACKAGEFUNCS -inherit package -PACKAGEFUNCS += "buildhistory_emit_pkghistory" - -# We don't want to force a rerun of do_package for everything -# if the buildhistory_emit_pkghistory function or any of the -# variables it refers to changes -do_package[vardepsexclude] += "buildhistory_emit_pkghistory" +SSTATEPOSTINSTFUNCS += "buildhistory_emit_pkghistory" # -# Called during do_package to write out metadata about this package -# for comparision when writing future packages +# Write out metadata about this package for comparision when writing future packages # python buildhistory_emit_pkghistory() { - import re + if not d.getVar('BB_CURRENTTASK', True) in ['packagedata', 'packagedata_setscene']: + return 0 if not "package" in (d.getVar('BUILDHISTORY_FEATURES', True) or "").split(): return 0 + import re + import json + import errno + pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) class RecipeInfo:
@@ -75,14 +72,6 @@ python buildhistory_emit_pkghistory() { # Should check PACKAGES here to see if anything removed - def getpkgvar(pkg, var): - val = bb.data.getVar('%s_%s' % (var, pkg), d, 1) - if val: - return val - val = bb.data.getVar('%s' % (var), d, 1) - - return val - def readPackageInfo(pkg, histfile): pkginfo = PackageInfo(pkg) with open(histfile, "r") as f:
@@ -156,7 +145,20 @@ python buildhistory_emit_pkghistory() { pv = d.getVar('PV', True) pr = d.getVar('PR', True) - packages = squashspaces(d.getVar('PACKAGES', True)) + pkgdata_dir = d.getVar('PKGDATA_DIR', True) + packages = "" + try: + with open(os.path.join(pkgdata_dir, pn)) as f: + for line in f.readlines(): + if line.startswith('PACKAGES: '): + packages = squashspaces(line.split(': ', 1)[1]) + break + except IOError as e: + if e.errno == errno.ENOENT: + # Probably a -cross recipe, just ignore + return 0 + else: + raise packagelist = packages.split() if not os.path.exists(pkghistdir):
@@ -181,9 +183,15 @@ python buildhistory_emit_pkghistory() { pkgdest = d.getVar('PKGDEST', True) for pkg in packagelist: - pkge = getpkgvar(pkg, 'PKGE') or "0" - pkgv = getpkgvar(pkg, 'PKGV') - pkgr = getpkgvar(pkg, 'PKGR') + pkgdata = {} + with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f: + for line in f.readlines(): + item = line.rstrip('\n').split(': ', 1) + pkgdata[item[0]] = item[1].decode('string_escape') + + pkge = pkgdata.get('PKGE', '0') + pkgv = pkgdata['PKGV'] + pkgr = pkgdata['PKGR'] # # Find out what the last version was # Make sure the version did not decrease
@@ -200,35 +208,32 @@ python buildhistory_emit_pkghistory() { pkginfo = PackageInfo(pkg) # Apparently the version can be different on a per-package basis (see Python) - pkginfo.pe = getpkgvar(pkg, 'PE') or "0" - pkginfo.pv = getpkgvar(pkg, 'PV') - pkginfo.pr = getpkgvar(pkg, 'PR') - pkginfo.pkg = getpkgvar(pkg, 'PKG') or pkg + pkginfo.pe = pkgdata.get('PE', '0') + pkginfo.pv = pkgdata['PV'] + pkginfo.pr = pkgdata['PR'] + pkginfo.pkg = pkgdata['PKG_%s' % pkg] pkginfo.pkge = pkge pkginfo.pkgv = pkgv pkginfo.pkgr = pkgr - pkginfo.rprovides = sortpkglist(squashspaces(getpkgvar(pkg, 'RPROVIDES') or "")) - pkginfo.rdepends = sortpkglist(squashspaces(getpkgvar(pkg, 'RDEPENDS') or "")) - pkginfo.rrecommends = sortpkglist(squashspaces(getpkgvar(pkg, 'RRECOMMENDS') or "")) - pkginfo.rsuggests = sortpkglist(squashspaces(getpkgvar(pkg, 'RSUGGESTS') or "")) - pkginfo.rreplaces = sortpkglist(squashspaces(getpkgvar(pkg, 'RREPLACES') or "")) - pkginfo.rconflicts = sortpkglist(squashspaces(getpkgvar(pkg, 'RCONFLICTS') or "")) - pkginfo.files = squashspaces(getpkgvar(pkg, 'FILES') or "") + pkginfo.rprovides = sortpkglist(squashspaces(pkgdata.get('RPROVIDES_%s' % pkg, ""))) + pkginfo.rdepends = sortpkglist(squashspaces(pkgdata.get('RDEPENDS_%s' % pkg, ""))) + pkginfo.rrecommends = sortpkglist(squashspaces(pkgdata.get('RRECOMMENDS_%s' % pkg, ""))) + pkginfo.rsuggests = sortpkglist(squashspaces(pkgdata.get('RSUGGESTS_%s' % pkg, ""))) + pkginfo.rreplaces = sortpkglist(squashspaces(pkgdata.get('RREPLACES_%s' % pkg, ""))) + pkginfo.rconflicts = sortpkglist(squashspaces(pkgdata.get('RCONFLICTS_%s' % pkg, ""))) + pkginfo.files = squashspaces(pkgdata.get('FILES_%s' % pkg, "")) for filevar in pkginfo.filevars: - pkginfo.filevars[filevar] = getpkgvar(pkg, filevar) + pkginfo.filevars[filevar] = pkgdata.get('%s_%s' % (filevar, pkg), "") # Gather information about packaged files - pkgdestpkg = os.path.join(pkgdest, pkg) - filelist = [] - pkginfo.size = 0 - for f in pkgfiles[pkg]: - relpth = os.path.relpath(f, pkgdestpkg) - fstat = os.lstat(f) - pkginfo.size += fstat.st_size - filelist.append(os.sep + relpth) + val = pkgdata.get('FILES_INFO', '') + dictval = json.loads(val) + filelist = dictval.keys() filelist.sort() pkginfo.filelist = " ".join(filelist) + pkginfo.size = int(pkgdata['PKGSIZE_%s' % pkg]) + write_pkghistory(pkginfo, d) }
Signature · 1 line
1.8.1.2
PATCH 7/8 classes/buildhistory: reduce parsing time
In reply to Paul EggletonDisable several python functions if not parsing within the worker context. This avoids executing expensive operations while parsing recipes (which is unnecessary).
(Thanks to Richard Purdie for pointing out the issue and suggesting the workaround.)
--- meta/classes/buildhistory.bbclass | 6 ++++++ 1 file changed, 6 insertions(+)
meta/classes/buildhistory.bbclass +6 -0
@@ -448,6 +448,8 @@ POPULATE_SDK_POST_HOST_COMMAND_append = "buildhistory_get_sdk_installed host ; " SDK_POSTPROCESS_COMMAND += "buildhistory_get_sdkinfo ; " def buildhistory_get_layers(d): + if d.getVar('BB_WORKERCONTEXT', True) != '1': + return "" layertext = "Configured metadata layers:\n%s\n" % '\n'.join(get_layers_branch_rev(d)) return layertext
@@ -478,11 +480,15 @@ def outputvars(vars, listvars, d): return ret.rstrip('\n') def buildhistory_get_imagevars(d): + if d.getVar('BB_WORKERCONTEXT', True) != '1': + return "" imagevars = "DISTRO DISTRO_VERSION USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE ROOTFS_POSTPROCESS_COMMAND IMAGE_POSTPROCESS_COMMAND" listvars = "USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS PACKAGE_EXCLUDE" return outputvars(imagevars, listvars, d) def buildhistory_get_sdkvars(d): + if d.getVar('BB_WORKERCONTEXT', True) != '1': + return "" sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE SDKIMAGE_FEATURES BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE" listvars = "SDKIMAGE_FEATURES BAD_RECOMMENDATIONS PACKAGE_EXCLUDE" return outputvars(sdkvars, listvars, d)
Signature · 1 line
1.8.1.2
PATCH 8/8 classes/buildhistory: do git garbage collection after committing
In reply to Paul EggletonWe don't normally perform any operations (such as "git pull") that trigger "git gc --auto", thus garbage collection never happens which means performance of accessing the repository degrades noticeably over time. Add an explicit "git gc --auto" to clean things up when needed.
Thanks to Elijah Newren and Ross Burton for suggesting this.
--- meta/classes/buildhistory.bbclass | 1 + 1 file changed, 1 insertion(+)
meta/classes/buildhistory.bbclass +1 -0
@@ -533,6 +533,7 @@ END for entry in `echo "$repostatus" | awk '{print $2}' | awk -F/ '{print $1}' | sort | uniq` ; do git commit $entry metadata-revs -m "$entry: Build ${BUILDNAME} of ${DISTRO} ${DISTRO_VERSION} for machine ${MACHINE} on $HOSTNAME" -m "cmd: $CMDLINE" --author "${BUILDHISTORY_COMMIT_AUTHOR}" > /dev/null done + git gc --auto if [ "${BUILDHISTORY_PUSH_REPO}" != "" ] ; then git push -q ${BUILDHISTORY_PUSH_REPO} fi
Signature · 1 line
1.8.1.2