Buildhistory fixes
cover Buildhistory fixes
Fixes for a couple of buildhistory bugs in YP bugzilla plus one extra that I found in the process.
The following changes since commit f9324af88a99eca28b160fa31aa4516fd397e44b:
distutils3: pass build arguments when doing a clean (2018-07-10 11:10:14 +0100)
are available in the Git repository at:
git://git.openembedded.org/openembedded-core-contrib paule/buildhistory-fixes2
http://cgit.openembedded.org/openembedded-core-contrib/log/?h=paule/buildhistory-fixes2
Paul Eggleton (3):
lib/oe/buildhistory_analysis: drop related field feature classes/buildhistory: handle packaged files with names containing
spaces
classes/buildhistory: properly process escaped chars from pkgdata
meta/classes/buildhistory.bbclass | 5 ++-- meta/lib/oe/buildhistory_analysis.py | 42 ++++++++-------------------- 2 files changed, 15 insertions(+), 32 deletions(-)
Signature · 1 line
2.17.1
PATCH 1/3 lib/oe/buildhistory_analysis: drop related field feature
The original idea here was that changes to certain fields might be able to be explained if there was a change to another field, for example if RDEPENDS changed it might be because DEPENDS changed. Thus we were printing this kind of thing out with each change. Unfortunately in practice this turned out to be noisy and not particularly useful, so we might as well remove it.
Fixes [YOCTO #7336].
--- meta/lib/oe/buildhistory_analysis.py | 26 -------------------------- 1 file changed, 26 deletions(-)
meta/lib/oe/buildhistory_analysis.py +0 -26
@@ -31,13 +31,6 @@ ver_monitor_fields = ['PKGE', 'PKGV', 'PKGR'] monitor_numeric_threshold = 10 # Image files to monitor (note that image-info.txt is handled separately) img_monitor_files = ['installed-package-names.txt', 'files-in-image.txt'] -# Related context fields for reporting (note: PE, PV & PR are always reported for monitored package fields) -related_fields = {} -related_fields['RDEPENDS'] = ['DEPENDS'] -related_fields['RRECOMMENDS'] = ['DEPENDS'] -related_fields['FILELIST'] = ['FILES'] -related_fields['files-in-image.txt'] = ['installed-package-names.txt', 'USER_CLASSES', 'IMAGE_CLASSES', 'ROOTFS_POSTPROCESS_COMMAND', 'IMAGE_POSTPROCESS_COMMAND'] -related_fields['installed-package-names.txt'] = ['IMAGE_FEATURES', 'IMAGE_LINGUAS', 'IMAGE_INSTALL', 'BAD_RECOMMENDATIONS', 'NO_RECOMMENDATIONS', 'PACKAGE_EXCLUDE'] colours = { 'colour_default': '',
@@ -67,7 +60,6 @@ class ChangeRecord: self.oldvalue = oldvalue self.newvalue = newvalue self.monitored = monitored - self.related = [] self.filechanges = None def __str__(self):
@@ -206,13 +198,6 @@ class ChangeRecord: else: out = '{} changed from "{colour_remove}{}{colour_default}" to "{colour_add}{}{colour_default}"'.format(self.fieldname, self.oldvalue, self.newvalue, **colours) - if self.related: - for chg in self.related: - if not outer and chg.fieldname in ['PE', 'PV', 'PR']: - continue - for line in chg._str_internal(False).splitlines(): - out += '\n * %s' % line - return '%s%s' % (prefix, out) if out else '' class FileChange:
@@ -635,17 +620,6 @@ def process_changes(repopath, revision1, revision2='HEAD', report_all=False, rep chg = ChangeRecord(path, filename[7:], d.a_blob.data_stream.read().decode('utf-8'), '', True) changes.append(chg) - # Link related changes - for chg in changes: - if chg.monitored: - for chg2 in changes: - # (Check dirname in the case of fields from recipe info files) - if chg.path == chg2.path or os.path.dirname(chg.path) == chg2.path: - if chg2.fieldname in related_fields.get(chg.fieldname, []): - chg.related.append(chg2) - elif chg.path == chg2.path and chg.path.startswith('packages/') and chg2.fieldname in ['PE', 'PV', 'PR']: - chg.related.append(chg2) - # filter out unwanted paths if exclude_path: for chg in changes:
Signature · 1 line
2.17.1
PATCH 2/3 classes/buildhistory: handle packaged files with names containing spaces
In reply to Paul EggletonThe FILELIST field of the package info file in the buildhistory repository is a space-separated list of all of the files in the package. If a name of a file packaged by a recipe contains a space character then of course the result was that we didn't handle its name properly. To fix that, use quotes around any filename containing spaces and at the other end use these quotes to extract the proper entries.
Fixes [YOCTO #12742].
--- meta/classes/buildhistory.bbclass | 3 ++- meta/lib/oe/buildhistory_analysis.py | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-)
meta/classes/buildhistory.bbclass +2 -1
@@ -77,6 +77,7 @@ python buildhistory_emit_pkghistory() { import re import json + import shlex import errno pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE')
@@ -287,7 +288,7 @@ python buildhistory_emit_pkghistory() { dictval = json.loads(val) filelist = list(dictval.keys()) filelist.sort() - pkginfo.filelist = " ".join(filelist) + pkginfo.filelist = " ".join([shlex.quote(x) for x in filelist]) pkginfo.size = int(pkgdata['PKGSIZE'])
meta/lib/oe/buildhistory_analysis.py +12 -4
@@ -13,6 +13,7 @@ import os.path import difflib import git import re +import shlex import hashlib import collections import bb.utils
@@ -115,10 +116,13 @@ class ChangeRecord: aitems = pkglist_combine(depvera) bitems = pkglist_combine(depverb) else: - aitems = self.oldvalue.split() - bitems = self.newvalue.split() if self.fieldname == 'FILELIST': + aitems = shlex.split(self.oldvalue) + bitems = shlex.split(self.newvalue) renamed_dirs, aitems, bitems = detect_renamed_dirs(aitems, bitems) + else: + aitems = self.oldvalue.split() + bitems = self.newvalue.split() removed = list(set(aitems) - set(bitems)) added = list(set(bitems) - set(aitems))
@@ -409,9 +413,13 @@ def compare_dict_blobs(path, ablob, bblob, report_all, report_ver): (depvera, depverb) = compare_pkg_lists(astr, bstr) if depvera == depverb: continue - alist = astr.split() + if key == 'FILELIST': + alist = shlex.split(astr) + blist = shlex.split(bstr) + else: + alist = astr.split() + blist = bstr.split() alist.sort() - blist = bstr.split() blist.sort() # We don't care about the removal of self-dependencies if pkgname in alist and not pkgname in blist:
Signature · 1 line
2.17.1
PATCH 3/3 classes/buildhistory: properly process escaped chars from pkgdata
In reply to Paul EggletonAll values written out to pkgdata are escaped (see write_if_exists() in package.bbclass). In practice there tend not to be characters that need escaping except in the scriptlets (pkg_preinst, pkg_postinst, pkg_prerm and pkg_postrm) where currently we still see the escape codes in the corresponding files within buildhistory (e.g. \n and \t) and thus also in the output of buildhistory-diff, hindering proper diffing of changes. To fix this, when we read values from pkgdata and write them out to buildhistory, we need to interpret the escape codes by doing the exact reverse of what we do in package.bbclass.
--- meta/classes/buildhistory.bbclass | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
meta/classes/buildhistory.bbclass +1 -1
@@ -245,7 +245,7 @@ python buildhistory_emit_pkghistory() { key = item[0] if key.endswith('_' + pkg): key = key[:-len(pkg)-1] - pkgdata[key] = item[1] + pkgdata[key] = item[1].encode('latin-1').decode('unicode_escape') pkge = pkgdata.get('PKGE', '0') pkgv = pkgdata['PKGV']
Signature · 1 line
2.17.1