classes/package: Add extended packaged data
PATCH 1/1 classes/package: Add extended packaged data
Adds extended package data which is encoded as JSON which allows it to encode more structure than the "flat" package data files. The extended data might be much larger than the standard package data, so it is not read by default and instead requires oe.packagedata.read_subpkgdata_extended() to be called
Currently, the file sizes and ELF debug sources are saved off into the extended package data
--- meta/classes/package.bbclass | 38 ++++++++++++++++++++++++++++++++++-- meta/lib/oe/packagedata.py | 9 +++++++++ 2 files changed, 45 insertions(+), 2 deletions(-)
meta/classes/package.bbclass +36 -2
@@ -1225,6 +1225,14 @@ python split_and_strip_files () { # Modified the file so clear the cache cpath.updatecache(file) + def strip_pkgd_prefix(f): + nonlocal dvar + + if f.startswith(dvar): + return f[len(dvar):] + + return f + # # First lets process debug splitting #
@@ -1238,6 +1246,8 @@ python split_and_strip_files () { for file in staticlibs: results.append( (file,source_info(file, d)) ) + d.setVar("PKGDEBUGSOURCES", {strip_pkgd_prefix(f): sorted(s) for f, s in results}) + sources = set() for r in results: sources.update(r[1])
@@ -1622,6 +1632,8 @@ fi with open(data_file, 'w') as fd: fd.write("PACKAGES: %s\n" % packages) + pkgdebugsource = d.getVar("PKGDEBUGSOURCES") or [] + pn = d.getVar('PN') global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split() variants = (d.getVar('MULTILIB_VARIANTS') or "").split()
@@ -1641,17 +1653,32 @@ fi pkgval = pkg d.setVar('PKG_%s' % pkg, pkg) + extended_data = { + "files_info": {} + } + pkgdestpkg = os.path.join(pkgdest, pkg) files = {} + files_extra = {} total_size = 0 seen = set() for f in pkgfiles[pkg]: - relpth = os.path.relpath(f, pkgdestpkg) + fpath = os.sep + os.path.relpath(f, pkgdestpkg) + fstat = os.lstat(f) - files[os.sep + relpth] = fstat.st_size + files[fpath] = fstat.st_size + + extended_data["files_info"].setdefault(fpath, {}) + extended_data["files_info"][fpath]['size'] = fstat.st_size + if fstat.st_ino not in seen: seen.add(fstat.st_ino) total_size += fstat.st_size + + if fpath in pkgdebugsource: + extended_data["files_info"][fpath]['debugsrc'] = pkgdebugsource[fpath] + del pkgdebugsource[fpath] + d.setVar('FILES_INFO', json.dumps(files, sort_keys=True)) process_postinst_on_target(pkg, d.getVar("MLPREFIX"))
@@ -1672,6 +1699,10 @@ fi sf.write('%s_%s: %d\n' % ('PKGSIZE', pkg, total_size)) + subdata_extended_file = pkgdatadir + "/runtime/%s.json" % pkg + with open(subdata_extended_file, "w") as f: + json.dump(extended_data, f, sort_keys=True, separators=(",", ":")) + # Symlinks needed for rprovides lookup rprov = d.getVar('RPROVIDES_%s' % pkg) or d.getVar('RPROVIDES') if rprov:
@@ -1694,6 +1725,9 @@ fi packagedfile = pkgdatadir + '/runtime/%s.packaged' % pkg open(packagedfile, 'w').close() + if pkgdebugsource: + bb.fatal("One or more file debug sources were not consumed by a package: %s" % " ".join(pkgdebugsource.keys())) + if bb.data.inherits_class('kernel', d) or bb.data.inherits_class('module-base', d): write_extra_runtime_pkgs(variants, packages, pkgdatadir)
meta/lib/oe/packagedata.py +9 -0
@@ -57,6 +57,15 @@ def read_subpkgdata_dict(pkg, d): ret[newvar] = subd[var] return ret +def read_subpkgdata_extended(pkg, d): + import json + fn = d.expand("${PKGDATA_DIR}/runtime/%s.json" % pkg) + try: + with open(fn, "r") as f: + return json.load(f) + except FileNotFoundError: + return None + def _pkgmap(d): """Return a dictionary mapping package to recipe name."""
Signature · 1 line
2.32.0
Hello Josh,
Quoted 9 lines · Joshua Watt
Adds extended package data which is encoded as JSON which allows it to encode more structure than the "flat" package data files. The extended data might be much larger than the standard package data, so it is not read by default and instead requires
oe.packagedata.read_subpkgdata_extended()to be calledCurrently, the file sizes and ELF debug sources are saved off into the extended package data
This break non-gpl3 builds:
https://autobuilder.yoctoproject.org/typhoon/#/builders/75/builds/3685
ERROR: libgcrypt-1.9.3-r0 do_package: One or more file debug sources were not consumed by a package: /usr/bin/dumpsexp
ERROR: libidn2-2.3.1-r0 do_package: One or more file debug sources were not consumed by a package: /usr/bin/idn2
Quoted 126 lines
Signed-off-by Joshua Watt <[email protected]>--- meta/classes/package.bbclass | 38 ++++++++++++++++++++++++++++++++++-- meta/lib/oe/packagedata.py | 9 +++++++++ 2 files changed, 45 insertions(+), 2 deletions(-)meta/classes/package.bbclass +36
-2@@ -1225,6 +1225,14 @@ python split_and_strip_files () { # Modified the file so clear the cache cpath.updatecache(file) + def strip_pkgd_prefix(f): + nonlocal dvar + + if f.startswith(dvar): + return f[len(dvar):] + + return f + # # First lets process debug splitting #@@ -1238,6 +1246,8 @@ python split_and_strip_files () { for file in staticlibs: results.append( (file,source_info(file, d)) ) + d.setVar("PKGDEBUGSOURCES", {strip_pkgd_prefix(f): sorted(s) for f, s in results}) + sources = set() for r in results: sources.update(r[1])@@ -1622,6 +1632,8 @@ fi with open(data_file, 'w') as fd: fd.write("PACKAGES: %s\n" % packages) + pkgdebugsource = d.getVar("PKGDEBUGSOURCES") or [] + pn = d.getVar('PN') global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split() variants = (d.getVar('MULTILIB_VARIANTS') or "").split()@@ -1641,17 +1653,32 @@ fi pkgval = pkg d.setVar('PKG_%s' % pkg, pkg) + extended_data = { + "files_info": {} + } + pkgdestpkg = os.path.join(pkgdest, pkg) files = {} + files_extra = {} total_size = 0 seen = set() for f in pkgfiles[pkg]: - relpth = os.path.relpath(f, pkgdestpkg) + fpath = os.sep + os.path.relpath(f, pkgdestpkg) + fstat = os.lstat(f) - files[os.sep + relpth] = fstat.st_size + files[fpath] = fstat.st_size + + extended_data["files_info"].setdefault(fpath, {}) + extended_data["files_info"][fpath]['size'] = fstat.st_size + if fstat.st_ino not in seen: seen.add(fstat.st_ino) total_size += fstat.st_size + + if fpath in pkgdebugsource: + extended_data["files_info"][fpath]['debugsrc'] = pkgdebugsource[fpath] + del pkgdebugsource[fpath] + d.setVar('FILES_INFO', json.dumps(files, sort_keys=True)) process_postinst_on_target(pkg, d.getVar("MLPREFIX"))@@ -1672,6 +1699,10 @@ fi sf.write('%s_%s: %d\n' % ('PKGSIZE', pkg, total_size)) + subdata_extended_file = pkgdatadir + "/runtime/%s.json" % pkg + with open(subdata_extended_file, "w") as f: + json.dump(extended_data, f, sort_keys=True, separators=(",", ":")) + # Symlinks needed for rprovides lookup rprov = d.getVar('RPROVIDES_%s' % pkg) or d.getVar('RPROVIDES') if rprov:@@ -1694,6 +1725,9 @@ fi packagedfile = pkgdatadir + '/runtime/%s.packaged' % pkg open(packagedfile, 'w').close() + if pkgdebugsource: + bb.fatal("One or more file debug sources were not consumed by a package: %s" % " ".join(pkgdebugsource.keys())) + if bb.data.inherits_class('kernel', d) or bb.data.inherits_class('module-base', d): write_extra_runtime_pkgs(variants, packages, pkgdatadir)meta/lib/oe/packagedata.py +9
-0@@ -57,6 +57,15 @@ def read_subpkgdata_dict(pkg, d): ret[newvar] = subd[var] return ret +def read_subpkgdata_extended(pkg, d): + import json + fn = d.expand("${PKGDATA_DIR}/runtime/%s.json" % pkg) + try: + with open(fn, "r") as f: + return json.load(f) + except FileNotFoundError: + return None + def _pkgmap(d): """Return a dictionary mapping package to recipe name."""Signature · 1 line
2.32.0
Signature · 3 lines
Alexandre Belloni, co-owner and COO, Bootlin Embedded Linux and Kernel engineering https://bootlin.com
PATCH v2 1/1 classes/package: Add extended packaged data
In reply to Joshua WattAdds extended package data which is encoded as JSON which allows it to encode more structure than the "flat" package data files. The extended data might be much larger than the standard package data, so it is not read by default and instead requires oe.packagedata.read_subpkgdata_extended() to be called
Currently, the file sizes and ELF debug sources are saved off into the extended package data
---
V2: Remove fatal message when debug data was not captured, since this
can happen on purpose if the package is removed for license reasons
meta/classes/package.bbclass | 35 +++++++++++++++++++++++++++++++++-- meta/lib/oe/packagedata.py | 9 +++++++++ 2 files changed, 42 insertions(+), 2 deletions(-)
meta/classes/package.bbclass +33 -2
@@ -1225,6 +1225,14 @@ python split_and_strip_files () { # Modified the file so clear the cache cpath.updatecache(file) + def strip_pkgd_prefix(f): + nonlocal dvar + + if f.startswith(dvar): + return f[len(dvar):] + + return f + # # First lets process debug splitting #
@@ -1238,6 +1246,8 @@ python split_and_strip_files () { for file in staticlibs: results.append( (file,source_info(file, d)) ) + d.setVar("PKGDEBUGSOURCES", {strip_pkgd_prefix(f): sorted(s) for f, s in results}) + sources = set() for r in results: sources.update(r[1])
@@ -1622,6 +1632,8 @@ fi with open(data_file, 'w') as fd: fd.write("PACKAGES: %s\n" % packages) + pkgdebugsource = d.getVar("PKGDEBUGSOURCES") or [] + pn = d.getVar('PN') global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split() variants = (d.getVar('MULTILIB_VARIANTS') or "").split()
@@ -1641,17 +1653,32 @@ fi pkgval = pkg d.setVar('PKG:%s' % pkg, pkg) + extended_data = { + "files_info": {} + } + pkgdestpkg = os.path.join(pkgdest, pkg) files = {} + files_extra = {} total_size = 0 seen = set() for f in pkgfiles[pkg]: - relpth = os.path.relpath(f, pkgdestpkg) + fpath = os.sep + os.path.relpath(f, pkgdestpkg) + fstat = os.lstat(f) - files[os.sep + relpth] = fstat.st_size + files[fpath] = fstat.st_size + + extended_data["files_info"].setdefault(fpath, {}) + extended_data["files_info"][fpath]['size'] = fstat.st_size + if fstat.st_ino not in seen: seen.add(fstat.st_ino) total_size += fstat.st_size + + if fpath in pkgdebugsource: + extended_data["files_info"][fpath]['debugsrc'] = pkgdebugsource[fpath] + del pkgdebugsource[fpath] + d.setVar('FILES_INFO', json.dumps(files, sort_keys=True)) process_postinst_on_target(pkg, d.getVar("MLPREFIX"))
@@ -1672,6 +1699,10 @@ fi sf.write('%s_%s: %d\n' % ('PKGSIZE', pkg, total_size)) + subdata_extended_file = pkgdatadir + "/runtime/%s.json" % pkg + with open(subdata_extended_file, "w") as f: + json.dump(extended_data, f, sort_keys=True, separators=(",", ":")) + # Symlinks needed for rprovides lookup rprov = d.getVar('RPROVIDES:%s' % pkg) or d.getVar('RPROVIDES') if rprov:
meta/lib/oe/packagedata.py +9 -0
@@ -57,6 +57,15 @@ def read_subpkgdata_dict(pkg, d): ret[newvar] = subd[var] return ret +def read_subpkgdata_extended(pkg, d): + import json + fn = d.expand("${PKGDATA_DIR}/runtime/%s.json" % pkg) + try: + with open(fn, "r") as f: + return json.load(f) + except FileNotFoundError: + return None + def _pkgmap(d): """Return a dictionary mapping package to recipe name."""
Signature · 1 line
2.32.0