extensible sdk changes
cover extensible sdk changes
This set of changes is to make sure the locked signatures are enforced as part of the extensible sdk usage.
Therefore it also allows for the unlocking of recipes that are part of the devtool workspace.
The following changes since commit 14e2b90893f57570c2002317b5afb0e61d15e9ca:
toasterconf.json: Add DL_DIR and SSTATE_DIR to poky toasterconf (2016-04-07 14:58:10 +0100)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib sigs
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=sigs
Randy Witt (5):
sstatesig.py: Split single locked sigs check into multiple checks sstatesig.py: Improve the SIGGEN_LOCKEDSIGS_TASKSIG_CHECK message populate_sdk_ext.bbclass: Enable locked sigs errors sstatesig.py: Add a method to "unlock" recipes
devtool: Create unlocked-sigs.inc containing items in the workspace
meta/classes/populate_sdk_ext.bbclass | 16 +++++++-- meta/classes/sstate.bbclass | 8 ++++- meta/lib/oe/sstatesig.py | 63 ++++++++++++++++++++++++++++------- meta/lib/oeqa/selftest/signing.py | 4 +-- scripts/devtool | 20 +++++++++++ 5 files changed, 94 insertions(+), 17 deletions(-)
Signature · 1 line
2.5.5
PATCH 1/5 sstatesig.py: Split single locked sigs check into multiple checks
Add the SIGGEN_LOCKEDSIGS_TASKSIG_CHECK and SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK variables to replace SIGGEN_LOCKEDSIGS_CHECK_LEVEL.
SIGGEN_LOCKEDSIGS_TASKSIG_CHECK will no control whether there is a warning or error if a task's hash in the locked signature file doesn't match the computed hash from the current metadata.
SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK will control whther there is a warning or error if a task that supports sstate is in the locked signature file, but no sstate exists for the task.
Previously you could only have warning/errors for both controlled by SIGGEN_LOCKEDSIGS_CHECK_LEVEL. This was an issue in the extensible sdk, because we know sstate won't exist for certain items in the reverse dependencies list for tasks. However, we still want to error if task signatures don't match.
[YOCTO #9195]
--- meta/classes/sstate.bbclass | 8 +++++++- meta/lib/oe/sstatesig.py | 27 +++++++++++++++++++++------ meta/lib/oeqa/selftest/signing.py | 4 ++-- 3 files changed, 30 insertions(+), 9 deletions(-)
meta/classes/sstate.bbclass +7 -1
@@ -53,7 +53,13 @@ SSTATEPOSTINSTFUNCS = "" EXTRA_STAGING_FIXMES ?= "" SSTATECLEANFUNCS = "" -SIGGEN_LOCKEDSIGS_CHECK_LEVEL ?= 'error' +# Check whether sstate exists for tasks that support sstate and are in the +# locked signatures file. +SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK ?= 'error' + +# Check whether the task's computed hash matches the task's hash in the +# locked signatures file. +SIGGEN_LOCKEDSIGS_TASKSIG_CHECK ?= "error" # The GnuPG key ID and passphrase to use to sign sstate archives (or unset to # not sign)
meta/lib/oe/sstatesig.py +21 -6
@@ -189,20 +189,35 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): f.write('SIGGEN_LOCKEDSIGS_TYPES_%s = "%s"' % (self.machine, " ".join(types.keys()))) def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d): - checklevel = d.getVar("SIGGEN_LOCKEDSIGS_CHECK_LEVEL", True) + warn_msgs = [] + error_msgs = [] + sstate_missing_msgs = [] + for task in range(len(sq_fn)): if task not in ret: for pn in self.lockedsigs: if sq_hash[task] in self.lockedsigs[pn].itervalues(): if sq_task[task] == 'do_shared_workdir': continue - self.mismatch_msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?" + sstate_missing_msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?" % (pn, sq_task[task], sq_hash[task])) - if self.mismatch_msgs and checklevel == 'warn': - bb.warn("\n".join(self.mismatch_msgs)) - elif self.mismatch_msgs and checklevel == 'error': - bb.fatal("\n".join(self.mismatch_msgs)) + checklevel = d.getVar("SIGGEN_LOCKEDSIGS_TASKSIG_CHECK", True) + if checklevel == 'warn': + warn_msgs += self.mismatch_msgs + elif checklevel == 'error': + error_msgs += self.mismatch_msgs + + checklevel = d.getVar("SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK", True) + if checklevel == 'warn': + warn_msgs += sstate_missing_msgs + elif checklevel == 'error': + error_msgs += sstate_missing_msgs + + if warn_msgs: + bb.warn("\n".join(warn_msgs)) + if error_msgs: + bb.fatal("\n".join(error_msgs)) # Insert these classes into siggen's namespace so it can see and select them
meta/lib/oeqa/selftest/signing.py +2 -2
@@ -160,7 +160,7 @@ class LockedSignatures(oeSelfTest): bitbake('-S none %s' % test_recipe) feature = 'require %s\n' % locked_sigs_file - feature += 'SIGGEN_LOCKEDSIGS_CHECK_LEVEL = "warn"\n' + feature += 'SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "warn"\n' self.write_config(feature) # Build a locked recipe
@@ -180,7 +180,7 @@ class LockedSignatures(oeSelfTest): ret = bitbake(test_recipe) # Verify you get the warning and that the real task *isn't* run (i.e. the locked signature has worked) - patt = r'WARNING: The %s:do_package sig \S+ changed, use locked sig \S+ to instead' % test_recipe + patt = r'WARNING: The %s:do_package sig is computed to be \S+, but the sig is locked to \S+ in SIGGEN_LOCKEDSIGS\S+' % test_recipe found_warn = re.search(patt, ret.output) self.assertIsNotNone(found_warn, "Didn't find the expected warning message. Output: %s" % ret.output)
Signature · 1 line
2.5.5
PATCH 2/5 sstatesig.py: Improve the SIGGEN_LOCKEDSIGS_TASKSIG_CHECK message
In reply to Randy WittThe previous message when signatures didn't match between the metadata and the locked signatures file, the message output was a bit confusing.
Now the message should be of the form:
The zlib-native:do_install sig is computed to be 53531910a2a7848432da89def942a91a, but the sig is locked to d25ba9035f7ccb308e51bbe1066e8d27 in SIGGEN_LOCKEDSIGS_t-x86-64
which will hopefully be more useful in understanding the problem.
[YOCTO #9195]
--- meta/lib/oe/sstatesig.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-)
meta/lib/oe/sstatesig.py +7 -5
@@ -65,12 +65,13 @@ def sstate_lockedsigs(d): sigs = {} types = (d.getVar("SIGGEN_LOCKEDSIGS_TYPES", True) or "").split() for t in types: - lockedsigs = (d.getVar("SIGGEN_LOCKEDSIGS_%s" % t, True) or "").split() + siggen_lockedsigs_var = "SIGGEN_LOCKEDSIGS_%s" % t + lockedsigs = (d.getVar(siggen_lockedsigs_var, True) or "").split() for ls in lockedsigs: pn, task, h = ls.split(":", 2) if pn not in sigs: sigs[pn] = {} - sigs[pn][task] = h + sigs[pn][task] = [h, siggen_lockedsigs_var] return sigs class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic):
@@ -138,14 +139,15 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): if recipename in self.lockedsigs: if task in self.lockedsigs[recipename]: k = fn + "." + task - h_locked = self.lockedsigs[recipename][task] + h_locked = self.lockedsigs[recipename][task][0] + var = self.lockedsigs[recipename][task][1] self.lockedhashes[k] = h_locked self.taskhash[k] = h_locked #bb.warn("Using %s %s %s" % (recipename, task, h)) if h != h_locked: - self.mismatch_msgs.append('The %s:%s sig (%s) changed, use locked sig %s to instead' - % (recipename, task, h, h_locked)) + self.mismatch_msgs.append('The %s:%s sig is computed to be %s, but the sig is locked to %s in %s' + % (recipename, task, h, h_locked, var)) return h_locked #bb.warn("%s %s %s" % (recipename, task, h))
Signature · 1 line
2.5.5
PATCH 3/5 populate_sdk_ext.bbclass: Enable locked sigs errors
In reply to Randy WittWith the extensible sdk we want there to be an error if a task tries to run without signatures that match locked-sigs.inc. This patch enables that error.
[YOCTO #9195]
--- meta/classes/populate_sdk_ext.bbclass | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-)
meta/classes/populate_sdk_ext.bbclass +9 -2
@@ -209,8 +209,15 @@ python copy_buildsystem () { # Bypass the default connectivity check if any f.write('CONNECTIVITY_CHECK_URIS = ""\n\n') - # Ensure locked sstate cache objects are re-used without error - f.write('SIGGEN_LOCKEDSIGS_CHECK_LEVEL = "none"\n\n') + # This warning will come out if reverse dependencies for a task + # don't have sstate as well as the task itself. We already know + # this will be the case for the extensible sdk, so turn off the + # warning. + f.write('SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK = "none"\n\n') + + # Error if the sigs in the locked-signature file don't match + # the sig computed from the metadata. + f.write('SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "error"\n\n') # Hide the config information from bitbake output (since it's fixed within the SDK) f.write('BUILDCFG_HEADER = ""\n')
Signature · 1 line
2.5.5
PATCH 4/5 sstatesig.py: Add a method to "unlock" recipes
In reply to Randy WittIn order to support workflows using devtool where a user might want to modify tasks that exist in locked-sigs.inc, there must be a way to unlock recipes.
This patch adds that support by allowing the user to add recipes to SIGGEN_UNLOCKED_RECIPES. Recipes that exist in that variable will have all their tasks unlocked, as well as any tasks that depend on that recipe.
For example if foo->bar->baz, if you unlock baz, it will also unlock bar so that foo can be rebuilt without explicitly specifying bar as being unlocked.
[YOCTO #9195]
--- meta/lib/oe/sstatesig.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-)
meta/lib/oe/sstatesig.py +23 -1
@@ -94,6 +94,9 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): self.lockedhashfn = {} self.machine = data.getVar("MACHINE", True) self.mismatch_msgs = [] + self.unlockedrecipes = (data.getVar("SIGGEN_UNLOCKED_RECIPES", True) or + "").split() + self.unlockedrecipes = { k: "" for k in self.unlockedrecipes } pass def tasks_resolved(self, virtmap, virtpnmap, dataCache):
@@ -136,7 +139,26 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): recipename = dataCache.pkg_fn[fn] self.lockedpnmap[fn] = recipename self.lockedhashfn[fn] = dataCache.hashfn[fn] - if recipename in self.lockedsigs: + + unlocked = False + if recipename in self.unlockedrecipes: + unlocked = True + else: + def recipename_from_dep(dep): + # The dep entry will look something like + # /path/path/recipename.bb.task, virtual:native:/p/foo.bb.task, + # ... + fn = dep.rsplit('.', 1)[0] + return dataCache.pkg_fn[fn] + + # If any unlocked recipe is in the direct dependencies then the + # current recipe should be unlocked as well. + depnames = [ recipename_from_dep(x) for x in deps ] + if any(x in y for y in depnames for x in self.unlockedrecipes): + self.unlockedrecipes[recipename] = '' + unlocked = True + + if not unlocked and recipename in self.lockedsigs: if task in self.lockedsigs[recipename]: k = fn + "." + task h_locked = self.lockedsigs[recipename][task][0]
Signature · 1 line
2.5.5
PATCH 5/5 devtool: Create unlocked-sigs.inc containing items in the workspace
In reply to Randy WittWhen a recipe is added to the workspace, the signatures for the tasks will change. This means that bitbake must be told to allow the signatures to be different if they are in locked-sigs.inc.
This is done by creating an unlocked-sigs.inc file which contains all the recipes in the workspace each time devtool reads the workspace.
So not only will necessary things get added, previously added items will be removed by virtue of them no longer being in the workspace.
This also makes sure that the extensible sdk picks up unlocked-sigs.inc as part of the configuration.
[YOCTO #9195]
--- meta/classes/populate_sdk_ext.bbclass | 5 +++++ scripts/devtool | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+)
meta/classes/populate_sdk_ext.bbclass +5 -0
@@ -140,6 +140,10 @@ python copy_buildsystem () { with open(os.path.join(baseoutpath, 'conf', 'devtool.conf'), 'w') as f: config.write(f) + unlockedsigs = os.path.join(baseoutpath, 'conf', 'unlocked-sigs.inc') + with open(unlockedsigs, 'w') as f: + pass + # Create a layer for new recipes / appends bbpath = d.getVar('BBPATH', True) bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath, 'workspace')])
@@ -238,6 +242,7 @@ python copy_buildsystem () { f.write(line.strip() + '\n') f.write('require conf/locked-sigs.inc\n') + f.write('require conf/unlocked-sigs.inc\n') if os.path.exists(builddir + '/conf/auto.conf'): if derivative:
scripts/devtool +20 -0
@@ -125,6 +125,25 @@ def read_workspace(): 'recipefile': recipefile} logger.debug('Found recipe %s' % workspace[pn]) +def create_unlockedsigs(): + """ This function will make unlocked-sigs.inc match the recipes in the + workspace. This runs on every run of devtool, but it lets us ensure + the unlocked items are in sync with the workspace. """ + + confdir = os.path.join(basepath, 'conf') + unlockedsigs = os.path.join(confdir, 'unlocked-sigs.inc') + bb.utils.mkdirhier(confdir) + with open(os.path.join(confdir, 'unlocked-sigs.inc'), 'w') as f: + f.write("# DO NOT MODIFY! YOUR CHANGES WILL BE LOST.\n" + + "# This layer was created by the OpenEmbedded devtool" + + " utility in order to\n" + + "# contain recipes that are unlocked.\n") + + f.write('SIGGEN_UNLOCKED_RECIPES += "\\\n') + for pn in workspace: + f.write(' ' + pn) + f.write('"') + def create_workspace(args, config, basepath, workspace): if args.layerpath: workspacedir = os.path.abspath(args.layerpath)
@@ -299,6 +318,7 @@ def main(): if not getattr(args, 'no_workspace', False): read_workspace() + create_unlockedsigs() try: ret = args.func(args, config, basepath, workspace)
Signature · 1 line
2.5.5