classes/reproducible_build: Skip SDE for non-finalizing virtual recipes
PATCH 1/1 classes/reproducible_build: Skip SDE for non-finalizing virtual recipes
Don't capture the source date epoch for recipes that are not being finalized. This prevents a virtual recipe (e.g. multilib) from attempting to read the source date epoch file from the non-virtual recipes workdir.
[YOCTO #13851]
--- meta/classes/reproducible_build.bbclass | 14 ++++++++++++++ 1 file changed, 14 insertions(+)
meta/classes/reproducible_build.bbclass +14 -0
@@ -172,6 +172,20 @@ python create_source_date_epoch_stamp() { } def get_source_date_epoch_value(d): + onlyfinalize = d.getVar('__ONLYFINALISE') + virtual = "%s:%s" % (d.getVar("BBEXTENDCURR") or "", d.getVar("BBEXTENDVARIANT") or "") + + if onlyfinalize == "default": + onlyfinalize = ":" + + # Only capture the source date epoch if the current virtual recipe is the + # one being finalized. This prevents looking for the SDE file in another + # recipes ${WORKDIR}, where it may or may not exist + if onlyfinalize: + if onlyfinalize != virtual: + bb.debug(2, "Skipping SOURCE_DATE_EPOCH: virtual '%s' != finalize '%s'" % (virtual, onlyfinalize)) + return 0 + cached = d.getVar('__CACHED_SOURCE_DATE_EPOCH') if cached: return cached
Signature · 1 line
2.17.1
Quoted 11 lines · Joshua Watt
Don't capture the source date epoch for recipes that are not being finalized. This prevents a virtual recipe (e.g. multilib) from attempting to read the source date epoch file from the non-virtual recipes workdir.
[YOCTO #13851]
Signed-off-by Joshua Watt <[email protected]>--- meta/classes/reproducible_build.bbclass | 14 ++++++++++++++ 1 file changed, 14 insertions(+)
Thanks, its a start. Unfortunately it pokes or assumes far too much knowledge of bitbake internals so we'll have to come up with something neater.
I'll give this some further thought.
Cheers,
Richard
Quoted 31 lines
meta/classes/reproducible_build.bbclass +7
-0@@ -172,6 +172,20 @@ python create_source_date_epoch_stamp() { } def get_source_date_epoch_value(d): + onlyfinalize = d.getVar('__ONLYFINALISE') + virtual = "%s:%s" % (d.getVar("BBEXTENDCURR") or "", d.getVar("BBEXTENDVARIANT") or "") + + if onlyfinalize == "default": + onlyfinalize = ":" + + # Only capture the source date epoch if the current virtualrecipe is the
+ # one being finalized. This prevents looking for the SDE file in
anothermeta/classes/reproducible_build.bbclass +4
-0+ # recipes ${WORKDIR}, where it may or may not exist + if onlyfinalize: + if onlyfinalize != virtual: + bb.debug(2, "Skipping SOURCE_DATE_EPOCH: virtual '%s' !=finalize '%s'" % (virtual, onlyfinalize))
meta/classes/reproducible_build.bbclass +2
-0+ return 0 + cached = d.getVar('__CACHED_SOURCE_DATE_EPOCH') if cached: return cached
Quoted 11 lines · Joshua Watt
Don't capture the source date epoch for recipes that are not being finalized. This prevents a virtual recipe (e.g. multilib) from attempting to read the source date epoch file from the non-virtual recipes workdir.
[YOCTO #13851]
Signed-off-by Joshua Watt <[email protected]>--- meta/classes/reproducible_build.bbclass | 14 ++++++++++++++ 1 file changed, 14 insertions(+)
What puzzles me with this is that we're telling bitbake to ignore the variable for hashing purposes, yet it still goes ahead and executes it.
If we decide that is a bug, we can come up with the patch that follows below, which I think should also fix this issue and perhaps some others?
data/siggen: Don't expand ignored variables
--- bitbake/lib/bb/data.py | 4 ++-- bitbake/lib/bb/siggen.py | 2 +- bitbake/lib/bb/tests/data.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-)
bitbake/lib/bb/data.py +2 -2
@@ -365,7 +365,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d): #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs))) #d.setVarFlag(key, "vardeps", deps) -def generate_dependencies(d): +def generate_dependencies(d, whitelist): keys = set(key for key in d if not key.startswith("__")) shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False))
@@ -380,7 +380,7 @@ def generate_dependencies(d): newdeps = deps[task] seen = set() while newdeps: - nextdeps = newdeps + nextdeps = newdeps - whitelist seen |= nextdeps newdeps = set() for dep in nextdeps:
bitbake/lib/bb/siggen.py +1 -1
@@ -146,7 +146,7 @@ class SignatureGeneratorBasic(SignatureGenerator): def _build_data(self, fn, d): ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1') - tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d) + tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, self.basewhitelist) taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basewhitelist, fn)
bitbake/lib/bb/tests/data.py +1 -1
@@ -476,7 +476,7 @@ class Contains(unittest.TestCase): class TaskHash(unittest.TestCase): def test_taskhashes(self): def gettask_bashhash(taskname, d): - tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d) + tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, set()) taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, set(), "somefile") bb.warn(str(lookupcache)) return basehash["somefile:" + taskname]
Quoted 8 lines · Richard Purdie
Quoted 7 lines · Joshua Watt
Don't capture the source date epoch for recipes that are not being finalized. This prevents a virtual recipe (e.g. multilib) from attempting to read the source date epoch file from the non-virtual recipes workdir.
[YOCTO #13851]
Quoted 9 lines
Quoted 4 lines
Signed-off-by Joshua Watt <[email protected]>--- meta/classes/reproducible_build.bbclass | 14 ++++++++++++++ 1 file changed, 14 insertions(+)Thanks, its a start. Unfortunately it pokes or assumes far too much knowledge of bitbake internals so we'll have to come up with something neater.
Ya I was in a hurry, but I was imagining some sort of bb.utils.is_finalizing(d) to wrap the logic
Quoted 39 lines
I'll give this some further thought.
Cheers,
Richard
Quoted 31 lines
meta/classes/reproducible_build.bbclass +7
-0@@ -172,6 +172,20 @@ python create_source_date_epoch_stamp() { } def get_source_date_epoch_value(d): + onlyfinalize = d.getVar('__ONLYFINALISE') + virtual = "%s:%s" % (d.getVar("BBEXTENDCURR") or "", d.getVar("BBEXTENDVARIANT") or "") + + if onlyfinalize == "default": + onlyfinalize = ":" + + # Only capture the source date epoch if the current virtualrecipe is the
+ # one being finalized. This prevents looking for the SDE file in
anothermeta/classes/reproducible_build.bbclass +4
-0+ # recipes ${WORKDIR}, where it may or may not exist + if onlyfinalize: + if onlyfinalize != virtual: + bb.debug(2, "Skipping SOURCE_DATE_EPOCH: virtual '%s' !=finalize '%s'" % (virtual, onlyfinalize))
meta/classes/reproducible_build.bbclass +2
-0+ return 0 + cached = d.getVar('__CACHED_SOURCE_DATE_EPOCH') if cached: return cached
Quoted 13 lines · Richard Purdie
Quoted 11 lines · Joshua Watt
Don't capture the source date epoch for recipes that are not being finalized. This prevents a virtual recipe (e.g. multilib) from attempting to read the source date epoch file from the non-virtual recipes workdir.
[YOCTO #13851]
Signed-off-by Joshua Watt <[email protected]>--- meta/classes/reproducible_build.bbclass | 14 ++++++++++++++ 1 file changed, 14 insertions(+)
Quoted 7 lines
What puzzles me with this is that we're telling bitbake to ignore the variable for hashing purposes, yet it still goes ahead and executes it.
If we decide that is a bug, we can come up with the patch that follows below, which I think should also fix this issue and perhaps some others?
It definitely seems like a good thing to do, question is can we make this change so close to the release? The very rare SDE race is the only thing I know of that is fixed by this, which seems like very little pay off for a lot of risk... We could do both changes; mine now for the release, then let this one soak on master for a while and backport if all is well.
Quoted 70 lines
data/siggen: Don't expand ignored variables
Signed-off-by Richard Purdie <[email protected]>--- bitbake/lib/bb/data.py | 4 ++-- bitbake/lib/bb/siggen.py | 2 +- bitbake/lib/bb/tests/data.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-)bitbake/lib/bb/data.py +0
-0@@ -365,7 +365,7 @@ def build_dependencies(key, keys, shelldeps,varflagsexcl, d):
#bb.note("Variable %s references %s and calls %s" % (key, str(deps),
str(execs)))
#d.setVarFlag(key, "vardeps", deps)bitbake/lib/bb/data.py +1
-1-def generate_dependencies(d): +def generate_dependencies(d, whitelist): keys = set(key for key in d if not key.startswith("__")) shelldeps = set(key for key in d.getVar("__exportlist", False) if
d.getVarFlag(key, "export", False)and notd.getVarFlag(key, "unexport", False))bitbake/lib/bb/data.py +1
-1@@ -380,7 +380,7 @@ def generate_dependencies(d): newdeps = deps[task] seen = set() while newdeps: - nextdeps = newdeps + nextdeps = newdeps - whitelist seen |= nextdeps newdeps = set() for dep in nextdeps:bitbake/lib/bb/siggen.py +1
-1@@ -146,7 +146,7 @@ class SignatureGeneratorBasic(SignatureGenerator): def _build_data(self, fn, d): ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1') - tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d) + tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, self.basewhitelist)taskdeps, basehash = bb.data.generate_dependency_hash(tasklist,
gendeps, lookupcache, self.basewhitelist, fn)bitbake/lib/bb/tests/data.py +1
-1@@ -476,7 +476,7 @@ class Contains(unittest.TestCase): class TaskHash(unittest.TestCase): def test_taskhashes(self): def gettask_bashhash(taskname, d): - tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d) + tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, set()) taskdeps, basehash =
bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, set(), "somefile")
bb.warn(str(lookupcache))
return basehash["somefile:" + taskname]
Quoted 69 lines · Richard Purdie
Quoted 11 lines · Joshua Watt
Don't capture the source date epoch for recipes that are not being finalized. This prevents a virtual recipe (e.g. multilib) from attempting to read the source date epoch file from the non-virtual recipes workdir.
[YOCTO #13851]
Signed-off-by Joshua Watt <[email protected]>--- meta/classes/reproducible_build.bbclass | 14 ++++++++++++++ 1 file changed, 14 insertions(+)What puzzles me with this is that we're telling bitbake to ignore the variable for hashing purposes, yet it still goes ahead and executes it.
If we decide that is a bug, we can come up with the patch that follows below, which I think should also fix this issue and perhaps some others?
data/siggen: Don't expand ignored variables
Signed-off-by Richard Purdie <[email protected]>--- bitbake/lib/bb/data.py | 4 ++-- bitbake/lib/bb/siggen.py | 2 +- bitbake/lib/bb/tests/data.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-)bitbake/lib/bb/data.py +2
-2@@ -365,7 +365,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d): #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs))) #d.setVarFlag(key, "vardeps", deps) -def generate_dependencies(d): +def generate_dependencies(d, whitelist): keys = set(key for key in d if not key.startswith("__")) shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False))@@ -380,7 +380,7 @@ def generate_dependencies(d): newdeps = deps[task] seen = set() while newdeps: - nextdeps = newdeps + nextdeps = newdeps - whitelist seen |= nextdeps newdeps = set() for dep in nextdeps:bitbake/lib/bb/siggen.py +1
-1@@ -146,7 +146,7 @@ class SignatureGeneratorBasic(SignatureGenerator): def _build_data(self, fn, d): ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1') - tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d) + tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, self.basewhitelist) taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basewhitelist, fn)bitbake/lib/bb/tests/data.py +0
-0
bitbake/lib/bb/tests/data.py class Contains(unittest.TestCase): · line 482
Quoted 8 lines
bitbake/lib/bb/tests/data.py +1
-1@@ -476,7 +476,7 @@ class Contains(unittest.TestCase): class TaskHash(unittest.TestCase): def test_taskhashes(self): def gettask_bashhash(taskname, d): - tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d) + tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, set()) taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, set(), "somefile") bb.warn(str(lookupcache)) return basehash["somefile:" + taskname]
LGTM