Skip to contentSkip to conversation

classes/reproducible_build: Skip SDE for non-finalizing virtual recipes

PATCH In review Joshua Watt 2 participants 6 messages Apr 2, 2020 – Apr 3, 2020

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 classes/reproducible_build: Skip SDE for non-finalizing virtual recipes

Don't capture the source date epoch for recipes that are not being finalized. This prevents a virtual recipe (e.g. multilib) from attempting to read the source date epoch file from the non-virtual recipes workdir.

[YOCTO #13851]

---
 meta/classes/reproducible_build.bbclass | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
meta/classes/reproducible_build.bbclass +14 -0
index 8da40f656a..48ec9108a5 100644
@@ -172,6 +172,20 @@ python create_source_date_epoch_stamp() {
 }
 
 def get_source_date_epoch_value(d):
+    onlyfinalize = d.getVar('__ONLYFINALISE')
+    virtual = "%s:%s" % (d.getVar("BBEXTENDCURR") or "", d.getVar("BBEXTENDVARIANT") or "")
+
+    if onlyfinalize == "default":
+        onlyfinalize = ":"
+
+    # Only capture the source date epoch if the current virtual recipe is the
+    # one being finalized. This prevents looking for the SDE file in another
+    # recipes ${WORKDIR}, where it may or may not exist
+    if onlyfinalize:
+        if onlyfinalize != virtual:
+            bb.debug(2, "Skipping SOURCE_DATE_EPOCH: virtual '%s' != finalize '%s'" % (virtual, onlyfinalize))
+            return 0
+
     cached = d.getVar('__CACHED_SOURCE_DATE_EPOCH')
     if cached:
         return cached
Signature · 1 line
2.17.1
#1 lore raw
Quoted 11 lines · Joshua Watt

Don't capture the source date epoch for recipes that are not being finalized. This prevents a virtual recipe (e.g. multilib) from attempting to read the source date epoch file from the non-virtual recipes workdir.

[YOCTO #13851]

---
 meta/classes/reproducible_build.bbclass | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

Thanks, its a start. Unfortunately it pokes or assumes far too much knowledge of bitbake internals so we'll have to come up with something neater.

I'll give this some further thought.

Cheers,

Richard

Quoted 31 lines
meta/classes/reproducible_build.bbclass +7 -0
index 8da40f656a..48ec9108a5 100644
@@ -172,6 +172,20 @@ python create_source_date_epoch_stamp() {
 }
 
 def get_source_date_epoch_value(d):
+    onlyfinalize = d.getVar('__ONLYFINALISE')
+    virtual = "%s:%s" % (d.getVar("BBEXTENDCURR") or "",
 d.getVar("BBEXTENDVARIANT") or "")
+
+    if onlyfinalize == "default":
+        onlyfinalize = ":"
+
+    # Only capture the source date epoch if the current virtual

recipe is the
+ # one being finalized. This prevents looking for the SDE file in
another

meta/classes/reproducible_build.bbclass +4 -0
+    # recipes ${WORKDIR}, where it may or may not exist
+    if onlyfinalize:
+        if onlyfinalize != virtual:
+            bb.debug(2, "Skipping SOURCE_DATE_EPOCH: virtual '%s' !=

finalize '%s'" % (virtual, onlyfinalize))

meta/classes/reproducible_build.bbclass +2 -0
+            return 0
+
     cached = d.getVar('__CACHED_SOURCE_DATE_EPOCH')
     if cached:
         return cached
#3 lore raw
In reply to Joshua Watt
Quoted 11 lines · Joshua Watt

Don't capture the source date epoch for recipes that are not being finalized. This prevents a virtual recipe (e.g. multilib) from attempting to read the source date epoch file from the non-virtual recipes workdir.

[YOCTO #13851]

---
 meta/classes/reproducible_build.bbclass | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

What puzzles me with this is that we're telling bitbake to ignore the variable for hashing purposes, yet it still goes ahead and executes it.

If we decide that is a bug, we can come up with the patch that follows below, which I think should also fix this issue and perhaps some others?

data/siggen: Don't expand ignored variables

---
 bitbake/lib/bb/data.py       | 4 ++--
 bitbake/lib/bb/siggen.py     | 2 +-
 bitbake/lib/bb/tests/data.py | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)
bitbake/lib/bb/data.py +2 -2
index 6dc02172cb..b0683c5180 100644
@@ -365,7 +365,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
     #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs)))
     #d.setVarFlag(key, "vardeps", deps)
 
-def generate_dependencies(d):
+def generate_dependencies(d, whitelist):
 
     keys = set(key for key in d if not key.startswith("__"))
     shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False))
@@ -380,7 +380,7 @@ def generate_dependencies(d):
         newdeps = deps[task]
         seen = set()
         while newdeps:
-            nextdeps = newdeps
+            nextdeps = newdeps - whitelist
             seen |= nextdeps
             newdeps = set()
             for dep in nextdeps:
bitbake/lib/bb/siggen.py +1 -1
index 8bfc45235d..4c8d81c5da 100644
@@ -146,7 +146,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
     def _build_data(self, fn, d):
 
         ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1')
-        tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d)
+        tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, self.basewhitelist)
 
         taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basewhitelist, fn)
 
bitbake/lib/bb/tests/data.py +1 -1
index 2b137706dd..5f195047de 100644
@@ -476,7 +476,7 @@ class Contains(unittest.TestCase):
 class TaskHash(unittest.TestCase):
     def test_taskhashes(self):
         def gettask_bashhash(taskname, d):
-            tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d)
+            tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, set())
             taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, set(), "somefile")
             bb.warn(str(lookupcache))
             return basehash["somefile:" + taskname]
#2 lore raw
In reply to Richard Purdie
Quoted 8 lines · Richard Purdie
Quoted 7 lines · Joshua Watt

Don't capture the source date epoch for recipes that are not being finalized. This prevents a virtual recipe (e.g. multilib) from attempting to read the source date epoch file from the non-virtual recipes workdir.

[YOCTO #13851]

Quoted 9 lines
Quoted 4 lines
---
 meta/classes/reproducible_build.bbclass | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

Thanks, its a start. Unfortunately it pokes or assumes far too much knowledge of bitbake internals so we'll have to come up with something neater.

Ya I was in a hurry, but I was imagining some sort of bb.utils.is_finalizing(d) to wrap the logic

Quoted 39 lines

I'll give this some further thought.

Cheers,

Richard

Quoted 31 lines
meta/classes/reproducible_build.bbclass +7 -0
index 8da40f656a..48ec9108a5 100644
@@ -172,6 +172,20 @@ python create_source_date_epoch_stamp() {
 }
 
 def get_source_date_epoch_value(d):
+    onlyfinalize = d.getVar('__ONLYFINALISE')
+    virtual = "%s:%s" % (d.getVar("BBEXTENDCURR") or "",
 d.getVar("BBEXTENDVARIANT") or "")
+
+    if onlyfinalize == "default":
+        onlyfinalize = ":"
+
+    # Only capture the source date epoch if the current virtual

recipe is the
+ # one being finalized. This prevents looking for the SDE file in
another

meta/classes/reproducible_build.bbclass +4 -0
+    # recipes ${WORKDIR}, where it may or may not exist
+    if onlyfinalize:
+        if onlyfinalize != virtual:
+            bb.debug(2, "Skipping SOURCE_DATE_EPOCH: virtual '%s' !=

finalize '%s'" % (virtual, onlyfinalize))

meta/classes/reproducible_build.bbclass +2 -0
+            return 0
+
     cached = d.getVar('__CACHED_SOURCE_DATE_EPOCH')
     if cached:
         return cached
#4 lore raw
In reply to Richard Purdie
Quoted 13 lines · Richard Purdie
Quoted 11 lines · Joshua Watt

Don't capture the source date epoch for recipes that are not being finalized. This prevents a virtual recipe (e.g. multilib) from attempting to read the source date epoch file from the non-virtual recipes workdir.

[YOCTO #13851]

---
 meta/classes/reproducible_build.bbclass | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
Quoted 7 lines

What puzzles me with this is that we're telling bitbake to ignore the variable for hashing purposes, yet it still goes ahead and executes it.

If we decide that is a bug, we can come up with the patch that follows below, which I think should also fix this issue and perhaps some others?

It definitely seems like a good thing to do, question is can we make this change so close to the release? The very rare SDE race is the only thing I know of that is fixed by this, which seems like very little pay off for a lot of risk... We could do both changes; mine now for the release, then let this one soak on master for a while and backport if all is well.

Quoted 70 lines

data/siggen: Don't expand ignored variables

---
 bitbake/lib/bb/data.py       | 4 ++--
 bitbake/lib/bb/siggen.py     | 2 +-
 bitbake/lib/bb/tests/data.py | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)
bitbake/lib/bb/data.py +0 -0
index 6dc02172cb..b0683c5180 100644
@@ -365,7 +365,7 @@ def build_dependencies(key, keys, shelldeps,

varflagsexcl, d):
#bb.note("Variable %s references %s and calls %s" % (key, str(deps),
str(execs)))
#d.setVarFlag(key, "vardeps", deps)

bitbake/lib/bb/data.py +1 -1
-def generate_dependencies(d):
+def generate_dependencies(d, whitelist):
 
     keys = set(key for key in d if not key.startswith("__"))
     shelldeps = set(key for key in d.getVar("__exportlist", False) if

d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False))

bitbake/lib/bb/data.py +1 -1
@@ -380,7 +380,7 @@ def generate_dependencies(d):
         newdeps = deps[task]
         seen = set()
         while newdeps:
-            nextdeps = newdeps
+            nextdeps = newdeps - whitelist
             seen |= nextdeps
             newdeps = set()
             for dep in nextdeps:
bitbake/lib/bb/siggen.py +1 -1
index 8bfc45235d..4c8d81c5da 100644
@@ -146,7 +146,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
     def _build_data(self, fn, d):
 
         ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') ==
 '1')
-        tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d)
+        tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d,
 self.basewhitelist)
 

taskdeps, basehash = bb.data.generate_dependency_hash(tasklist,
gendeps, lookupcache, self.basewhitelist, fn)

bitbake/lib/bb/tests/data.py +1 -1
index 2b137706dd..5f195047de 100644
@@ -476,7 +476,7 @@ class Contains(unittest.TestCase):
 class TaskHash(unittest.TestCase):
     def test_taskhashes(self):
         def gettask_bashhash(taskname, d):
-            tasklist, gendeps, lookupcache =
 bb.data.generate_dependencies(d)
+            tasklist, gendeps, lookupcache =
 bb.data.generate_dependencies(d, set())
             taskdeps, basehash =

bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, set(), "somefile")
bb.warn(str(lookupcache))
return basehash["somefile:" + taskname]

#5 lore raw
In reply to Richard Purdie
Quoted 69 lines · Richard Purdie
Quoted 11 lines · Joshua Watt

Don't capture the source date epoch for recipes that are not being finalized. This prevents a virtual recipe (e.g. multilib) from attempting to read the source date epoch file from the non-virtual recipes workdir.

[YOCTO #13851]

---
  meta/classes/reproducible_build.bbclass | 14 ++++++++++++++
  1 file changed, 14 insertions(+)

What puzzles me with this is that we're telling bitbake to ignore the variable for hashing purposes, yet it still goes ahead and executes it.

If we decide that is a bug, we can come up with the patch that follows below, which I think should also fix this issue and perhaps some others?

data/siggen: Don't expand ignored variables

---
  bitbake/lib/bb/data.py       | 4 ++--
  bitbake/lib/bb/siggen.py     | 2 +-
  bitbake/lib/bb/tests/data.py | 2 +-
  3 files changed, 4 insertions(+), 4 deletions(-)
bitbake/lib/bb/data.py +2 -2
index 6dc02172cb..b0683c5180 100644
@@ -365,7 +365,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
      #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs)))
      #d.setVarFlag(key, "vardeps", deps)
 
-def generate_dependencies(d):
+def generate_dependencies(d, whitelist):
 
      keys = set(key for key in d if not key.startswith("__"))
      shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False))
@@ -380,7 +380,7 @@ def generate_dependencies(d):
          newdeps = deps[task]
          seen = set()
          while newdeps:
-            nextdeps = newdeps
+            nextdeps = newdeps - whitelist
              seen |= nextdeps
              newdeps = set()
              for dep in nextdeps:
bitbake/lib/bb/siggen.py +1 -1
index 8bfc45235d..4c8d81c5da 100644
@@ -146,7 +146,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
      def _build_data(self, fn, d):
 
          ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1')
-        tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d)
+        tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, self.basewhitelist)
 
          taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basewhitelist, fn)
 
bitbake/lib/bb/tests/data.py +0 -0
index 2b137706dd..5f195047de 100644

bitbake/lib/bb/tests/data.py class Contains(unittest.TestCase): · line 482

Quoted 8 lines
bitbake/lib/bb/tests/data.py +1 -1
@@ -476,7 +476,7 @@ class Contains(unittest.TestCase):
  class TaskHash(unittest.TestCase):
      def test_taskhashes(self):
          def gettask_bashhash(taskname, d):
-            tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d)
+            tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, set())
              taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, set(), "somefile")
              bb.warn(str(lookupcache))
              return basehash["somefile:" + taskname]

LGTM