Skip to contentSkip to conversation

Include mcdepends in signature generation

RFC PATCH Superseded Joshua Watt 2 participants 9 messages Jun 5, 2020 – Jun 10, 2020

lore
Share & tools
lore
#0 lore raw

cover Include mcdepends in signature generation

Updates signature generation so that mcdepends are included. Previously, they were not which meant that if recipe A mcdepends on recipe B, and recipe B changed, it would not automatically cause recipe A to also rebuild.

In order for signature generation classes to correctly handle mcdepends, they need access to all the data caches, so all are passed instead of just the one for the recipe in question.

This breaks the bitbake siggen API and makes it incompatible with previous signature generators, hence the cross-posting between bitbake and oe-core.

Joshua Watt (3):
bitbake: siggen: Pass all data caches to hash functions
bitbake: tests: Add mcdepends test
sstatesig: Account for all dataCaches being passed

 bitbake/lib/bb/runqueue.py                    |  6 +-
 bitbake/lib/bb/siggen.py                      | 32 ++++----
 .../lib/bb/tests/runqueue-tests/recipes/f1.bb |  1 +
 bitbake/lib/bb/tests/runqueue.py              | 30 ++++++++
 meta/lib/oe/sstatesig.py                      | 77 ++++++++++---------
 5 files changed, 89 insertions(+), 57 deletions(-)
 create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/f1.bb
Signature · 1 line
2.26.2
#1 lore raw

RFC PATCH 1/3 bitbake: siggen: Pass all data caches to hash functions

Passing all the data caches to the task hashing functions allows them to correctly account for mcdepends in task signatures. This allows tasks to be correctly re-run when a mcdepends changes.

[YOCTO #13724]

---
 bitbake/lib/bb/runqueue.py |  6 +++---
 bitbake/lib/bb/siggen.py   | 32 +++++++++++++++-----------------
 2 files changed, 18 insertions(+), 20 deletions(-)
bitbake/lib/bb/runqueue.py +3 -3
index 5b7dab8d79..372df63593 100644
@@ -1190,8 +1190,8 @@ class RunQueueData:
         return len(self.runtaskentries)
 
     def prepare_task_hash(self, tid):
-        bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches[mc_from_tid(tid)])
-        self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches[mc_from_tid(tid)])
+        bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches)
+        self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches)
         self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid)
 
     def dump_data(self):
@@ -2305,7 +2305,7 @@ class RunQueueExecute:
                 if len(self.rqdata.runtaskentries[p].depends) and not self.rqdata.runtaskentries[tid].depends.isdisjoint(total):
                     continue
                 orighash = self.rqdata.runtaskentries[tid].hash
-                newhash = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, self.rqdata.dataCaches[mc_from_tid(tid)])
+                newhash = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, self.rqdata.dataCaches)
                 origuni = self.rqdata.runtaskentries[tid].unihash
                 newuni = bb.parse.siggen.get_unihash(tid)
                 # FIXME, need to check it can come from sstate at all for determinism?
bitbake/lib/bb/siggen.py +15 -17
index 4c8d81c5da..cffd0fa76d 100644
@@ -58,10 +58,10 @@ class SignatureGenerator(object):
     def get_unihash(self, tid):
         return self.taskhash[tid]
 
-    def prep_taskhash(self, tid, deps, dataCache):
+    def prep_taskhash(self, tid, deps, dataCaches):
         return
 
-    def get_taskhash(self, tid, deps, dataCache):
+    def get_taskhash(self, tid, deps, dataCaches):
         self.taskhash[tid] = hashlib.sha256(tid.encode("utf-8")).hexdigest()
         return self.taskhash[tid]
 
@@ -200,7 +200,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
         self.lookupcache = {}
         self.taskdeps = {}
 
-    def rundep_check(self, fn, recipename, task, dep, depname, dataCache):
+    def rundep_check(self, fn, recipename, task, dep, depname, dataCaches):
         # Return True if we should keep the dependency, False to drop it
         # We only manipulate the dependencies for packages not in the whitelist
         if self.twl and not self.twl.search(recipename):
@@ -218,37 +218,35 @@ class SignatureGeneratorBasic(SignatureGenerator):
             pass
         return taint
 
-    def prep_taskhash(self, tid, deps, dataCache):
+    def prep_taskhash(self, tid, deps, dataCaches):
 
         (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid)
 
-        self.basehash[tid] = dataCache.basetaskhash[tid]
+        self.basehash[tid] = dataCaches[mc].basetaskhash[tid]
         self.runtaskdeps[tid] = []
         self.file_checksum_values[tid] = []
-        recipename = dataCache.pkg_fn[fn]
+        recipename = dataCaches[mc].pkg_fn[fn]
 
         self.tidtopn[tid] = recipename
 
         for dep in sorted(deps, key=clean_basepath):
-            (depmc, _, deptaskname, depfn) = bb.runqueue.split_tid_mcfn(dep)
-            if mc != depmc:
-                continue
-            depname = dataCache.pkg_fn[depfn]
-            if not self.rundep_check(fn, recipename, task, dep, depname, dataCache):
+            (depmc, _, _, depmcfn) = bb.runqueue.split_tid_mcfn(dep)
+            depname = dataCaches[depmc].pkg_fn[depmcfn]
+            if not self.rundep_check(fn, recipename, task, dep, depname, dataCaches):
                 continue
             if dep not in self.taskhash:
                 bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?" % dep)
             self.runtaskdeps[tid].append(dep)
 
-        if task in dataCache.file_checksums[fn]:
+        if task in dataCaches[mc].file_checksums[fn]:
             if self.checksum_cache:
-                checksums = self.checksum_cache.get_checksums(dataCache.file_checksums[fn][task], recipename, self.localdirsexclude)
+                checksums = self.checksum_cache.get_checksums(dataCaches[mc].file_checksums[fn][task], recipename, self.localdirsexclude)
             else:
-                checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename, self.localdirsexclude)
+                checksums = bb.fetch2.get_file_checksums(dataCaches[mc].file_checksums[fn][task], recipename, self.localdirsexclude)
             for (f,cs) in checksums:
                 self.file_checksum_values[tid].append((f,cs))
 
-        taskdep = dataCache.task_deps[fn]
+        taskdep = dataCaches[mc].task_deps[fn]
         if 'nostamp' in taskdep and task in taskdep['nostamp']:
             # Nostamp tasks need an implicit taint so that they force any dependent tasks to run
             if tid in self.taints and self.taints[tid].startswith("nostamp:"):
@@ -259,14 +257,14 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 taint = str(uuid.uuid4())
                 self.taints[tid] = "nostamp:" + taint
 
-        taint = self.read_taint(fn, task, dataCache.stamp[fn])
+        taint = self.read_taint(fn, task, dataCaches[mc].stamp[fn])
         if taint:
             self.taints[tid] = taint
             logger.warning("%s is tainted from a forced run" % tid)
 
         return
 
-    def get_taskhash(self, tid, deps, dataCache):
+    def get_taskhash(self, tid, deps, dataCaches):
 
         data = self.basehash[tid]
         for dep in self.runtaskdeps[tid]:
Signature · 1 line
2.26.2
#2 lore raw

RFC PATCH 2/3 bitbake: tests: Add mcdepends test

In reply to Joshua Watt

Adds a test to validate that mcdepends causes the dependent tasks to build, and also that a change in the dependent task causes the dependee task to re-execute.

---
 .../lib/bb/tests/runqueue-tests/recipes/f1.bb |  1 +
 bitbake/lib/bb/tests/runqueue.py              | 30 +++++++++++++++++++
 2 files changed, 31 insertions(+)
 create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/f1.bb
bitbake/lib/bb/tests/runqueue-tests/recipes/f1.bb new file +1 -0
new file mode 100644
index 0000000000..d45a4cff52
@@ -0,0 +1 @@
+do_install[mcdepends] = "mc:mc1:mc2:a1:do_build"
bitbake/lib/bb/tests/runqueue.py +30 -0
index 091b5e41e0..988272b19f 100644
@@ -248,6 +248,36 @@ class RunQueueTests(unittest.TestCase):
             cmd = ["bitbake", "mc:mc1:fails-mc2", "mc:mc2:fails-mc1"]
             self.run_bitbakecmd(cmd, tempdir, "", extraenv=extraenv)
 
+    def test_multiconfig_mcdepends(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            extraenv = {
+                "BBMULTICONFIG" : "mc1 mc2",
+                "BB_SIGNATURE_HANDLER" : "basichash",
+                "EXTRA_BBFILES": "${COREBASE}/recipes/fails-mc/*.bb",
+            }
+            tasks = self.run_bitbakecmd(["bitbake", "mc:mc1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
+            expected = ["mc1:f1:%s" % t for t in self.alltasks] + \
+                       ["mc2:a1:%s" % t for t in self.alltasks]
+            self.assertEqual(set(tasks), set(expected))
+
+            # A rebuild does nothing
+            tasks = self.run_bitbakecmd(["bitbake", "mc:mc1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
+            self.assertEqual(set(tasks), set())
+
+            # Test that a signature change in the dependent task causes
+            # mcdepends to rebuild
+            tasks = self.run_bitbakecmd(["bitbake", "mc:mc2:a1", "-c", "compile", "-f"], tempdir, "", extraenv=extraenv, cleanup=True)
+            expected = ["mc2:a1:compile"]
+            self.assertEqual(set(tasks), set(expected))
+
+            rerun_tasks = self.alltasks[:]
+            for x in ("fetch", "unpack", "patch", "prepare_recipe_sysroot", "configure", "compile"):
+                rerun_tasks.remove(x)
+            tasks = self.run_bitbakecmd(["bitbake", "mc:mc1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
+            expected = ["mc1:f1:%s" % t for t in rerun_tasks] + \
+                       ["mc2:a1:%s" % t for t in rerun_tasks]
+            self.assertEqual(set(tasks), set(expected))
+
     @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
     def test_hashserv_single(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
Signature · 1 line
2.26.2
#3 lore raw

RFC PATCH 3/3 sstatesig: Account for all dataCaches being passed

In reply to Joshua Watt

Bitbake now passes all the dataCaches to the taskhash API, so use this to correctly filter mcdepends.

[YOCTO #13724]

---
 meta/lib/oe/sstatesig.py | 77 +++++++++++++++++++++-------------------
 1 file changed, 40 insertions(+), 37 deletions(-)
meta/lib/oe/sstatesig.py +40 -37
index d24e3738ae..55f305c3a9 100644
@@ -2,9 +2,10 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 import bb.siggen
+import bb.runqueue
 import oe
 
-def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCache):
+def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCaches):
     # Return True if we should keep the dependency, False to drop it
     def isNative(x):
         return x.endswith("-native")
@@ -12,23 +13,26 @@ def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCache):
         return "-cross-" in x
     def isNativeSDK(x):
         return x.startswith("nativesdk-")
-    def isKernel(fn):
-        inherits = " ".join(dataCache.inherits[fn])
+    def isKernel(mc, fn):
+        inherits = " ".join(dataCaches[mc].inherits[fn])
         return inherits.find("/module-base.bbclass") != -1 or inherits.find("/linux-kernel-base.bbclass") != -1
-    def isPackageGroup(fn):
-        inherits = " ".join(dataCache.inherits[fn])
+    def isPackageGroup(mc, fn):
+        inherits = " ".join(dataCaches[mc].inherits[fn])
         return "/packagegroup.bbclass" in inherits
-    def isAllArch(fn):
-        inherits = " ".join(dataCache.inherits[fn])
+    def isAllArch(mc, fn):
+        inherits = " ".join(dataCaches[mc].inherits[fn])
         return "/allarch.bbclass" in inherits
-    def isImage(fn):
-        return "/image.bbclass" in " ".join(dataCache.inherits[fn])
-
-    # (Almost) always include our own inter-task dependencies.
-    # The exception is the special do_kernel_configme->do_unpack_and_patch
-    # dependency from archiver.bbclass.
-    if recipename == depname:
-        if task == "do_kernel_configme" and dep.endswith(".do_unpack_and_patch"):
+    def isImage(mc, fn):
+        return "/image.bbclass" in " ".join(dataCaches[mc].inherits[fn])
+
+    depmc, _, deptaskname, depmcfn = bb.runqueue.split_tid_mcfn(dep)
+    mc, _ = bb.runqueue.split_mc(fn)
+
+    # (Almost) always include our own inter-task dependencies (unless it comes
+    # from a mcdepends). The exception is the special
+    # do_kernel_configme->do_unpack_and_patch dependency from archiver.bbclass.
+    if recipename == depname and depmc == mc:
+        if task == "do_kernel_configme" and deptaskname == "do_unpack_and_patch":
             return False
         return True
 
@@ -47,11 +51,11 @@ def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCache):
     # Only target packages beyond here
 
     # allarch packagegroups are assumed to have well behaved names which don't change between architecures/tunes
-    if isPackageGroup(fn) and isAllArch(fn) and not isNative(depname):
+    if isPackageGroup(mc, fn) and isAllArch(mc, fn) and not isNative(depname):
         return False
 
     # Exclude well defined machine specific configurations which don't change ABI
-    if depname in siggen.abisaferecipes and not isImage(fn):
+    if depname in siggen.abisaferecipes and not isImage(mc, fn):
         return False
 
     # Kernel modules are well namespaced. We don't want to depend on the kernel's checksum
@@ -59,10 +63,9 @@ def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCache):
     # is machine specific.
     # Therefore if we're not a kernel or a module recipe (inheriting the kernel classes)
     # and we reccomend a kernel-module, we exclude the dependency.
-    depfn = dep.rsplit(":", 1)[0]
-    if dataCache and isKernel(depfn) and not isKernel(fn):
-        for pkg in dataCache.runrecs[fn]:
-            if " ".join(dataCache.runrecs[fn][pkg]).find("kernel-module-") != -1:
+    if dataCaches and isKernel(depmc, depmcfn) and not isKernel(mc, fn):
+        for pkg in dataCaches[mc].runrecs[fn]:
+            if " ".join(dataCaches[mc].runrecs[fn][pkg]).find("kernel-module-") != -1:
                 return False
 
     # Default to keep dependencies
@@ -87,8 +90,8 @@ class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic):
         self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE") or "").split()
         self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS") or "").split()
         pass
-    def rundep_check(self, fn, recipename, task, dep, depname, dataCache = None):
-        return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCache)
+    def rundep_check(self, fn, recipename, task, dep, depname, dataCaches = None):
+        return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCaches)
 
 class SignatureGeneratorOEBasicHashMixIn(object):
     def init_rundepcheck(self, data):
@@ -126,8 +129,8 @@ class SignatureGeneratorOEBasicHashMixIn(object):
             newsafedeps.append(a1 + "->" + a2)
         self.saferecipedeps = newsafedeps
 
-    def rundep_check(self, fn, recipename, task, dep, depname, dataCache = None):
-        return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCache)
+    def rundep_check(self, fn, recipename, task, dep, depname, dataCaches = None):
+        return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCaches)
 
     def get_taskdata(self):
         return (self.lockedpnmap, self.lockedhashfn, self.lockedhashes) + super().get_taskdata()
@@ -142,41 +145,41 @@ class SignatureGeneratorOEBasicHashMixIn(object):
         self.dump_lockedsigs(sigfile)
         return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options)
 
-    def prep_taskhash(self, tid, deps, dataCache):
-        super().prep_taskhash(tid, deps, dataCache)
+    def prep_taskhash(self, tid, deps, dataCaches):
+        super().prep_taskhash(tid, deps, dataCaches)
         if hasattr(self, "extramethod"):
-            (_, _, _, fn) = bb.runqueue.split_tid_mcfn(tid)
-            inherits = " ".join(dataCache.inherits[fn])    
+            (mc, _, _, fn) = bb.runqueue.split_tid_mcfn(tid)
+            inherits = " ".join(dataCaches[mc].inherits[fn])
             if inherits.find("/native.bbclass") != -1 or inherits.find("/cross.bbclass") != -1:
                 self.extramethod[tid] = ":" + self.buildarch
 
-    def get_taskhash(self, tid, deps, dataCache):
+    def get_taskhash(self, tid, deps, dataCaches):
         if tid in self.lockedhashes:
             if self.lockedhashes[tid]:
                 return self.lockedhashes[tid]
             else:
-                return super().get_taskhash(tid, deps, dataCache)
+                return super().get_taskhash(tid, deps, dataCaches)
 
-        # get_taskhash will call get_unihash internally in the parent class, we 
+        # get_taskhash will call get_unihash internally in the parent class, we
         # need to disable our filter of it whilst this runs else
         # incorrect hashes can be calculated.
         self._internal = True
-        h = super().get_taskhash(tid, deps, dataCache)
+        h = super().get_taskhash(tid, deps, dataCaches)
         self._internal = False
 
         (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid)
 
-        recipename = dataCache.pkg_fn[fn]
+        recipename = dataCaches[mc].pkg_fn[fn]
         self.lockedpnmap[fn] = recipename
-        self.lockedhashfn[fn] = dataCache.hashfn[fn]
+        self.lockedhashfn[fn] = dataCaches[mc].hashfn[fn]
 
         unlocked = False
         if recipename in self.unlockedrecipes:
             unlocked = True
         else:
             def recipename_from_dep(dep):
-                fn = bb.runqueue.fn_from_tid(dep)
-                return dataCache.pkg_fn[fn]
+                (depmc, _, _, depfn) = bb.runqueue.split_tid_mcfn(dep)
+                return dataCaches[depmc].pkg_fn[depfn]
 
             # If any unlocked recipe is in the direct dependencies then the
             # current recipe should be unlocked as well.
Signature · 1 line
2.26.2
#4 lore raw
In reply to Joshua Watt

Joshua,

I see you've been actively working recently on multiconfig-related fixes in bitbake, thanks! While at it, would you also be able to take a look at multiconfig-aware RDEPENDS implementation, as discussed here[1]? Thanks!

[1] https://lists.openembedded.org/g/bitbake-devel/topic/74485221

Quoted 31 lines · Joshua Watt

Updates signature generation so that mcdepends are included. Previously, they were not which meant that if recipe A mcdepends on recipe B, and recipe B changed, it would not automatically cause recipe A to also rebuild.

In order for signature generation classes to correctly handle mcdepends, they need access to all the data caches, so all are passed instead of just the one for the recipe in question.

This breaks the bitbake siggen API and makes it incompatible with previous signature generators, hence the cross-posting between bitbake and oe-core.

Joshua Watt (3):
bitbake: siggen: Pass all data caches to hash functions
bitbake: tests: Add mcdepends test
sstatesig: Account for all dataCaches being passed

 bitbake/lib/bb/runqueue.py                    |  6 +-
 bitbake/lib/bb/siggen.py                      | 32 ++++----
 .../lib/bb/tests/runqueue-tests/recipes/f1.bb |  1 +
 bitbake/lib/bb/tests/runqueue.py              | 30 ++++++++
 meta/lib/oe/sstatesig.py                      | 77 ++++++++++---------
 5 files changed, 89 insertions(+), 57 deletions(-)
 create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/f1.bb
Signature · 1 line
2.26.2
#5 lore raw
Quoted 7 lines · Denys Dmytriyenko

Joshua,

I see you've been actively working recently on multiconfig-related fixes in bitbake, thanks! While at it, would you also be able to take a look at multiconfig-aware RDEPENDS implementation, as discussed here[1]? Thanks!

[1] https://lists.openembedded.org/g/bitbake-devel/topic/74485221

I've though about this a little bit, but I'm not exactly sure what it would entail.

It might help if there was a concrete example of what you are trying to do?

Quoted 33 lines · Joshua Watt
Quoted 30 lines

Updates signature generation so that mcdepends are included. Previously, they were not which meant that if recipe A mcdepends on recipe B, and recipe B changed, it would not automatically cause recipe A to also rebuild.

In order for signature generation classes to correctly handle mcdepends, they need access to all the data caches, so all are passed instead of just the one for the recipe in question.

This breaks the bitbake siggen API and makes it incompatible with previous signature generators, hence the cross-posting between bitbake and oe-core.

Joshua Watt (3):
bitbake: siggen: Pass all data caches to hash functions
bitbake: tests: Add mcdepends test
sstatesig: Account for all dataCaches being passed

  bitbake/lib/bb/runqueue.py                    |  6 +-
  bitbake/lib/bb/siggen.py                      | 32 ++++----
  .../lib/bb/tests/runqueue-tests/recipes/f1.bb |  1 +
  bitbake/lib/bb/tests/runqueue.py              | 30 ++++++++
  meta/lib/oe/sstatesig.py                      | 77 ++++++++++---------
  5 files changed, 89 insertions(+), 57 deletions(-)

create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/f1.bb

Signature · 1 line
2.26.2
#6 lore raw

v2 cover Include mcdepends in signature generation

In reply to Joshua Watt

Updates signature generation so that mcdepends are included. Previously, they were not which meant that if recipe A mcdepends on recipe B, and recipe B changed, it would not automatically cause recipe A to also rebuild.

In order for signature generation classes to correctly handle mcdepends, they need access to all the data caches, so all are passed instead of just the one for the recipe in question.

V2: Add flag to maintain backward compatibility with older derived signature generators

Joshua Watt (2):
bitbake: siggen: Pass all data caches to hash functions
bitbake: tests: Add mcdepends test

 bitbake/lib/bb/runqueue.py                    |  8 +-
 bitbake/lib/bb/siggen.py                      | 78 +++++++++++++++----
 .../lib/bb/tests/runqueue-tests/recipes/f1.bb |  1 +
 bitbake/lib/bb/tests/runqueue.py              | 30 +++++++
 4 files changed, 98 insertions(+), 19 deletions(-)
 create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/f1.bb
Signature · 1 line
2.26.2
#7 lore raw

PATCH v2 1/2 bitbake: siggen: Pass all data caches to hash functions

Passing all the data caches to the task hashing functions allows them to correctly account for mcdepends in task signatures. This allows tasks to be correctly re-run when a mcdepends changes.

By default, the legacy behavior is maintained for derived signature generators by passing a special proxy object that can either be used to access all multiconfigs or the legacy behavior. If a derived signature generator is updated, it can set the supports_multiconfig_datacaces property to instruct bitbake it deals with multiconfigs properly.

[YOCTO #13724]

---
 bitbake/lib/bb/runqueue.py |  8 ++--
 bitbake/lib/bb/siggen.py   | 78 ++++++++++++++++++++++++++++++--------
 2 files changed, 67 insertions(+), 19 deletions(-)
bitbake/lib/bb/runqueue.py +5 -3
index 5b7dab8d79..adb34a8cf2 100644
@@ -1190,8 +1190,9 @@ class RunQueueData:
         return len(self.runtaskentries)
 
     def prepare_task_hash(self, tid):
-        bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches[mc_from_tid(tid)])
-        self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches[mc_from_tid(tid)])
+        dc = bb.parse.siggen.get_data_caches(self.dataCaches, mc_from_tid(tid))
+        bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, dc)
+        self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, dc)
         self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid)
 
     def dump_data(self):
@@ -2305,7 +2306,8 @@ class RunQueueExecute:
                 if len(self.rqdata.runtaskentries[p].depends) and not self.rqdata.runtaskentries[tid].depends.isdisjoint(total):
                     continue
                 orighash = self.rqdata.runtaskentries[tid].hash
-                newhash = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, self.rqdata.dataCaches[mc_from_tid(tid)])
+                dc = bb.parse.siggen.get_data_caches(self.rqdata.dataCaches, mc_from_tid(tid))
+                newhash = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, dc)
                 origuni = self.rqdata.runtaskentries[tid].unihash
                 newuni = bb.parse.siggen.get_unihash(tid)
                 # FIXME, need to check it can come from sstate at all for determinism?
bitbake/lib/bb/siggen.py +62 -16

148 lines not on this page · raw message

Signature · 1 line
2.26.2
#8 lore raw

PATCH v2 2/2 bitbake: tests: Add mcdepends test

In reply to Joshua Watt

Adds a test to validate that mcdepends causes the dependent tasks to build, and also that a change in the dependent task causes the dependee task to re-execute.

---
 .../lib/bb/tests/runqueue-tests/recipes/f1.bb |  1 +
 bitbake/lib/bb/tests/runqueue.py              | 30 +++++++++++++++++++
 2 files changed, 31 insertions(+)
 create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/f1.bb
bitbake/lib/bb/tests/runqueue-tests/recipes/f1.bb new file +1 -0
new file mode 100644
index 0000000000..d45a4cff52
@@ -0,0 +1 @@
+do_install[mcdepends] = "mc:mc1:mc2:a1:do_build"
bitbake/lib/bb/tests/runqueue.py +30 -0
index 091b5e41e0..d3d62b98f9 100644
@@ -248,6 +248,36 @@ class RunQueueTests(unittest.TestCase):
             cmd = ["bitbake", "mc:mc1:fails-mc2", "mc:mc2:fails-mc1"]
             self.run_bitbakecmd(cmd, tempdir, "", extraenv=extraenv)
 
+    def test_multiconfig_mcdepends(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            extraenv = {
+                "BBMULTICONFIG" : "mc1 mc2",
+                "BB_SIGNATURE_HANDLER" : "TestMulticonfigDepends",
+                "EXTRA_BBFILES": "${COREBASE}/recipes/fails-mc/*.bb",
+            }
+            tasks = self.run_bitbakecmd(["bitbake", "mc:mc1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
+            expected = ["mc1:f1:%s" % t for t in self.alltasks] + \
+                       ["mc2:a1:%s" % t for t in self.alltasks]
+            self.assertEqual(set(tasks), set(expected))
+
+            # A rebuild does nothing
+            tasks = self.run_bitbakecmd(["bitbake", "mc:mc1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
+            self.assertEqual(set(tasks), set())
+
+            # Test that a signature change in the dependent task causes
+            # mcdepends to rebuild
+            tasks = self.run_bitbakecmd(["bitbake", "mc:mc2:a1", "-c", "compile", "-f"], tempdir, "", extraenv=extraenv, cleanup=True)
+            expected = ["mc2:a1:compile"]
+            self.assertEqual(set(tasks), set(expected))
+
+            rerun_tasks = self.alltasks[:]
+            for x in ("fetch", "unpack", "patch", "prepare_recipe_sysroot", "configure", "compile"):
+                rerun_tasks.remove(x)
+            tasks = self.run_bitbakecmd(["bitbake", "mc:mc1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
+            expected = ["mc1:f1:%s" % t for t in rerun_tasks] + \
+                       ["mc2:a1:%s" % t for t in rerun_tasks]
+            self.assertEqual(set(tasks), set(expected))
+
     @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
     def test_hashserv_single(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
Signature · 1 line
2.26.2