runqueue: dry-run real tasks when BB_SETSCENE_ENFORCE is set
v2 cover runqueue: dry-run real tasks when BB_SETSCENE_ENFORCE is set
Changes since v1:
* Ensure the worker does actually run the task during setscene tasks, which it erroneously wasn't in v1 due to the same main worker instance being shared between the setscene and normal tasks - I clearly hadn't tested the patch properly.
* Compare BB_SETSCENE_ENFORCE to "1" rather than just checking it's assigned
The following changes since commit a2768ecae7846d72a1bdb7cbbc5e8d242af854f6:
bitbake-user-manual: Added new "Line Joining" section. (2016-12-08 16:35:56 +0000)
are available in the git repository at:
git://git.yoctoproject.org/poky-contrib paule/setscene-enforce-dryrun
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=paule/setscene-enforce-dryrun
Paul Eggleton (1):
runqueue: dry-run real tasks when BB_SETSCENE_ENFORCE is set
bin/bitbake-worker | 12 +++++++----- lib/bb/runqueue.py | 13 +++++++------ 2 files changed, 14 insertions(+), 11 deletions(-)
Signature · 1 line
2.5.5
PATCH v2 1/1 runqueue: dry-run real tasks when BB_SETSCENE_ENFORCE is set
For the purposes BB_SETSCENE_ENFORCE is designed for (in OE, it is used by the installation process for the extensible SDK), we don't actually need the whitelisted real tasks to execute - we just need to have them in the dependency tree so that we get all of the setscene tasks they depend on to run. Therefore we can actually dry-run those real tasks i.e. they won't be run (and thus we won't waste a significant amount of time doing so) and won't be stamped as having run either. We do already have a dry-run mode in BitBake (activated by the -n or --dry-run command line option), but it dry-runs the setscene tasks as well which we don't want here.
Note that this has no effect on the checking we are doing with BB_SETSCENE_ENFORCE to ensure that only whitelisted real tasks are scheduled to run - that's handled separately.
Fixes [YOCTO #10369].
--- bin/bitbake-worker | 12 +++++++----- lib/bb/runqueue.py | 13 +++++++------ 2 files changed, 14 insertions(+), 11 deletions(-)
bin/bitbake-worker +7 -5
@@ -136,7 +136,7 @@ def sigterm_handler(signum, frame): os.killpg(0, signal.SIGTERM) sys.exit() -def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, appends, taskdepdata, quieterrors=False): +def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, appends, taskdepdata, quieterrors=False, dry_run_exec=False): # We need to setup the environment BEFORE the fork, since # a fork() or exec*() activates PSEUDO...
@@ -152,8 +152,10 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, append except TypeError: umask = taskdep['umask'][taskname] + dry_run = cfg.dry_run or dry_run_exec + # We can't use the fakeroot environment in a dry run as it possibly hasn't been built - if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not cfg.dry_run: + if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not dry_run: envvars = (workerdata["fakerootenv"][fn] or "").split() for key, value in (var.split('=') for var in envvars): envbackup[key] = os.environ.get(key)
@@ -260,7 +262,7 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, append logger.critical(traceback.format_exc()) os._exit(1) try: - if cfg.dry_run: + if dry_run: return 0 return bb.build.exec_task(fn, taskname, the_data, cfg.profile) except:
@@ -413,10 +415,10 @@ class BitbakeWorker(object): sys.exit(0) def handle_runtask(self, data): - fn, task, taskname, quieterrors, appends, taskdepdata = pickle.loads(data) + fn, task, taskname, quieterrors, appends, taskdepdata, dry_run_exec = pickle.loads(data) workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname)) - pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, appends, taskdepdata, quieterrors) + pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, appends, taskdepdata, quieterrors, dry_run_exec) self.build_pids[pid] = task self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout)
lib/bb/runqueue.py +7 -6
@@ -266,6 +266,7 @@ class RunQueueData: self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST") or "").split() self.setscenewhitelist = get_setscene_enforce_whitelist(cfgData) self.setscenewhitelist_checked = False + self.setscene_enforce = (cfgData.getVar('BB_SETSCENE_ENFORCE') == "1") self.init_progress_reporter = bb.progress.DummyMultiStageProcessProgressReporter() self.reset()
@@ -1789,7 +1790,7 @@ class RunQueueExecuteTasks(RunQueueExecute): bb.event.fire(startevent, self.cfgData) self.runq_running.add(task) self.stats.taskActive() - if not self.cooker.configuration.dry_run: + if not (self.cooker.configuration.dry_run or self.rqdata.setscene_enforce): bb.build.make_stamp(taskname, self.rqdata.dataCaches[mc], taskfn) self.task_complete(task) return True
@@ -1800,7 +1801,7 @@ class RunQueueExecuteTasks(RunQueueExecute): taskdepdata = self.build_taskdepdata(task) taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] - if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: + if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not (self.cooker.configuration.dry_run or self.rqdata.setscene_enforce): if not self.rq.fakeworker: try: self.rq.start_fakeworker(self)
@@ -1809,10 +1810,10 @@ class RunQueueExecuteTasks(RunQueueExecute): self.rq.state = runQueueFailed self.stats.taskFailed() return True - self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, False, self.cooker.collection.get_file_appends(fn), taskdepdata)) + b"</runtask>") + self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, False, self.cooker.collection.get_file_appends(fn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") self.rq.fakeworker[mc].process.stdin.flush() else: - self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata)) + b"</runtask>") + self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") self.rq.worker[mc].process.stdin.flush() self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
@@ -2218,10 +2219,10 @@ class RunQueueExecuteScenequeue(RunQueueExecute): if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: if not self.rq.fakeworker: self.rq.start_fakeworker(self) - self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata)) + b"</runtask>") + self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") self.rq.fakeworker[mc].process.stdin.flush() else: - self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata)) + b"</runtask>") + self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") self.rq.worker[mc].process.stdin.flush() self.runq_running.add(task)
Signature · 1 line
2.5.5