Fix bitbake-runtask

Fixes bug [YOCTO #1229]

The bitbake doesn't use bitbake-runtask to excute the task any more, but bitbake-runtask is a useful tool for excuting the task without the scheduler of bitbake, so that the external tool can invoke it easily. The main changes are:

* It used 4 arguments in the past, but currently uses 2, the reasons:
1) The argv[1] was the tmp/cache/hashdata.dat, but bitbake doesn't dump the hashdata.dat currently, and since the bitbake-runtask is only used by the external tool currently, so we don't have to dump the hashdata.dat, I had tried to add code in lib/bb/runqueue.py to dump it, but failed.

2) The argv[4] was used to check whether it is "True" or not, if true then don't excute the task, otherwise, excute it, I dont' know why we need this, so remove it, but it would be easy to add it back.

* Since there is no hashdata.dat, there is no way to get the value of debug or debug_domains, so we don't use them any more.

* Since there is no hashdata.dat, so we can't get the hashdata, so the function make_stamp doesn't work when run bitbake-runtask, the make_stamp is use for making stamps and dump sigdata to ${STAMP} (e.g, tmp/stamps/i586-poky-linux), making stamps are ok, but dump sigdata failed since there is no hashdata , so I added a variable BB_NO_DUMPSIG to prevent it dump the sigdata when run by btibake-runtask.

* Add the log handler to bitbake-runtask, otherwise it would not print some useful information, for example, when run:

   $ bitbake-runtask gzip do_fetch

There are more than one recipes of gzip, it just prints that "MultipleMatches", but doesn't print which recipes without the
log handler.

* It prints out some strange lines, for exmaple:

(clogging
LogRecord
p2
c__builtin__
object
p3
NtRp4
(dp5
S'taskpid'
p6
I6653
sS'threadName'
p7
S'MainThread'
p8

It printed such lines in the past, but now it printed much more lines (more than 2000 lines), though I think this is harmless, I don't know how to reduce them.

Signed-off-by Robert Yang<[email protected]>

---
  bitbake/bin/bitbake-runtask |   74 ++++++++++++++++++++++--------------------
  bitbake/lib/bb/build.py     |    4 +-
  2 files changed, 41 insertions(+), 37 deletions(-)

bitbake/bin/bitbake-runtask +39 -35

index bee0f42..68f6c33 100755
@@ -4,6 +4,10 @@ import os
  import sys
  import warnings
  sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
+from bb import fetch2
+import logging
+
+logger = logging.getLogger("BitBake")
 
  try:
      import cPickle as pickle

@@ -11,18 +15,24 @@ except ImportError:
      import pickle
      bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.")
 
+
  class BBConfiguration(object):
      """
      Manages build options and configurations for one run
      """
 
-    def __init__(self, debug, debug_domains):
-        setattr(self, "data", {})
-        setattr(self, "file", [])
-        setattr(self, "cmd", None)
-        setattr(self, "dump_signatures", True)
-        setattr(self, "debug", debug)
-        setattr(self, "debug_domains", debug_domains)
+    def __init__(self, **options):
+        self.prefile = []
+        self.postfile = []
+        self.parse_only = True
+
+    def __getattr__(self, attribute):
+        try:
+            return super(BBConfiguration, self).__getattribute__(attribute)
+        except AttributeError:
+            return None
+
+
 
  _warnings_showwarning = warnings.showwarning
  def _showwarning(message, category, filename, lineno, file=None, line=None):

@@ -68,37 +78,38 @@ os.setpgrp()
  bb.event.worker_pid = os.getpid()
  bb.event.useStdout = False
 
-hashfile = sys.argv[1]
-buildfile = sys.argv[2]
-taskname = sys.argv[3]
+buildfile = sys.argv[1]
+taskname = sys.argv[2]
 
  import bb.cooker
 
-p = pickle.Unpickler(file(hashfile, "rb"))
-hashdata = p.load()
+def register_idle_function(self, function, data):
+    pass
 
-debug = hashdata["msg-debug"]
-debug_domains = hashdata["msg-debug-domains"]
-verbose = hashdata["verbose"]
+# Print the log information
+handler = bb.event.LogHandler()
+logger.setLevel(logging.INFO)
+logger.addHandler(handler)
 
-bb.utils.init_logger(bb.msg, verbose, debug, debug_domains)
+initialenv = os.environ.copy()
+config = BBConfiguration()
 
-cooker = bb.cooker.BBCooker(BBConfiguration(debug, debug_domains), None)
-cooker.parseConfiguration()
+cooker = bb.cooker.BBCooker(config, register_idle_function, initialenv)
+config_data = cooker.configuration.data
+cooker.status = config_data
+cooker.handleCollections(bb.data.getVar("BBFILE_COLLECTIONS", config_data, 1))
 
-cooker.bb_cache = bb.cache.init(cooker)
-cooker.status = bb.cache.CacheData()
-
-(fn, cls) = cooker.bb_cache.virtualfn2realfn(buildfile)
+fn, cls = bb.cache.Cache.virtualfn2realfn(buildfile)
  buildfile = cooker.matchFile(fn)
-fn = cooker.bb_cache.realfn2virtual(buildfile, cls)
+fn = bb.cache.Cache.realfn2virtual(buildfile, cls)
 
  cooker.buildSetVars()
 
  # Load data into the cache for fn and parse the loaded cache data
-the_data = cooker.bb_cache.loadDataFull(fn, cooker.get_file_appends(fn), cooker.configuration.data)
-cooker.bb_cache.setData(fn, buildfile, the_data)
-cooker.bb_cache.handle_data(fn, cooker.status)
+the_data = bb.cache.Cache.loadDataFull(fn, cooker.get_file_appends(fn), cooker.configuration.data)
+
+# Don't dump the signature for the task since we have no hashdata
+the_data.setVar("BB_NO_DUMPSIG", True)
 
  #exportlist = bb.utils.preserved_envvars_export_list()
  #bb.utils.filter_environment(exportlist)

@@ -106,15 +117,8 @@ cooker.bb_cache.handle_data(fn, cooker.status)
  if taskname.endswith("_setscene"):
      the_data.setVarFlag(taskname, "quieterrors", "1")
 
-bb.parse.siggen.set_taskdata(hashdata["hashes"], hashdata["deps"])
-
-for h in hashdata["hashes"]:
-    bb.data.setVar("BBHASH_%s" % h, hashdata["hashes"][h], the_data)
-for h in hashdata["deps"]:
-    bb.data.setVar("BBHASHDEPS_%s" % h, hashdata["deps"][h], the_data)
-
  ret = 0
-if sys.argv[4] != "True":
-    ret = bb.build.exec_task(fn, taskname, the_data)
+
+ret = bb.build.exec_task(fn, taskname, the_data)
  sys.exit(ret)
 

bitbake/lib/bb/build.py +2 -2

index 8937f08..898678a 100644
@@ -415,8 +415,8 @@ def make_stamp(task, d, file_name = None):
          f.close()
 
      # If we're in task context, write out a signature file for each task
-    # as it completes
-    if not task.endswith("_setscene") and task != "do_setscene" and not file_name:
+    # as it completes, the BB_NO_DUMPSIG is used by bitbake-runtask currently
+    if not task.endswith("_setscene") and task != "do_setscene" and not file_name and not d.getVar('BB_NO_DUMPSIG', True):
          file_name = d.getVar('BB_FILENAME', True)
          bb.parse.siggen.dump_sigtask(file_name, task, d.getVar('STAMP', True), True)
 

Fixes bug [YOCTO #1229]

The bitbake doesn't use bitbake-runtask to excute the task any more, but bitbake-runtask is a useful tool for excuting the task without the scheduler of bitbake, so that the external tool can invoke it easily. The main changes are:

* It used 4 arguments in the past, but currently uses 2, the reasons:
1) The argv[1] was the tmp/cache/hashdata.dat, but bitbake doesn't dump the hashdata.dat currently, and since the bitbake-runtask is only used by the external tool currently, so we don't have to dump the hashdata.dat, I had tried to add code in lib/bb/runqueue.py to dump it, but failed.

2) The argv[4] was used to check whether it is "True" or not, if true then don't excute the task, otherwise, excute it, I dont' know why we need this, so remove it, but it would be easy to add it back.

* Since there is no hashdata.dat, there is no way to get the value of debug or debug_domains, so we don't use them any more.

* Since there is no hashdata.dat, so we can't get the hashdata, so the function make_stamp doesn't work when run bitbake-runtask, the make_stamp is use for making stamps and dump sigdata to ${STAMP} (e.g, tmp/stamps/i586-poky-linux), making stamps are ok, but dump sigdata failed since there is no hashdata , so I added a variable BB_NO_DUMPSIG to prevent it dump the sigdata when run by btibake-runtask.

* Add the log handler to bitbake-runtask, otherwise it would not print some useful information, for example, when run:

  $ bitbake-runtask gzip do_fetch

There are more than one recipes of gzip, it just prints that "MultipleMatches", but doesn't print which recipes without the
log handler.

* It prints out some strange lines, for exmaple:
(clogging
LogRecord
p2
c__builtin__
object
p3
NtRp4
(dp5
S'taskpid'
p6
I6653
sS'threadName'
p7
S'MainThread'
p8

It printed such lines in the past, but now it printed much more lines (more than 2000 lines), though I think this is harmless, I don't know how to reduce them.

Hi Robert,

Quoted 54 lines · Robert Yang

Fixes bug [YOCTO #1229]

The bitbake doesn't use bitbake-runtask to excute the task any more, but bitbake-runtask is a useful tool for excuting the task without the scheduler of bitbake, so that the external tool can invoke it easily. The main changes are:

* It used 4 arguments in the past, but currently uses 2, the reasons:
1) The argv[1] was the tmp/cache/hashdata.dat, but bitbake doesn't dump the hashdata.dat currently, and since the bitbake-runtask is only used by the external tool currently, so we don't have to dump the hashdata.dat, I had tried to add code in lib/bb/runqueue.py to dump it, but failed.

2) The argv[4] was used to check whether it is "True" or not, if true then don't excute the task, otherwise, excute it, I dont' know why we need this, so remove it, but it would be easy to add it back.

* Since there is no hashdata.dat, there is no way to get the value of debug or debug_domains, so we don't use them any more.

* Since there is no hashdata.dat, so we can't get the hashdata, so the function make_stamp doesn't work when run bitbake-runtask, the make_stamp is use for making stamps and dump sigdata to ${STAMP} (e.g, tmp/stamps/i586-poky-linux), making stamps are ok, but dump sigdata failed since there is no hashdata , so I added a variable BB_NO_DUMPSIG to prevent it dump the sigdata when run by btibake-runtask.

* Add the log handler to bitbake-runtask, otherwise it would not print some useful information, for example, when run:

   $ bitbake-runtask gzip do_fetch

There are more than one recipes of gzip, it just prints that "MultipleMatches", but doesn't print which recipes without the
log handler.

* It prints out some strange lines, for exmaple:
(clogging
LogRecord
p2
c__builtin__
object
p3
NtRp4
(dp5
S'taskpid'
p6
I6653
sS'threadName'
p7
S'MainThread'
p8

Quoted 3 lines

It printed such lines in the past, but now it printed much more lines (more than 2000 lines), though I think this is harmless, I don't know how to reduce them.

Sorry for the delay in getting back to you. This patch raised some questions I couldn't answer off the top of my head without experimenting with the codebase a bit.

I've created an updated version of this patch which is at:
http://git.yoctoproject.org/cgit.cgi/poky-contrib/commit/?h=rpurdie/useradd2&id=f11b9e26d806557e7f79538b87e20f5d63a11762
and also pasted below.

The changes:

I reworked the arguments passed to the code to be cleaner and to make some of them optional. This makes it more intuitive to use.

I changed the log handling in this code entirely. It now hooks into the log parsing code within bitbake so all log messages are nicely formatted. If someone did want the raw messages, we could add a mode which passed them to stdout like that but I think this is a good default mode.

I also found a neater way to avoid the BB_NO_DUMPSIG option and retain some of the signature generation code.

Can you have a look and see if this revised code does everything you need? Reading the patch, I need to clean up the mess I made of the BBConfiguration class but please see if there are any other problems.

Cheers,

Richard

Quoted 1 line

From f11b9e26d806557e7f79538b87e20f5d63a11762 Mon Sep 17 00:00:00 2001

From: Robert Yang<[email protected]>
Date: Thu, 24 Nov 2011 12:55:14 +0000
Subject: Fix bitbake-runtask

Signed-off-by Richard Purdie<[email protected]>

---

bitbake/bin/bitbake-runtask +68 -66

index bee0f42..7a3775c 100755
@@ -4,6 +4,10 @@ import os
  import sys
  import warnings
  sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
+from bb import fetch2
+import logging
+
+logger = logging.getLogger("BitBake")
 
  try:
      import cPickle as pickle

@@ -16,13 +20,23 @@ class BBConfiguration(object):
      Manages build options and configurations for one run
      """
 
-    def __init__(self, debug, debug_domains):
-        setattr(self, "data", {})
-        setattr(self, "file", [])
-        setattr(self, "cmd", None)
-        setattr(self, "dump_signatures", True)
-        setattr(self, "debug", debug)
-        setattr(self, "debug_domains", debug_domains)
+    def __init__(self, **options):
+    #def __init__(self, debug, debug_domains):
+        self.data = {}
+        self.file = []
+        self.cmd = None
+        self.dump_signatures = True
+        #self.debug = debug
+        #self.debug_domains", debug_domains)
+        self.prefile = []
+        self.postfile = []
+        self.parse_only = True
+
+    def __getattr__(self, attribute):
+        try:
+            return super(BBConfiguration, self).__getattribute__(attribute)
+        except AttributeError:
+            return None
 
  _warnings_showwarning = warnings.showwarning
  def _showwarning(message, category, filename, lineno, file=None, line=None):

@@ -39,82 +53,70 @@ warnings.showwarning = _showwarning
  warnings.simplefilter("ignore", DeprecationWarning)
 
  import bb.event
-
-# Need to map our I/O correctly. stdout is a pipe to the server expecting
-# events. We save this and then map stdout to stderr.
-
-eventfd = os.dup(sys.stdout.fileno())
-bb.event.worker_pipe = os.fdopen(eventfd, 'w', 0)
-
-# map stdout to stderr
-os.dup2(sys.stderr.fileno(), sys.stdout.fileno())
-
-# Replace those fds with our own
-#logout = data.expand("${TMPDIR}/log/stdout.%s" % os.getpid(), self.cfgData, True)
-#mkdirhier(os.path.dirname(logout))
-#newso = open("/tmp/stdout.%s" % os.getpid(), 'w')
-#os.dup2(newso.fileno(), sys.stdout.fileno())
-#os.dup2(newso.fileno(), sys.stderr.fileno())
-
-# Don't read from stdin from the parent
-si = file("/dev/null", 'r')
-os.dup2(si.fileno( ), sys.stdin.fileno( ))
-
-# We don't want to see signals to our parent, e.g. Ctrl+C
-os.setpgrp()
-
-# Save out the PID so that the event can include it the
-# events
-bb.event.worker_pid = os.getpid()
-bb.event.useStdout = False
-
-hashfile = sys.argv[1]
-buildfile = sys.argv[2]
-taskname = sys.argv[3]
-
  import bb.cooker
 
-p = pickle.Unpickler(file(hashfile, "rb"))
-hashdata = p.load()
-
-debug = hashdata["msg-debug"]
-debug_domains = hashdata["msg-debug-domains"]
-verbose = hashdata["verbose"]
+buildfile = sys.argv[1]
+taskname = sys.argv[2]
+if len(sys.argv)>= 4:
+    dryrun = sys.argv[3]
+else:
+    dryrun = False
+if len(sys.argv)>= 5:
+    hashfile = sys.argv[4]
+    p = pickle.Unpickler(file(hashfile, "rb"))
+    hashdata = p.load()
+else:
+    hashdata = None
+
+handler = bb.event.LogHandler()
+logger.addHandler(handler)
+
+#An example to make debug log messages show up
+#bb.msg.init_msgconfig(True, 3, [])
+
+console = logging.StreamHandler(sys.stdout)
+format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
+bb.msg.addDefaultlogFilter(console)
+console.setFormatter(format)
+
+def worker_fire(event, d):
+    if isinstance(event, logging.LogRecord):
+        console.handle(event)
+bb.event.worker_fire = worker_fire
+bb.event.worker_pid = os.getpid()
 
-bb.utils.init_logger(bb.msg, verbose, debug, debug_domains)
+initialenv = os.environ.copy()
+config = BBConfiguration()
 
-cooker = bb.cooker.BBCooker(BBConfiguration(debug, debug_domains), None)
-cooker.parseConfiguration()
+def register_idle_function(self, function, data):
+    pass
 
-cooker.bb_cache = bb.cache.init(cooker)
-cooker.status = bb.cache.CacheData()
+cooker = bb.cooker.BBCooker(config, register_idle_function, initialenv)
+config_data = cooker.configuration.data
+cooker.status = config_data
+cooker.handleCollections(bb.data.getVar("BBFILE_COLLECTIONS", config_data, 1))
 
-(fn, cls) = cooker.bb_cache.virtualfn2realfn(buildfile)
+fn, cls = bb.cache.Cache.virtualfn2realfn(buildfile)
  buildfile = cooker.matchFile(fn)
-fn = cooker.bb_cache.realfn2virtual(buildfile, cls)
+fn = bb.cache.Cache.realfn2virtual(buildfile, cls)
 
  cooker.buildSetVars()
 
  # Load data into the cache for fn and parse the loaded cache data
-the_data = cooker.bb_cache.loadDataFull(fn, cooker.get_file_appends(fn), cooker.configuration.data)
-cooker.bb_cache.setData(fn, buildfile, the_data)
-cooker.bb_cache.handle_data(fn, cooker.status)
-
-#exportlist = bb.utils.preserved_envvars_export_list()
-#bb.utils.filter_environment(exportlist)
+the_data = bb.cache.Cache.loadDataFull(fn, cooker.get_file_appends(fn), cooker.configuration.data)
 
  if taskname.endswith("_setscene"):
      the_data.setVarFlag(taskname, "quieterrors", "1")
 
-bb.parse.siggen.set_taskdata(hashdata["hashes"], hashdata["deps"])
-
-for h in hashdata["hashes"]:
-    bb.data.setVar("BBHASH_%s" % h, hashdata["hashes"][h], the_data)
-for h in hashdata["deps"]:
-    bb.data.setVar("BBHASHDEPS_%s" % h, hashdata["deps"][h], the_data)
+if hashdata:
+    bb.parse.siggen.set_taskdata(hashdata["hashes"], hashdata["deps"])
+    for h in hashdata["hashes"]:
+        bb.data.setVar("BBHASH_%s" % h, hashdata["hashes"][h], the_data)
+    for h in hashdata["deps"]:
+        bb.data.setVar("BBHASHDEPS_%s" % h, hashdata["deps"][h], the_data)
 
  ret = 0
-if sys.argv[4] != "True":
+if dryrun != "True":
      ret = bb.build.exec_task(fn, taskname, the_data)
  sys.exit(ret)
 

bitbake/lib/bb/siggen.py +2 -2

index 9231291..3aa3fe1 100644
@@ -160,7 +160,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
          k = fn + "." + task
          if runtime == "customfile":
              sigfile = stampbase
-        elif runtime:
+        elif runtime and k in self.taskhash:
              sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[k]
          else:
              sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[k]

@@ -181,7 +181,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
              data['gendeps'][dep] = self.gendeps[fn][dep]
              data['varvals'][dep] = self.lookupcache[fn][dep]
 
-        if runtime:
+        if runtime and k in self.taskhash:
              data['runtaskdeps'] = self.runtaskdeps[k]
              data['runtaskhashes'] = {}
              for dep in data['runtaskdeps']:

Thank you very much, this is pretty good, I've tested it, it worked well. What shall I do for it then? Should I send the pull request or you will merge it from you branch?