Skip to contentSkip to conversation

Hob: bitbake related changes

GIT PULL In review Dongxiao Xu 2 participants 11 messages Jan 13, 2012 – Jan 16, 2012

lore
Share & tools
lore
#0 lore raw

Hi Richard,

This is the third pull request for Hob bitbake related changes.

Please help to review and pull.

Changes from v2:
1. Fix parameter checking issue on --bind and --server-only.
2. Add a new command triggerEvent(), which enables the client to request triggering specific event from bitbake server.
3. Add a specific event 'RequestPackageInfo' to get available package information.
4. Change the way to get the CPU information on server side.

Thanks,
Dongxiao

The following changes since commit 9d8f45407c67ed0d3c4f820cf646de3c385067c7:
Lianhao Lu (1):
bitbake: Automatically start local PR service.

are available in the git repository at:

git://git.pokylinux.org/poky-contrib dxu4/hob-bitbake-changes-v3
http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=dxu4/hob-bitbake-changes-v3

Dongxiao Xu (7):
cache: Use configuration's hash value to validate cache command.py: add resolve option for generateTargetsTree API event.py: Add new events RequestPackageInfo and PackageInfo
bitbake: add -B option to bind with interface
bitbake: Add client socket info for BitBakeServerConnection command.py: Add a new API triggerEvent()
runqueue: fire sceneQueueTaskStarted event when a setscene queue
starts

Shane Wang (2):
bitbake: change for adding progress bar in Hob2. command.py: add new API to get the cpu count on the server

 bin/bitbake             |   17 +++++++-
 lib/bb/cache.py         |   34 ++++++----------
 lib/bb/command.py       |   23 ++++++++++-
 lib/bb/cooker.py        |   74 ++++++++++++++++++++++++++++--------
 lib/bb/data_smart.py    |   21 ++++++++++
 lib/bb/event.py         |   96 +++++++++++++++++++++++++++++++++++-----------
 lib/bb/runqueue.py      |   10 +++++
 lib/bb/server/xmlrpc.py |   10 ++--
 lib/bb/ui/knotty.py     |    5 ++-
 lib/bb/ui/uievent.py    |    7 ++-
 lib/bb/utils.py         |    4 ++
 11 files changed, 227 insertions(+), 74 deletions(-)
#4 lore raw

PATCH 3/9 event.py: Add new events RequestPackageInfo and PackageInfo

RequestPackageInfo is triggered by GUI client to request the available package information.

PackageInfo event is to pass package information back to GUI.

---
 lib/bb/event.py |   13 +++++++++++++
 1 files changed, 13 insertions(+), 0 deletions(-)
lib/bb/event.py +13 -0
index 8d7f941..10036c0 100644
@@ -467,3 +467,16 @@ class LogHandler(logging.Handler):
     def filter(self, record):
         record.taskpid = worker_pid
         return True
+
+class RequestPackageInfo(Event):
+    """
+    Event to request package information
+    """
+
+class PackageInfo(Event):
+    """
+    Package information for GUI
+    """
+    def __init__(self, pkginfolist):
+        Event.__init__(self)
+        self._pkginfolist = pkginfolist
Signature · 1 line
1.7.0.4
#5 lore raw

PATCH 4/9 bitbake: change for adding progress bar in Hob2.

In reply to Dongxiao Xu

From: Shane Wang <[email protected]>

The changes include:
- Clean some events in event.py
- Fire essential events for Hob2 to handle with more information.
- knotty changes

---
 lib/bb/cache.py     |    2 +-
 lib/bb/cooker.py    |   12 +++++--
 lib/bb/event.py     |   83 ++++++++++++++++++++++++++++++++++++--------------
 lib/bb/ui/knotty.py |    5 ++-
 4 files changed, 73 insertions(+), 29 deletions(-)
lib/bb/cache.py +1 -1
index 955b6df..c268fe6 100644
@@ -345,7 +345,7 @@ class Cache(object):
                         current_percent = 100 * current_progress / cachesize
                         if current_percent > previous_percent:
                             previous_percent = current_percent
-                            bb.event.fire(bb.event.CacheLoadProgress(current_progress),
+                            bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize),
                                           self.data)
 
                     previous_progress += current_progress
lib/bb/cooker.py +8 -4
index 875693f..7dd7c2a 100644
@@ -345,6 +345,7 @@ class BBCooker:
         """
         Prepare a runqueue and taskdata object for iteration over pkgs_to_build
         """
+        bb.event.fire(bb.event.TreeDataPreparationStarted(), self.configuration.data)
         # Need files parsed
         self.updateCache()
         # If we are told to do the None task then query the default task
@@ -361,11 +362,14 @@ class BBCooker:
         taskdata = bb.taskdata.TaskData(False, skiplist=self.skiplist)
 
         runlist = []
+        current = 0
         for k in pkgs_to_build:
             taskdata.add_provider(localdata, self.status, k)
             runlist.append([k, "do_%s" % task])
+            current += 1
+            bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(pkgs_to_build)), self.configuration.data)
         taskdata.add_unresolved(localdata, self.status)
-
+        bb.event.fire(bb.event.TreeDataPreparationCompleted(len(pkgs_to_build)), self.configuration.data)
         return runlist, taskdata
 
     def generateTaskDepTreeData(self, pkgs_to_build, task):
@@ -1106,7 +1110,7 @@ class BBCooker:
                 return False
 
             if not retval:
-                bb.event.fire(bb.event.BuildCompleted(buildname, item, failures), self.configuration.event_data)
+                bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, item, failures), self.configuration.event_data)
                 self.command.finishAsyncCommand()
                 return False
             if retval is True:
@@ -1147,7 +1151,7 @@ class BBCooker:
                 return False
 
             if not retval:
-                bb.event.fire(bb.event.BuildCompleted(buildname, targets, failures), self.configuration.data)
+                bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, targets, failures), self.configuration.data)
                 self.command.finishAsyncCommand()
                 return False
             if retval is True:
@@ -1565,7 +1569,7 @@ class CookerParser(object):
         if parsed:
             self.parsed += 1
             if self.parsed % self.progress_chunk == 0:
-                bb.event.fire(bb.event.ParseProgress(self.parsed),
+                bb.event.fire(bb.event.ParseProgress(self.parsed, self.toparse),
                               self.cfgdata)
         else:
             self.cached += 1
lib/bb/event.py +60 -23

130 lines not on this page · raw message

lib/bb/ui/knotty.py +4 -1
index 0340619..b4d901f 100644
@@ -251,7 +251,10 @@ def main(server, eventHandler):
                                   bb.event.RecipeParsed,
                                   bb.event.RecipePreFinalise,
                                   bb.runqueue.runQueueEvent,
-                                  bb.runqueue.runQueueExitWait)):
+                                  bb.runqueue.runQueueExitWait,
+                                  bb.event.OperationStarted,
+                                  bb.event.OperationCompleted,
+                                  bb.event.OperationProgress)):
                 continue
 
             logger.error("Unknown event: %s", event)
Signature · 1 line
1.7.0.4
#6 lore raw

PATCH 5/9 bitbake: add -B option to bind with interface

In reply to Dongxiao Xu

When start bitbake as a server only process, we need to assign certain interface to it.

---
 bin/bitbake             |   17 ++++++++++++++---
 lib/bb/server/xmlrpc.py |    6 +++---
 2 files changed, 17 insertions(+), 6 deletions(-)
bin/bitbake +14 -3
index c2e6822..a180373 100755
@@ -168,6 +168,8 @@ Default BBFILES are the .bb files in the current directory.""")
     parser.add_option("", "--server-only", help = "Run bitbake without UI,  the frontend can connect with bitbake server itself",
                action = "store_true", dest = "server_only", default = False)
 
+    parser.add_option("-B", "--bind", help = "The name/address for the bitbake server to bind to",
+               action = "store", dest = "bind", default = False)
     options, args = parser.parse_args(sys.argv)
 
     configuration = BBConfiguration(options)
@@ -189,8 +191,14 @@ Default BBFILES are the .bb files in the current directory.""")
         sys.exit("FATAL: Invalid server type '%s' specified.\n"
                  "Valid interfaces: xmlrpc, process [default], none." % servertype)
 
-    if configuration.server_only and configuration.servertype != "xmlrpc":
-        sys.exit("FATAL: If '--server-only' is defined, we must set the servertype as 'xmlrpc'.\n")
+    if configuration.server_only:
+        if configuration.servertype != "xmlrpc":
+            sys.exit("FATAL: If '--server-only' is defined, we must set the servertype as 'xmlrpc'.\n")
+        if not configuration.bind:
+            sys.exit("FATAL: The '--server-only' option requires a name/address to bind to with the -B option.\n")
+
+    if configuration.bind and configuration.servertype != "xmlrpc":
+        sys.exit("FATAL: If '-B' or '--bind' is defined, we must set the servertype as 'xmlrpc'.\n")
 
     # Save a logfile for cooker into the current working directory. When the
     # server is daemonized this logfile will be truncated.
@@ -212,8 +220,11 @@ Default BBFILES are the .bb files in the current directory.""")
     bb.utils.clean_environment()
 
     server = server.BitBakeServer()
+    if configuration.bind:
+        server.initServer((configuration.bind, 0))
+    else:
+        server.initServer()
 
-    server.initServer()
     idle = server.getServerIdleCB()
 
     cooker = bb.cooker.BBCooker(configuration, idle, initialenv)
lib/bb/server/xmlrpc.py +3 -3
index b5980c6..c53cee4 100644
@@ -163,7 +163,7 @@ class BitBakeXMLRPCServer(SimpleXMLRPCServer):
     # remove this when you're done with debugging
     # allow_reuse_address = True
 
-    def __init__(self, interface = ("localhost", 0)):
+    def __init__(self, interface):
         """
         Constructor
         """
@@ -267,8 +267,8 @@ class BitBakeServerConnection():
             pass
 
 class BitBakeServer(object):
-    def initServer(self):
-        self.server = BitBakeXMLRPCServer()
+    def initServer(self, interface = ("localhost", 0)):
+        self.server = BitBakeXMLRPCServer(interface)
 
     def addcooker(self, cooker):
         self.cooker = cooker
Signature · 1 line
1.7.0.4
#8 lore raw

PATCH 7/9 command.py: add new API to get the cpu count on the server

In reply to Dongxiao Xu

From: Shane Wang <[email protected]>

Add a new API in command.py to get the cpu count in order to set the appropriate default BB_NUMBER_THREADS and PARALLEL_MAKE variables.

---
 lib/bb/command.py |    6 ++++++
 lib/bb/utils.py   |    4 ++++
 2 files changed, 10 insertions(+), 0 deletions(-)
lib/bb/command.py +6 -0
index 05555c5..43875f7 100644
@@ -173,6 +173,12 @@ class CommandsSync:
         """
         command.cooker.reset()
 
+    def getCpuCount(self, command, params):
+        """
+        Get the CPU count on the bitbake server
+        """
+        return bb.utils.cpu_count()
+
 
 class CommandsAsync:
     """
lib/bb/utils.py +4 -0
index a26635a..d7cefb2 100644
@@ -25,6 +25,7 @@ import errno
 import logging
 import bb
 import bb.msg
+import multiprocessing
 from commands import getstatusoutput
 from contextlib import contextmanager
 
@@ -862,3 +863,6 @@ def contains(variable, checkvalues, truevalue, falsevalue, d):
     if checkvalues.issubset(val): 
         return truevalue
     return falsevalue
+
+def cpu_count():
+    return multiprocessing.cpu_count()
Signature · 1 line
1.7.0.4
#7 lore raw

PATCH 6/9 bitbake: Add client socket info for BitBakeServerConnection

In reply to Dongxiao Xu

In server/client split model, the client will bind to a specific address and port. We need to pass the values to BitBakeServerConnection().

---
 lib/bb/server/xmlrpc.py |    4 ++--
 lib/bb/ui/uievent.py    |    7 ++++---
 2 files changed, 6 insertions(+), 5 deletions(-)
lib/bb/server/xmlrpc.py +2 -2
index c53cee4..eff8009 100644
@@ -247,9 +247,9 @@ class BitbakeServerInfo():
         self.port = port
 
 class BitBakeServerConnection():
-    def __init__(self, serverinfo):
+    def __init__(self, serverinfo, clientinfo=("localhost", 0)):
         self.connection = _create_server(serverinfo.host, serverinfo.port)
-        self.events = uievent.BBUIEventQueue(self.connection)
+        self.events = uievent.BBUIEventQueue(self.connection, clientinfo)
         for event in bb.event.ui_queue:
             self.events.queue_event(event)
 
lib/bb/ui/uievent.py +4 -3
index 0e73817..28817a2 100644
@@ -28,13 +28,14 @@ import socket, threading, pickle
 from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
 
 class BBUIEventQueue:
-    def __init__(self, BBServer):
+    def __init__(self, BBServer, clientinfo=("localhost, 0")):
 
         self.eventQueue = []
         self.eventQueueLock = threading.Lock()
         self.eventQueueNotify = threading.Event()
 
         self.BBServer = BBServer
+        self.clientinfo = clientinfo
 
         self.t = threading.Thread()
         self.t.setDaemon(True)
@@ -72,7 +73,7 @@ class BBUIEventQueue:
 
     def startCallbackHandler(self):
 
-        server = UIXMLRPCServer()
+        server = UIXMLRPCServer(self.clientinfo)
         self.host, self.port = server.socket.getsockname()
 
         server.register_function( self.system_quit, "event.quit" )
@@ -98,7 +99,7 @@ class BBUIEventQueue:
 
 class UIXMLRPCServer (SimpleXMLRPCServer):
 
-    def __init__( self, interface = ("localhost", 0) ):
+    def __init__( self, interface ):
         self.quit = False
         SimpleXMLRPCServer.__init__( self,
                                     interface,
Signature · 1 line
1.7.0.4
#9 lore raw

PATCH 8/9 command.py: Add a new API triggerEvent()

In reply to Dongxiao Xu

This functions enables the client to request triggering specific event from bitbake server.

---
 lib/bb/command.py |    6 ++++++
 1 files changed, 6 insertions(+), 0 deletions(-)
lib/bb/command.py +6 -0
index 43875f7..a2c464f 100644
@@ -179,6 +179,12 @@ class CommandsSync:
         """
         return bb.utils.cpu_count()
 
+    def triggerEvent(self, command, params):
+        """
+        Trigger a certain event
+        """
+        event = params[0]
+        bb.event.fire(event, command.cooker.configuration.data)
 
 class CommandsAsync:
     """
Signature · 1 line
1.7.0.4
#10 lore raw

PATCH 9/9 runqueue: fire sceneQueueTaskStarted event when a setscene queue starts

In reply to Dongxiao Xu

The current code prints a log when a setscene task starts, therefore the progressbar in hob will not receive it. Use a sceneQueueTaskStarted event instead.

---
 lib/bb/runqueue.py |   10 ++++++++++
 1 files changed, 10 insertions(+), 0 deletions(-)
lib/bb/runqueue.py +10 -0
index 0e83d05..0ba414f 100644
@@ -1619,6 +1619,9 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
                 self.task_skip(task)
                 return True
 
+            startevent = sceneQueueTaskStarted(task, self.stats, self.rq)
+            bb.event.fire(startevent, self.cfgData)
+
             pid, pipein, pipeout = self.fork_off_task(fn, realtask, taskname)
 
             self.build_pids[pid] = task
@@ -1686,6 +1689,13 @@ class runQueueTaskStarted(runQueueEvent):
         runQueueEvent.__init__(self, task, stats, rq)
         self.noexec = noexec
 
+class sceneQueueTaskStarted(runQueueTaskStarted):
+    """
+    Event notifing a setscene task was started
+    """
+    def __init__(self, task, stats, rq, noexec=False):
+        runQueueTaskStarted.__init__(self, task, stats, rq, noexec)
+
 class runQueueTaskFailed(runQueueEvent):
     """
     Event notifing a task failed
Signature · 1 line
1.7.0.4
#1 lore raw

PATCH 1/9 cache: Use configuration's hash value to validate cache

In reply to Dongxiao Xu

Previously we use the file time stamp to judge if a cache is valid. Here this commit introduce a new method, which calculates the total hash value for a certain configuration's key/value paris, and tag it into cache filename, for example, bb_cache.dat.xxxyyyzzz.

This mechanism also ensures the cache's correctness if user dynamically setting variables from some frontend GUI, like HOB.

---
 lib/bb/cache.py      |   32 ++++++++++++--------------------
 lib/bb/cooker.py     |    4 +++-
 lib/bb/data_smart.py |   21 +++++++++++++++++++++
 3 files changed, 36 insertions(+), 21 deletions(-)
lib/bb/cache.py +12 -20
index 6b7fa6f..955b6df 100644
@@ -42,10 +42,10 @@ except ImportError:
     logger.info("Importing cPickle failed. "
                 "Falling back to a very slow implementation.")
 
-__cache_version__ = "142"
+__cache_version__ = "143"
 
-def getCacheFile(path, filename):
-    return os.path.join(path, filename)
+def getCacheFile(path, filename, data_hash):
+    return os.path.join(path, filename + "." + data_hash)
 
 # RecipeInfoCommon defines common data retrieving methods
 # from meta data for caches. CoreRecipeInfo as well as other
@@ -254,7 +254,7 @@ class Cache(object):
     BitBake Cache implementation
     """
 
-    def __init__(self, data, caches_array):
+    def __init__(self, data, data_hash, caches_array):
         # Pass caches_array information into Cache Constructor
         # It will be used in later for deciding whether we 
         # need extra cache file dump/load support 
@@ -266,6 +266,7 @@ class Cache(object):
         self.data = None
         self.data_fn = None
         self.cacheclean = True
+        self.data_hash = data_hash
 
         if self.cachedir in [None, '']:
             self.has_cache = False
@@ -274,26 +275,17 @@ class Cache(object):
             return
 
         self.has_cache = True
-        self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat")
+        self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash)
 
         logger.debug(1, "Using cache in '%s'", self.cachedir)
         bb.utils.mkdirhier(self.cachedir)
 
-        # If any of configuration.data's dependencies are newer than the
-        # cache there isn't even any point in loading it...
-        newest_mtime = 0
-        deps = data.getVar("__base_depends")
-
-        old_mtimes = [old_mtime for _, old_mtime in deps]
-        old_mtimes.append(newest_mtime)
-        newest_mtime = max(old_mtimes)
-
         cache_ok = True
         if self.caches_array:
             for cache_class in self.caches_array:
                 if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                    cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
-                    cache_ok = cache_ok and (bb.parse.cached_mtime_noerror(cachefile) >= newest_mtime)
+                    cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
+                    cache_ok = cache_ok and os.path.exists(cachefile)
                     cache_class.init_cacheData(self)
         if cache_ok:
             self.load_cachefile()
@@ -327,7 +319,7 @@ class Cache(object):
         # Calculate the correct cachesize of all those cache files
         for cache_class in self.caches_array:
             if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
+                cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
                 with open(cachefile, "rb") as cachefile:
                     cachesize += os.fstat(cachefile.fileno()).st_size
 
@@ -335,7 +327,7 @@ class Cache(object):
         
         for cache_class in self.caches_array:
             if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
+                cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
                 with open(cachefile, "rb") as cachefile:
                     pickled = pickle.Unpickler(cachefile)                    
                     while cachefile:
@@ -588,7 +580,7 @@ class Cache(object):
         for cache_class in self.caches_array:
             if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
                 cache_class_name = cache_class.__name__
-                cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
+                cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
                 file_dict[cache_class_name] = open(cachefile, "wb")
                 pickler_dict[cache_class_name] =  pickle.Pickler(file_dict[cache_class_name], pickle.HIGHEST_PROTOCOL)
                    
@@ -693,7 +685,7 @@ def init(cooker):
     Files causing parsing errors are evicted from the cache.
 
     """
-    return Cache(cooker.configuration.data)
+    return Cache(cooker.configuration.data, cooker.configuration.data_hash)
 
 
 class CacheData(object):
lib/bb/cooker.py +3 -1
index 194046e..403aa88 100644
@@ -858,6 +858,7 @@ class BBCooker:
         bb.parse.init_parser(data)
         bb.event.fire(bb.event.ConfigParsed(), data)
         self.configuration.data = data
+        self.configuration.data_hash = data.get_hash()
 
     def handleCollections( self, collections ):
         """Handle collections"""
@@ -1431,6 +1432,7 @@ class CookerParser(object):
         self.filelist = filelist
         self.cooker = cooker
         self.cfgdata = cooker.configuration.data
+        self.cfghash = cooker.configuration.data_hash
 
         # Accounting statistics
         self.parsed = 0
@@ -1446,7 +1448,7 @@ class CookerParser(object):
         self.num_processes = int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or
                                  multiprocessing.cpu_count())
 
-        self.bb_cache = bb.cache.Cache(self.cfgdata, cooker.caches_array)
+        self.bb_cache = bb.cache.Cache(self.cfgdata, self.cfghash, cooker.caches_array)
         self.fromcache = []
         self.willparse = []
         for filename in self.filelist:
lib/bb/data_smart.py +21 -0
index ea13478..9864034 100644
@@ -31,6 +31,7 @@ BitBake build tools.
 import copy, re
 from collections import MutableMapping
 import logging
+import hashlib
 import bb, bb.codeparser
 from bb   import utils
 from bb.COW  import COWDictBase
@@ -459,3 +460,23 @@ class DataSmart(MutableMapping):
 
     def __delitem__(self, var):
         self.delVar(var)
+
+    def get_hash(self):
+        data = ""
+        keys = iter(self)
+        for key in keys:
+            if key == "TIME":
+                continue
+            if key == "__depends":
+                deps = list(self.getVar(key, False))
+                deps.sort()
+                value = [deps[i][0] for i in range(len(deps))]
+            elif key == "PATH":
+                path = list(set(self.getVar(key, False).split(':')))
+                path.sort()
+                value = " ".join(path)
+            else:
+                value = self.getVar(key, False) or ""
+            data = data + key + ': ' + str(value) + '\n'
+
+        return hashlib.md5(data).hexdigest()
Signature · 1 line
1.7.0.4
#3 lore raw

PATCH 2/9 command.py: add resolve option for generateTargetsTree API

In reply to Dongxiao Xu

Currently we have generateTargetsTree API, which is used to get dependency information. However in that tree, there will be "virtual/xxx" in depends fields. Therefore we add the resolve option to replace it with its real providers.

Besides, for packages that provided by multiple recipes, we will find their preverred provider.

---
 lib/bb/command.py |   11 ++++++++-
 lib/bb/cooker.py  |   58 ++++++++++++++++++++++++++++++++++++++++++----------
 2 files changed, 56 insertions(+), 13 deletions(-)
lib/bb/command.py +9 -2
index 2a3a3af..05555c5 100644
@@ -230,14 +230,21 @@ class CommandsAsync:
         included in the package list.
         If pkg_list provided use that list (plus any extras brought in by
         klass) rather than generating a tree for all packages.
+
+        Add a new option "resolve" to indicate if we need to resolve the
+        replacement for "virtual/xxx" like pn.
         """
         klass = params[0]
-        if len(params) > 1:
+        resolve = False
+        if len(params) > 2:
+            pkg_list = params[1]
+            resolve = params[2]
+        elif len(params) > 1:
             pkg_list = params[1]
         else:
             pkg_list = []
 
-        command.cooker.generateTargetsTree(klass, pkg_list)
+        command.cooker.generateTargetsTree(klass, pkg_list, resolve)
         command.finishAsyncCommand()
     generateTargetsTree.needcache = True
 
lib/bb/cooker.py +47 -11

111 lines not on this page · raw message

Signature · 1 line
1.7.0.4
#2 lore raw

Re: [PATCH 1/9] cache: Use configuration's hash value to validate cache

In reply to Dongxiao Xu

Is it possbile to add a "force-reparse" option to bitbake, since some scripts are using the conf file time stamp to retrigger the parsing now?

Best Regards,
Lianhao

Quoted 196 lines

-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Dongxiao Xu
Sent: Friday, January 13, 2012 4:31 PM
To: [email protected]
Subject: [bitbake-devel] [PATCH 1/9] cache: Use configuration's hash value to validate cache

Previously we use the file time stamp to judge if a cache is valid. Here this commit introduce a new method, which calculates the total hash value for a certain configuration's key/value paris, and tag it into cache filename, for example, bb_cache.dat.xxxyyyzzz.

This mechanism also ensures the cache's correctness if user dynamically setting variables from some frontend GUI, like HOB.

---
 lib/bb/cache.py      |   32 ++++++++++++--------------------
 lib/bb/cooker.py     |    4 +++-
 lib/bb/data_smart.py |   21 +++++++++++++++++++++
 3 files changed, 36 insertions(+), 21 deletions(-)
lib/bb/cache.py +12 -20
index 6b7fa6f..955b6df 100644
@@ -42,10 +42,10 @@ except ImportError:
     logger.info("Importing cPickle failed. "
                 "Falling back to a very slow implementation.")
 
-__cache_version__ = "142"
+__cache_version__ = "143"
 
-def getCacheFile(path, filename):
-    return os.path.join(path, filename)
+def getCacheFile(path, filename, data_hash):
+    return os.path.join(path, filename + "." + data_hash)
 
 # RecipeInfoCommon defines common data retrieving methods
 # from meta data for caches. CoreRecipeInfo as well as other
@@ -254,7 +254,7 @@ class Cache(object):
     BitBake Cache implementation
     """
 
-    def __init__(self, data, caches_array):
+    def __init__(self, data, data_hash, caches_array):
         # Pass caches_array information into Cache Constructor
         # It will be used in later for deciding whether we
         # need extra cache file dump/load support
@@ -266,6 +266,7 @@ class Cache(object):
         self.data = None
         self.data_fn = None
         self.cacheclean = True
+        self.data_hash = data_hash
 
         if self.cachedir in [None, '']:
             self.has_cache = False
@@ -274,26 +275,17 @@ class Cache(object):
             return
 
         self.has_cache = True
-        self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat")
+        self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash)
 
         logger.debug(1, "Using cache in '%s'", self.cachedir)
         bb.utils.mkdirhier(self.cachedir)
 
-        # If any of configuration.data's dependencies are newer than the
-        # cache there isn't even any point in loading it...
-        newest_mtime = 0
-        deps = data.getVar("__base_depends")
-
-        old_mtimes = [old_mtime for _, old_mtime in deps]
-        old_mtimes.append(newest_mtime)
-        newest_mtime = max(old_mtimes)
-
         cache_ok = True
         if self.caches_array:
             for cache_class in self.caches_array:
                 if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                    cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
-                    cache_ok = cache_ok and (bb.parse.cached_mtime_noerror(cachefile) >= newest_mtime)
+                    cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
+                    cache_ok = cache_ok and os.path.exists(cachefile)
                     cache_class.init_cacheData(self)
         if cache_ok:
             self.load_cachefile()
@@ -327,7 +319,7 @@ class Cache(object):
         # Calculate the correct cachesize of all those cache files
         for cache_class in self.caches_array:
             if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
+                cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
                 with open(cachefile, "rb") as cachefile:
                     cachesize += os.fstat(cachefile.fileno()).st_size
 
@@ -335,7 +327,7 @@ class Cache(object):
 
         for cache_class in self.caches_array:
             if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
+                cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
                 with open(cachefile, "rb") as cachefile:
                     pickled = pickle.Unpickler(cachefile)
                     while cachefile:
@@ -588,7 +580,7 @@ class Cache(object):
         for cache_class in self.caches_array:
             if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
                 cache_class_name = cache_class.__name__
-                cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
+                cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
                 file_dict[cache_class_name] = open(cachefile, "wb")
                 pickler_dict[cache_class_name] =  pickle.Pickler(file_dict[cache_class_name], pickle.HIGHEST_PROTOCOL)
 
@@ -693,7 +685,7 @@ def init(cooker):
     Files causing parsing errors are evicted from the cache.
 
     """
-    return Cache(cooker.configuration.data)
+    return Cache(cooker.configuration.data, cooker.configuration.data_hash)
 
 
 class CacheData(object):
lib/bb/cooker.py +3 -1
index 194046e..403aa88 100644
@@ -858,6 +858,7 @@ class BBCooker:
         bb.parse.init_parser(data)
         bb.event.fire(bb.event.ConfigParsed(), data)
         self.configuration.data = data
+        self.configuration.data_hash = data.get_hash()
 
     def handleCollections( self, collections ):
         """Handle collections"""
@@ -1431,6 +1432,7 @@ class CookerParser(object):
         self.filelist = filelist
         self.cooker = cooker
         self.cfgdata = cooker.configuration.data
+        self.cfghash = cooker.configuration.data_hash
 
         # Accounting statistics
         self.parsed = 0
@@ -1446,7 +1448,7 @@ class CookerParser(object):
         self.num_processes = int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or
                                  multiprocessing.cpu_count())
 
-        self.bb_cache = bb.cache.Cache(self.cfgdata, cooker.caches_array)
+        self.bb_cache = bb.cache.Cache(self.cfgdata, self.cfghash, cooker.caches_array)
         self.fromcache = []
         self.willparse = []
         for filename in self.filelist:
lib/bb/data_smart.py +21 -1
index ea13478..9864034 100644
@@ -31,6 +31,7 @@ BitBake build tools.
 import copy, re
 from collections import MutableMapping
 import logging
+import hashlib
 import bb, bb.codeparser
 from bb   import utils
 from bb.COW  import COWDictBase
@@ -459,3 +460,23 @@ class DataSmart(MutableMapping):
 
     def __delitem__(self, var):
         self.delVar(var)
+
+    def get_hash(self):
+        data = ""
+        keys = iter(self)
+        for key in keys:
+            if key == "TIME":
+                continue
+            if key == "__depends":
+                deps = list(self.getVar(key, False))
+                deps.sort()
+                value = [deps[i][0] for i in range(len(deps))]
+            elif key == "PATH":
+                path = list(set(self.getVar(key, False).split(':')))
+                path.sort()
+                value = " ".join(path)
+            else:
+                value = self.getVar(key, False) or ""
+            data = data + key + ': ' + str(value) + '\n'
+
+        return hashlib.md5(data).hexdigest()
--

1.7.0.4

_______________________________________________
bitbake-devel mailing list
[email protected]
http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/bitbake-devel