Hob: bitbake related changes
Hi Richard,
This is the third pull request for Hob bitbake related changes.
Please help to review and pull.
Changes from v2:
1. Fix parameter checking issue on --bind and --server-only.
2. Add a new command triggerEvent(), which enables the client to request triggering specific event from bitbake server.
3. Add a specific event 'RequestPackageInfo' to get available package information.
4. Change the way to get the CPU information on server side.
Thanks,
Dongxiao
The following changes since commit 9d8f45407c67ed0d3c4f820cf646de3c385067c7:
Lianhao Lu (1):
bitbake: Automatically start local PR service.
are available in the git repository at:
git://git.pokylinux.org/poky-contrib dxu4/hob-bitbake-changes-v3
http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=dxu4/hob-bitbake-changes-v3
Dongxiao Xu (7):
cache: Use configuration's hash value to validate cache command.py: add resolve option for generateTargetsTree API event.py: Add new events RequestPackageInfo and PackageInfo
bitbake: add -B option to bind with interface
bitbake: Add client socket info for BitBakeServerConnection command.py: Add a new API triggerEvent()
runqueue: fire sceneQueueTaskStarted event when a setscene queue
starts
Shane Wang (2):
bitbake: change for adding progress bar in Hob2. command.py: add new API to get the cpu count on the server
bin/bitbake | 17 +++++++- lib/bb/cache.py | 34 ++++++---------- lib/bb/command.py | 23 ++++++++++- lib/bb/cooker.py | 74 ++++++++++++++++++++++++++++-------- lib/bb/data_smart.py | 21 ++++++++++ lib/bb/event.py | 96 +++++++++++++++++++++++++++++++++++----------- lib/bb/runqueue.py | 10 +++++ lib/bb/server/xmlrpc.py | 10 ++-- lib/bb/ui/knotty.py | 5 ++- lib/bb/ui/uievent.py | 7 ++- lib/bb/utils.py | 4 ++ 11 files changed, 227 insertions(+), 74 deletions(-)
PATCH 3/9 event.py: Add new events RequestPackageInfo and PackageInfo
RequestPackageInfo is triggered by GUI client to request the available package information.
PackageInfo event is to pass package information back to GUI.
--- lib/bb/event.py | 13 +++++++++++++ 1 files changed, 13 insertions(+), 0 deletions(-)
lib/bb/event.py +13 -0
@@ -467,3 +467,16 @@ class LogHandler(logging.Handler): def filter(self, record): record.taskpid = worker_pid return True + +class RequestPackageInfo(Event): + """ + Event to request package information + """ + +class PackageInfo(Event): + """ + Package information for GUI + """ + def __init__(self, pkginfolist): + Event.__init__(self) + self._pkginfolist = pkginfolist
Signature · 1 line
1.7.0.4
PATCH 4/9 bitbake: change for adding progress bar in Hob2.
In reply to Dongxiao XuFrom: Shane Wang <[email protected]>
The changes include:
- Clean some events in event.py
- Fire essential events for Hob2 to handle with more information.
- knotty changes
--- lib/bb/cache.py | 2 +- lib/bb/cooker.py | 12 +++++-- lib/bb/event.py | 83 ++++++++++++++++++++++++++++++++++++-------------- lib/bb/ui/knotty.py | 5 ++- 4 files changed, 73 insertions(+), 29 deletions(-)
lib/bb/cache.py +1 -1
@@ -345,7 +345,7 @@ class Cache(object): current_percent = 100 * current_progress / cachesize if current_percent > previous_percent: previous_percent = current_percent - bb.event.fire(bb.event.CacheLoadProgress(current_progress), + bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize), self.data) previous_progress += current_progress
lib/bb/cooker.py +8 -4
@@ -345,6 +345,7 @@ class BBCooker: """ Prepare a runqueue and taskdata object for iteration over pkgs_to_build """ + bb.event.fire(bb.event.TreeDataPreparationStarted(), self.configuration.data) # Need files parsed self.updateCache() # If we are told to do the None task then query the default task
@@ -361,11 +362,14 @@ class BBCooker: taskdata = bb.taskdata.TaskData(False, skiplist=self.skiplist) runlist = [] + current = 0 for k in pkgs_to_build: taskdata.add_provider(localdata, self.status, k) runlist.append([k, "do_%s" % task]) + current += 1 + bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(pkgs_to_build)), self.configuration.data) taskdata.add_unresolved(localdata, self.status) - + bb.event.fire(bb.event.TreeDataPreparationCompleted(len(pkgs_to_build)), self.configuration.data) return runlist, taskdata def generateTaskDepTreeData(self, pkgs_to_build, task):
@@ -1106,7 +1110,7 @@ class BBCooker: return False if not retval: - bb.event.fire(bb.event.BuildCompleted(buildname, item, failures), self.configuration.event_data) + bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, item, failures), self.configuration.event_data) self.command.finishAsyncCommand() return False if retval is True:
@@ -1147,7 +1151,7 @@ class BBCooker: return False if not retval: - bb.event.fire(bb.event.BuildCompleted(buildname, targets, failures), self.configuration.data) + bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, targets, failures), self.configuration.data) self.command.finishAsyncCommand() return False if retval is True:
@@ -1565,7 +1569,7 @@ class CookerParser(object): if parsed: self.parsed += 1 if self.parsed % self.progress_chunk == 0: - bb.event.fire(bb.event.ParseProgress(self.parsed), + bb.event.fire(bb.event.ParseProgress(self.parsed, self.toparse), self.cfgdata) else: self.cached += 1
lib/bb/event.py +60 -23
130 lines not on this page · raw message
lib/bb/ui/knotty.py +4 -1
@@ -251,7 +251,10 @@ def main(server, eventHandler): bb.event.RecipeParsed, bb.event.RecipePreFinalise, bb.runqueue.runQueueEvent, - bb.runqueue.runQueueExitWait)): + bb.runqueue.runQueueExitWait, + bb.event.OperationStarted, + bb.event.OperationCompleted, + bb.event.OperationProgress)): continue logger.error("Unknown event: %s", event)
Signature · 1 line
1.7.0.4
PATCH 5/9 bitbake: add -B option to bind with interface
In reply to Dongxiao XuWhen start bitbake as a server only process, we need to assign certain interface to it.
--- bin/bitbake | 17 ++++++++++++++--- lib/bb/server/xmlrpc.py | 6 +++--- 2 files changed, 17 insertions(+), 6 deletions(-)
bin/bitbake +14 -3
@@ -168,6 +168,8 @@ Default BBFILES are the .bb files in the current directory.""") parser.add_option("", "--server-only", help = "Run bitbake without UI, the frontend can connect with bitbake server itself", action = "store_true", dest = "server_only", default = False) + parser.add_option("-B", "--bind", help = "The name/address for the bitbake server to bind to", + action = "store", dest = "bind", default = False) options, args = parser.parse_args(sys.argv) configuration = BBConfiguration(options)
@@ -189,8 +191,14 @@ Default BBFILES are the .bb files in the current directory.""") sys.exit("FATAL: Invalid server type '%s' specified.\n" "Valid interfaces: xmlrpc, process [default], none." % servertype) - if configuration.server_only and configuration.servertype != "xmlrpc": - sys.exit("FATAL: If '--server-only' is defined, we must set the servertype as 'xmlrpc'.\n") + if configuration.server_only: + if configuration.servertype != "xmlrpc": + sys.exit("FATAL: If '--server-only' is defined, we must set the servertype as 'xmlrpc'.\n") + if not configuration.bind: + sys.exit("FATAL: The '--server-only' option requires a name/address to bind to with the -B option.\n") + + if configuration.bind and configuration.servertype != "xmlrpc": + sys.exit("FATAL: If '-B' or '--bind' is defined, we must set the servertype as 'xmlrpc'.\n") # Save a logfile for cooker into the current working directory. When the # server is daemonized this logfile will be truncated.
@@ -212,8 +220,11 @@ Default BBFILES are the .bb files in the current directory.""") bb.utils.clean_environment() server = server.BitBakeServer() + if configuration.bind: + server.initServer((configuration.bind, 0)) + else: + server.initServer() - server.initServer() idle = server.getServerIdleCB() cooker = bb.cooker.BBCooker(configuration, idle, initialenv)
lib/bb/server/xmlrpc.py +3 -3
@@ -163,7 +163,7 @@ class BitBakeXMLRPCServer(SimpleXMLRPCServer): # remove this when you're done with debugging # allow_reuse_address = True - def __init__(self, interface = ("localhost", 0)): + def __init__(self, interface): """ Constructor """
@@ -267,8 +267,8 @@ class BitBakeServerConnection(): pass class BitBakeServer(object): - def initServer(self): - self.server = BitBakeXMLRPCServer() + def initServer(self, interface = ("localhost", 0)): + self.server = BitBakeXMLRPCServer(interface) def addcooker(self, cooker): self.cooker = cooker
Signature · 1 line
1.7.0.4
PATCH 7/9 command.py: add new API to get the cpu count on the server
In reply to Dongxiao XuFrom: Shane Wang <[email protected]>
Add a new API in command.py to get the cpu count in order to set the appropriate default BB_NUMBER_THREADS and PARALLEL_MAKE variables.
--- lib/bb/command.py | 6 ++++++ lib/bb/utils.py | 4 ++++ 2 files changed, 10 insertions(+), 0 deletions(-)
lib/bb/command.py +6 -0
@@ -173,6 +173,12 @@ class CommandsSync: """ command.cooker.reset() + def getCpuCount(self, command, params): + """ + Get the CPU count on the bitbake server + """ + return bb.utils.cpu_count() + class CommandsAsync: """
lib/bb/utils.py +4 -0
@@ -25,6 +25,7 @@ import errno import logging import bb import bb.msg +import multiprocessing from commands import getstatusoutput from contextlib import contextmanager
@@ -862,3 +863,6 @@ def contains(variable, checkvalues, truevalue, falsevalue, d): if checkvalues.issubset(val): return truevalue return falsevalue + +def cpu_count(): + return multiprocessing.cpu_count()
Signature · 1 line
1.7.0.4
PATCH 6/9 bitbake: Add client socket info for BitBakeServerConnection
In reply to Dongxiao XuIn server/client split model, the client will bind to a specific address and port. We need to pass the values to BitBakeServerConnection().
--- lib/bb/server/xmlrpc.py | 4 ++-- lib/bb/ui/uievent.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-)
lib/bb/server/xmlrpc.py +2 -2
@@ -247,9 +247,9 @@ class BitbakeServerInfo(): self.port = port class BitBakeServerConnection(): - def __init__(self, serverinfo): + def __init__(self, serverinfo, clientinfo=("localhost", 0)): self.connection = _create_server(serverinfo.host, serverinfo.port) - self.events = uievent.BBUIEventQueue(self.connection) + self.events = uievent.BBUIEventQueue(self.connection, clientinfo) for event in bb.event.ui_queue: self.events.queue_event(event)
lib/bb/ui/uievent.py +4 -3
@@ -28,13 +28,14 @@ import socket, threading, pickle from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler class BBUIEventQueue: - def __init__(self, BBServer): + def __init__(self, BBServer, clientinfo=("localhost, 0")): self.eventQueue = [] self.eventQueueLock = threading.Lock() self.eventQueueNotify = threading.Event() self.BBServer = BBServer + self.clientinfo = clientinfo self.t = threading.Thread() self.t.setDaemon(True)
@@ -72,7 +73,7 @@ class BBUIEventQueue: def startCallbackHandler(self): - server = UIXMLRPCServer() + server = UIXMLRPCServer(self.clientinfo) self.host, self.port = server.socket.getsockname() server.register_function( self.system_quit, "event.quit" )
@@ -98,7 +99,7 @@ class BBUIEventQueue: class UIXMLRPCServer (SimpleXMLRPCServer): - def __init__( self, interface = ("localhost", 0) ): + def __init__( self, interface ): self.quit = False SimpleXMLRPCServer.__init__( self, interface,
Signature · 1 line
1.7.0.4
PATCH 8/9 command.py: Add a new API triggerEvent()
In reply to Dongxiao XuThis functions enables the client to request triggering specific event from bitbake server.
--- lib/bb/command.py | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-)
lib/bb/command.py +6 -0
@@ -179,6 +179,12 @@ class CommandsSync: """ return bb.utils.cpu_count() + def triggerEvent(self, command, params): + """ + Trigger a certain event + """ + event = params[0] + bb.event.fire(event, command.cooker.configuration.data) class CommandsAsync: """
Signature · 1 line
1.7.0.4
PATCH 9/9 runqueue: fire sceneQueueTaskStarted event when a setscene queue starts
In reply to Dongxiao XuThe current code prints a log when a setscene task starts, therefore the progressbar in hob will not receive it. Use a sceneQueueTaskStarted event instead.
--- lib/bb/runqueue.py | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-)
lib/bb/runqueue.py +10 -0
@@ -1619,6 +1619,9 @@ class RunQueueExecuteScenequeue(RunQueueExecute): self.task_skip(task) return True + startevent = sceneQueueTaskStarted(task, self.stats, self.rq) + bb.event.fire(startevent, self.cfgData) + pid, pipein, pipeout = self.fork_off_task(fn, realtask, taskname) self.build_pids[pid] = task
@@ -1686,6 +1689,13 @@ class runQueueTaskStarted(runQueueEvent): runQueueEvent.__init__(self, task, stats, rq) self.noexec = noexec +class sceneQueueTaskStarted(runQueueTaskStarted): + """ + Event notifing a setscene task was started + """ + def __init__(self, task, stats, rq, noexec=False): + runQueueTaskStarted.__init__(self, task, stats, rq, noexec) + class runQueueTaskFailed(runQueueEvent): """ Event notifing a task failed
Signature · 1 line
1.7.0.4
PATCH 1/9 cache: Use configuration's hash value to validate cache
In reply to Dongxiao XuPreviously we use the file time stamp to judge if a cache is valid. Here this commit introduce a new method, which calculates the total hash value for a certain configuration's key/value paris, and tag it into cache filename, for example, bb_cache.dat.xxxyyyzzz.
This mechanism also ensures the cache's correctness if user dynamically setting variables from some frontend GUI, like HOB.
--- lib/bb/cache.py | 32 ++++++++++++-------------------- lib/bb/cooker.py | 4 +++- lib/bb/data_smart.py | 21 +++++++++++++++++++++ 3 files changed, 36 insertions(+), 21 deletions(-)
lib/bb/cache.py +12 -20
@@ -42,10 +42,10 @@ except ImportError: logger.info("Importing cPickle failed. " "Falling back to a very slow implementation.") -__cache_version__ = "142" +__cache_version__ = "143" -def getCacheFile(path, filename): - return os.path.join(path, filename) +def getCacheFile(path, filename, data_hash): + return os.path.join(path, filename + "." + data_hash) # RecipeInfoCommon defines common data retrieving methods # from meta data for caches. CoreRecipeInfo as well as other
@@ -254,7 +254,7 @@ class Cache(object): BitBake Cache implementation """ - def __init__(self, data, caches_array): + def __init__(self, data, data_hash, caches_array): # Pass caches_array information into Cache Constructor # It will be used in later for deciding whether we # need extra cache file dump/load support
@@ -266,6 +266,7 @@ class Cache(object): self.data = None self.data_fn = None self.cacheclean = True + self.data_hash = data_hash if self.cachedir in [None, '']: self.has_cache = False
@@ -274,26 +275,17 @@ class Cache(object): return self.has_cache = True - self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat") + self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash) logger.debug(1, "Using cache in '%s'", self.cachedir) bb.utils.mkdirhier(self.cachedir) - # If any of configuration.data's dependencies are newer than the - # cache there isn't even any point in loading it... - newest_mtime = 0 - deps = data.getVar("__base_depends") - - old_mtimes = [old_mtime for _, old_mtime in deps] - old_mtimes.append(newest_mtime) - newest_mtime = max(old_mtimes) - cache_ok = True if self.caches_array: for cache_class in self.caches_array: if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): - cachefile = getCacheFile(self.cachedir, cache_class.cachefile) - cache_ok = cache_ok and (bb.parse.cached_mtime_noerror(cachefile) >= newest_mtime) + cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + cache_ok = cache_ok and os.path.exists(cachefile) cache_class.init_cacheData(self) if cache_ok: self.load_cachefile()
@@ -327,7 +319,7 @@ class Cache(object): # Calculate the correct cachesize of all those cache files for cache_class in self.caches_array: if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): - cachefile = getCacheFile(self.cachedir, cache_class.cachefile) + cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) with open(cachefile, "rb") as cachefile: cachesize += os.fstat(cachefile.fileno()).st_size
@@ -335,7 +327,7 @@ class Cache(object): for cache_class in self.caches_array: if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): - cachefile = getCacheFile(self.cachedir, cache_class.cachefile) + cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) with open(cachefile, "rb") as cachefile: pickled = pickle.Unpickler(cachefile) while cachefile:
@@ -588,7 +580,7 @@ class Cache(object): for cache_class in self.caches_array: if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): cache_class_name = cache_class.__name__ - cachefile = getCacheFile(self.cachedir, cache_class.cachefile) + cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) file_dict[cache_class_name] = open(cachefile, "wb") pickler_dict[cache_class_name] = pickle.Pickler(file_dict[cache_class_name], pickle.HIGHEST_PROTOCOL)
@@ -693,7 +685,7 @@ def init(cooker): Files causing parsing errors are evicted from the cache. """ - return Cache(cooker.configuration.data) + return Cache(cooker.configuration.data, cooker.configuration.data_hash) class CacheData(object):
lib/bb/cooker.py +3 -1
@@ -858,6 +858,7 @@ class BBCooker: bb.parse.init_parser(data) bb.event.fire(bb.event.ConfigParsed(), data) self.configuration.data = data + self.configuration.data_hash = data.get_hash() def handleCollections( self, collections ): """Handle collections"""
@@ -1431,6 +1432,7 @@ class CookerParser(object): self.filelist = filelist self.cooker = cooker self.cfgdata = cooker.configuration.data + self.cfghash = cooker.configuration.data_hash # Accounting statistics self.parsed = 0
@@ -1446,7 +1448,7 @@ class CookerParser(object): self.num_processes = int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or multiprocessing.cpu_count()) - self.bb_cache = bb.cache.Cache(self.cfgdata, cooker.caches_array) + self.bb_cache = bb.cache.Cache(self.cfgdata, self.cfghash, cooker.caches_array) self.fromcache = [] self.willparse = [] for filename in self.filelist:
lib/bb/data_smart.py +21 -0
@@ -31,6 +31,7 @@ BitBake build tools. import copy, re from collections import MutableMapping import logging +import hashlib import bb, bb.codeparser from bb import utils from bb.COW import COWDictBase
@@ -459,3 +460,23 @@ class DataSmart(MutableMapping): def __delitem__(self, var): self.delVar(var) + + def get_hash(self): + data = "" + keys = iter(self) + for key in keys: + if key == "TIME": + continue + if key == "__depends": + deps = list(self.getVar(key, False)) + deps.sort() + value = [deps[i][0] for i in range(len(deps))] + elif key == "PATH": + path = list(set(self.getVar(key, False).split(':'))) + path.sort() + value = " ".join(path) + else: + value = self.getVar(key, False) or "" + data = data + key + ': ' + str(value) + '\n' + + return hashlib.md5(data).hexdigest()
Signature · 1 line
1.7.0.4
PATCH 2/9 command.py: add resolve option for generateTargetsTree API
In reply to Dongxiao XuCurrently we have generateTargetsTree API, which is used to get dependency information. However in that tree, there will be "virtual/xxx" in depends fields. Therefore we add the resolve option to replace it with its real providers.
Besides, for packages that provided by multiple recipes, we will find their preverred provider.
--- lib/bb/command.py | 11 ++++++++- lib/bb/cooker.py | 58 ++++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 56 insertions(+), 13 deletions(-)
lib/bb/command.py +9 -2
@@ -230,14 +230,21 @@ class CommandsAsync: included in the package list. If pkg_list provided use that list (plus any extras brought in by klass) rather than generating a tree for all packages. + + Add a new option "resolve" to indicate if we need to resolve the + replacement for "virtual/xxx" like pn. """ klass = params[0] - if len(params) > 1: + resolve = False + if len(params) > 2: + pkg_list = params[1] + resolve = params[2] + elif len(params) > 1: pkg_list = params[1] else: pkg_list = [] - command.cooker.generateTargetsTree(klass, pkg_list) + command.cooker.generateTargetsTree(klass, pkg_list, resolve) command.finishAsyncCommand() generateTargetsTree.needcache = True
lib/bb/cooker.py +47 -11
111 lines not on this page · raw message
Signature · 1 line
1.7.0.4
Re: [PATCH 1/9] cache: Use configuration's hash value to validate cache
In reply to Dongxiao XuIs it possbile to add a "force-reparse" option to bitbake, since some scripts are using the conf file time stamp to retrigger the parsing now?
Best Regards,
Lianhao
Quoted 196 lines
-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Dongxiao Xu
Sent: Friday, January 13, 2012 4:31 PM
To: [email protected]
Subject: [bitbake-devel] [PATCH 1/9] cache: Use configuration's hash value to validate cachePreviously we use the file time stamp to judge if a cache is valid. Here this commit introduce a new method, which calculates the total hash value for a certain configuration's key/value paris, and tag it into cache filename, for example, bb_cache.dat.xxxyyyzzz.
This mechanism also ensures the cache's correctness if user dynamically setting variables from some frontend GUI, like HOB.
Signed-off-by Dongxiao Xu <[email protected]>--- lib/bb/cache.py | 32 ++++++++++++-------------------- lib/bb/cooker.py | 4 +++- lib/bb/data_smart.py | 21 +++++++++++++++++++++ 3 files changed, 36 insertions(+), 21 deletions(-)lib/bb/cache.py +12
-20@@ -42,10 +42,10 @@ except ImportError: logger.info("Importing cPickle failed. " "Falling back to a very slow implementation.") -__cache_version__ = "142" +__cache_version__ = "143" -def getCacheFile(path, filename): - return os.path.join(path, filename) +def getCacheFile(path, filename, data_hash): + return os.path.join(path, filename + "." + data_hash) # RecipeInfoCommon defines common data retrieving methods # from meta data for caches. CoreRecipeInfo as well as other@@ -254,7 +254,7 @@ class Cache(object): BitBake Cache implementation """ - def __init__(self, data, caches_array): + def __init__(self, data, data_hash, caches_array): # Pass caches_array information into Cache Constructor # It will be used in later for deciding whether we # need extra cache file dump/load support@@ -266,6 +266,7 @@ class Cache(object): self.data = None self.data_fn = None self.cacheclean = True + self.data_hash = data_hash if self.cachedir in [None, '']: self.has_cache = False@@ -274,26 +275,17 @@ class Cache(object): return self.has_cache = True - self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat") + self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash) logger.debug(1, "Using cache in '%s'", self.cachedir) bb.utils.mkdirhier(self.cachedir) - # If any of configuration.data's dependencies are newer than the - # cache there isn't even any point in loading it... - newest_mtime = 0 - deps = data.getVar("__base_depends") - - old_mtimes = [old_mtime for _, old_mtime in deps] - old_mtimes.append(newest_mtime) - newest_mtime = max(old_mtimes) - cache_ok = True if self.caches_array: for cache_class in self.caches_array: if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): - cachefile = getCacheFile(self.cachedir, cache_class.cachefile) - cache_ok = cache_ok and (bb.parse.cached_mtime_noerror(cachefile) >= newest_mtime) + cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + cache_ok = cache_ok and os.path.exists(cachefile) cache_class.init_cacheData(self) if cache_ok: self.load_cachefile()@@ -327,7 +319,7 @@ class Cache(object): # Calculate the correct cachesize of all those cache files for cache_class in self.caches_array: if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): - cachefile = getCacheFile(self.cachedir, cache_class.cachefile) + cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) with open(cachefile, "rb") as cachefile: cachesize += os.fstat(cachefile.fileno()).st_size@@ -335,7 +327,7 @@ class Cache(object): for cache_class in self.caches_array: if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): - cachefile = getCacheFile(self.cachedir, cache_class.cachefile) + cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) with open(cachefile, "rb") as cachefile: pickled = pickle.Unpickler(cachefile) while cachefile:@@ -588,7 +580,7 @@ class Cache(object): for cache_class in self.caches_array: if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): cache_class_name = cache_class.__name__ - cachefile = getCacheFile(self.cachedir, cache_class.cachefile) + cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) file_dict[cache_class_name] = open(cachefile, "wb") pickler_dict[cache_class_name] = pickle.Pickler(file_dict[cache_class_name], pickle.HIGHEST_PROTOCOL)@@ -693,7 +685,7 @@ def init(cooker): Files causing parsing errors are evicted from the cache. """ - return Cache(cooker.configuration.data) + return Cache(cooker.configuration.data, cooker.configuration.data_hash) class CacheData(object):lib/bb/cooker.py +3
-1@@ -858,6 +858,7 @@ class BBCooker: bb.parse.init_parser(data) bb.event.fire(bb.event.ConfigParsed(), data) self.configuration.data = data + self.configuration.data_hash = data.get_hash() def handleCollections( self, collections ): """Handle collections"""@@ -1431,6 +1432,7 @@ class CookerParser(object): self.filelist = filelist self.cooker = cooker self.cfgdata = cooker.configuration.data + self.cfghash = cooker.configuration.data_hash # Accounting statistics self.parsed = 0@@ -1446,7 +1448,7 @@ class CookerParser(object): self.num_processes = int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or multiprocessing.cpu_count()) - self.bb_cache = bb.cache.Cache(self.cfgdata, cooker.caches_array) + self.bb_cache = bb.cache.Cache(self.cfgdata, self.cfghash, cooker.caches_array) self.fromcache = [] self.willparse = [] for filename in self.filelist:lib/bb/data_smart.py +21
-1@@ -31,6 +31,7 @@ BitBake build tools. import copy, re from collections import MutableMapping import logging +import hashlib import bb, bb.codeparser from bb import utils from bb.COW import COWDictBase@@ -459,3 +460,23 @@ class DataSmart(MutableMapping): def __delitem__(self, var): self.delVar(var) + + def get_hash(self): + data = "" + keys = iter(self) + for key in keys: + if key == "TIME": + continue + if key == "__depends": + deps = list(self.getVar(key, False)) + deps.sort() + value = [deps[i][0] for i in range(len(deps))] + elif key == "PATH": + path = list(set(self.getVar(key, False).split(':'))) + path.sort() + value = " ".join(path) + else: + value = self.getVar(key, False) or "" + data = data + key + ': ' + str(value) + '\n' + + return hashlib.md5(data).hexdigest() --1.7.0.4
_______________________________________________
bitbake-devel mailing list
[email protected]
http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/bitbake-devel