Hob2 related bitbake changes
Hi Richard,
This pull request introduces several hob2 related bitbake changes. Please help to review and pull.
Thanks,
Dongxiao
The following changes since commit 71b53a3f0766ca464560a1f6a449f9424fbdf7ae:
Matthew McClintock (1):
siggen.py: If both sigs have a variable in it's whitelist then don't say it's changed
are available in the git repository at:
git://git.pokylinux.org/poky-contrib dxu4/hob2-bitbake-change
http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=dxu4/hob2-bitbake-change
Dongxiao Xu (10):
command.py: Modify needcache value for certain functions
cache: Use configuration's hash value to validate cache
cooker: user bb.configuration.data to inject events command.py: add initCooker API
command.py: add parseConfigurationFiles API command.py: add generateTargetsTreePro API event.py: Add a new event PackageInfo
xmlrpc: Change BitbakeServerInfo init function
cooker: remove command import in cooker.py
bitbake: add a new option "--server-only"
bin/bitbake | 23 ++++++--- lib/bb/cache.py | 32 +++++-------- lib/bb/command.py | 34 ++++++++++++- lib/bb/cooker.py | 123 +++++++++++++++++++++++++++++++++++++++++++++-- lib/bb/data_smart.py | 17 +++++++ lib/bb/event.py | 9 ++++ lib/bb/server/xmlrpc.py | 8 ++-- 7 files changed, 208 insertions(+), 38 deletions(-)
PATCH 1/10 command.py: Modify needcache value for certain functions
for findConfigFiels() and findFilesMatchingInDir() functions, they don't need to parse all the bb files, thus setting the needcache value to be False.
--- lib/bb/command.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
lib/bb/command.py +2 -2
@@ -241,7 +241,7 @@ class CommandsAsync: command.cooker.findConfigFiles(varname) command.finishAsyncCommand() - findConfigFiles.needcache = True + findConfigFiles.needcache = False def findFilesMatchingInDir(self, command, params): """
@@ -253,7 +253,7 @@ class CommandsAsync: command.cooker.findFilesMatchingInDir(pattern, directory) command.finishAsyncCommand() - findFilesMatchingInDir.needcache = True + findFilesMatchingInDir.needcache = False def findConfigFilePath(self, command, params): """
Signature · 1 line
1.7.0.4
Re: [PATCH 01/10] command.py: Modify needcache value for certain functions
Quoted 5 lines · Dongxiao Xu
for
findConfigFiels()andfindFilesMatchingInDir()functions, they don't need to parse all the bb files, thus setting the needcache value to be False.Signed-off-by Dongxiao Xu <[email protected]>
If we can't rely on event_data to be consistent/up-to-date do we need to keep it around? The comment around its creation implies we want/need the copy of data.
This feels like we're working around on issue rather than fixing one.
Joshua
Quoted 26 lines
--- lib/bb/command.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)lib/bb/command.py +2
-2@@ -241,7 +241,7 @@ class CommandsAsync: command.cooker.findConfigFiles(varname) command.finishAsyncCommand() - findConfigFiles.needcache = True + findConfigFiles.needcache = False def findFilesMatchingInDir(self, command, params): """@@ -253,7 +253,7 @@ class CommandsAsync: command.cooker.findFilesMatchingInDir(pattern, directory) command.finishAsyncCommand() - findFilesMatchingInDir.needcache = True + findFilesMatchingInDir.needcache = False def findConfigFilePath(self, command, params): """
Signature · 3 lines
Joshua Lock
Yocto Project "Johannes factotum"
Intel Open Source Technology CentrePATCH 2/10 cache: Use configuration's hash value to validate cache
In reply to Dongxiao XuPreviously we use the file time stamp to judge if a cache is valid. Here this commit introduce a new method, which calculates the total hash value for a certain configuration's key/value paris, and tag it into cache filename, for example, bb_cache.dat.xxxyyyzzz.
This mechanism also ensures the cache's correctness if user dynamically setting variables from some frontend GUI, like HOB.
--- lib/bb/cache.py | 32 ++++++++++++-------------------- lib/bb/cooker.py | 4 +++- lib/bb/data_smart.py | 17 +++++++++++++++++ 3 files changed, 32 insertions(+), 21 deletions(-)
lib/bb/cache.py +12 -20
@@ -42,10 +42,10 @@ except ImportError: logger.info("Importing cPickle failed. " "Falling back to a very slow implementation.") -__cache_version__ = "142" +__cache_version__ = "143" -def getCacheFile(path, filename): - return os.path.join(path, filename) +def getCacheFile(path, filename, data_hash): + return os.path.join(path, filename + "." + data_hash) # RecipeInfoCommon defines common data retrieving methods # from meta data for caches. CoreRecipeInfo as well as other
@@ -254,7 +254,7 @@ class Cache(object): BitBake Cache implementation """ - def __init__(self, data, caches_array): + def __init__(self, data, data_hash, caches_array): # Pass caches_array information into Cache Constructor # It will be used in later for deciding whether we # need extra cache file dump/load support
@@ -266,6 +266,7 @@ class Cache(object): self.data = None self.data_fn = None self.cacheclean = True + self.data_hash = data_hash if self.cachedir in [None, '']: self.has_cache = False
@@ -274,26 +275,17 @@ class Cache(object): return self.has_cache = True - self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat") + self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash) logger.debug(1, "Using cache in '%s'", self.cachedir) bb.utils.mkdirhier(self.cachedir) - # If any of configuration.data's dependencies are newer than the - # cache there isn't even any point in loading it... - newest_mtime = 0 - deps = data.getVar("__base_depends") - - old_mtimes = [old_mtime for _, old_mtime in deps] - old_mtimes.append(newest_mtime) - newest_mtime = max(old_mtimes) - cache_ok = True if self.caches_array: for cache_class in self.caches_array: if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): - cachefile = getCacheFile(self.cachedir, cache_class.cachefile) - cache_ok = cache_ok and (bb.parse.cached_mtime_noerror(cachefile) >= newest_mtime) + cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + cache_ok = cache_ok and os.path.exists(cachefile) cache_class.init_cacheData(self) if cache_ok: self.load_cachefile()
@@ -327,7 +319,7 @@ class Cache(object): # Calculate the correct cachesize of all those cache files for cache_class in self.caches_array: if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): - cachefile = getCacheFile(self.cachedir, cache_class.cachefile) + cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) with open(cachefile, "rb") as cachefile: cachesize += os.fstat(cachefile.fileno()).st_size
@@ -335,7 +327,7 @@ class Cache(object): for cache_class in self.caches_array: if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): - cachefile = getCacheFile(self.cachedir, cache_class.cachefile) + cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) with open(cachefile, "rb") as cachefile: pickled = pickle.Unpickler(cachefile) while cachefile:
@@ -588,7 +580,7 @@ class Cache(object): for cache_class in self.caches_array: if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon): cache_class_name = cache_class.__name__ - cachefile = getCacheFile(self.cachedir, cache_class.cachefile) + cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) file_dict[cache_class_name] = open(cachefile, "wb") pickler_dict[cache_class_name] = pickle.Pickler(file_dict[cache_class_name], pickle.HIGHEST_PROTOCOL)
@@ -693,7 +685,7 @@ def init(cooker): Files causing parsing errors are evicted from the cache. """ - return Cache(cooker.configuration.data) + return Cache(cooker.configuration.data, cooker.configuration.data_hash) class CacheData(object):
lib/bb/cooker.py +3 -1
@@ -832,6 +832,7 @@ class BBCooker: bb.parse.init_parser(data) bb.event.fire(bb.event.ConfigParsed(), data) self.configuration.data = data + self.configuration.data_hash = data.get_hash() def handleCollections( self, collections ): """Handle collections"""
@@ -1399,6 +1400,7 @@ class CookerParser(object): self.filelist = filelist self.cooker = cooker self.cfgdata = cooker.configuration.data + self.cfghash = cooker.configuration.data_hash # Accounting statistics self.parsed = 0
@@ -1414,7 +1416,7 @@ class CookerParser(object): self.num_processes = int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or multiprocessing.cpu_count()) - self.bb_cache = bb.cache.Cache(self.cfgdata, cooker.caches_array) + self.bb_cache = bb.cache.Cache(self.cfgdata, self.cfghash, cooker.caches_array) self.fromcache = [] self.willparse = [] for filename in self.filelist:
lib/bb/data_smart.py +17 -0
@@ -31,6 +31,7 @@ BitBake build tools. import copy, re from collections import MutableMapping import logging +import hashlib import bb, bb.codeparser from bb import utils from bb.COW import COWDictBase
@@ -459,3 +460,19 @@ class DataSmart(MutableMapping): def __delitem__(self, var): self.delVar(var) + + def get_hash(self): + data = "" + keys = iter(self) + for key in keys: + if key == "TIME": + continue + if key == "__depends": + deps = list(self.getVar(key, False)) + deps.sort() + value = [deps[i][0] for i in range(len(deps))] + else: + value = self.getVar(key, False) or "" + data = data + key + str(value) + + return hashlib.md5(data).hexdigest()
Signature · 1 line
1.7.0.4
PATCH 3/10 cooker: user bb.configuration.data to inject events
In reply to Dongxiao XuFor buildTargets function, we use bb.configuration.data as parameter to inject events, since in hob environment, some variables are modified dynamically and bb.configuration.event_data may out of date.
--- lib/bb/cooker.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
lib/bb/cooker.py +2 -2
@@ -1085,7 +1085,7 @@ class BBCooker: return False if not retval: - bb.event.fire(bb.event.BuildCompleted(buildname, targets, failures), self.configuration.event_data) + bb.event.fire(bb.event.BuildCompleted(buildname, targets, failures), self.configuration.data) self.command.finishAsyncCommand() return False if retval is True:
@@ -1095,7 +1095,7 @@ class BBCooker: self.buildSetVars() buildname = self.configuration.data.getVar("BUILDNAME") - bb.event.fire(bb.event.BuildStarted(buildname, targets), self.configuration.event_data) + bb.event.fire(bb.event.BuildStarted(buildname, targets), self.configuration.data) localdata = data.createCopy(self.configuration.data) bb.data.update_data(localdata)
Signature · 1 line
1.7.0.4
Re: [PATCH 03/10] cooker: user bb.configuration.data to inject events
Quoted 5 lines · Dongxiao Xu
For buildTargets function, we use bb.configuration.data as parameter to inject events, since in hob environment, some variables are modified dynamically and bb.configuration.event_data may out of date.
Signed-off-by Dongxiao Xu <[email protected]>
I discussed this patch with Dongxiao when he worked on it so it has my ACK.
Quoted 26 lines
--- lib/bb/cooker.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)lib/bb/cooker.py +2
-2@@ -1085,7 +1085,7 @@ class BBCooker: return False if not retval: - bb.event.fire(bb.event.BuildCompleted(buildname, targets, failures), self.configuration.event_data) + bb.event.fire(bb.event.BuildCompleted(buildname, targets, failures), self.configuration.data) self.command.finishAsyncCommand() return False if retval is True:@@ -1095,7 +1095,7 @@ class BBCooker: self.buildSetVars() buildname = self.configuration.data.getVar("BUILDNAME") - bb.event.fire(bb.event.BuildStarted(buildname, targets), self.configuration.event_data) + bb.event.fire(bb.event.BuildStarted(buildname, targets), self.configuration.data) localdata = data.createCopy(self.configuration.data) bb.data.update_data(localdata)
Signature · 3 lines
Joshua Lock
Yocto Project "Johannes factotum"
Intel Open Source Technology CentreRe: [PATCH 03/10] cooker: user bb.configuration.data to inject events
Quoted 18 lines
-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Joshua Lock
Sent: Wednesday, December 14, 2011 12:32 AM
To: [email protected]
Subject: Re: [bitbake-devel] [PATCH 03/10] cooker: user bb.configuration.data to inject eventsQuoted 4 lines · Dongxiao Xu
For buildTargets function, we use bb.configuration.data as parameter to inject events, since in hob environment, some variables are modified dynamically and bb.configuration.event_data may out of date.
Quoted 5 lines
Quoted 1 line
Signed-off-by Dongxiao Xu <[email protected]>If we can't rely on
event_datato be consistent/up-to-date do we need to keep it around? The comment around its creation implies we want/need the copy of data.
Hmm, I am also not sure why we need the event_data. In the cooker.py, some place we use event_data to inject events, and some other place, we use bb.configuration.data. This event_data is introduced from an old bitbake commit: b708cf280fd1bc9566f822eeff4cc670a3ce28a0.
Hope Richard can give more hints on it?
Thanks,
Dongxiao
Quoted 46 lines
This feels like we're working around on issue rather than fixing one.
Joshua
lib/bb/cooker.py class BBCooker: · line 1088
Quoted 13 lines
--- lib/bb/cooker.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)lib/bb/cooker.py index renamed from lib/bb/cooker.py +0
-02c02e28..6ddb38e 100644
lib/bb/cooker.py +0
-1@@ -1085,7 +1085,7 @@ class BBCooker: return False if not retval: - bb.event.fire(bb.event.BuildCompleted(buildname,targets, failures), self.configuration.event_data)
lib/bb/cooker.py class BBCooker: · line 1098
Quoted 10 lines
lib/bb/cooker.py +2
-1+ bb.event.fire(bb.event.BuildCompleted(buildname, + targets, failures), self.configuration.data) self.command.finishAsyncCommand() return False if retval is True:@@ -1095,7 +1095,7 @@ class BBCooker: self.buildSetVars() buildname = self.configuration.data.getVar("BUILDNAME") - bb.event.fire(bb.event.BuildStarted(buildname, targets),self.configuration.event_data)
lib/bb/cooker.py class BBCooker: · line 1101
Quoted 5 lines
lib/bb/cooker.py +2
-0+ bb.event.fire(bb.event.BuildStarted(buildname, targets), + self.configuration.data) localdata = data.createCopy(self.configuration.data) bb.data.update_data(localdata)--
Joshua Lock
Yocto Project "Johannes factotum"
Intel Open Source Technology Centre_______________________________________________
bitbake-devel mailing list
[email protected]
http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/bitbake-devel
PATCH 4/10 command.py: add initCooker API
In reply to Dongxiao XuinitCooker is to set the cooker to the initial state with nothing parsed.
--- lib/bb/command.py | 6 ++++++ lib/bb/cooker.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 0 deletions(-)
lib/bb/command.py +6 -0
@@ -157,6 +157,12 @@ class CommandsSync: value = params[1] command.cooker.configuration.data.setVar(varname, value) + def initCooker(self, command, params): + """ + Init the cooker to initial state with nothing parsed + """ + command.cooker.initialize() + def resetCooker(self, command, params): """ Reset the cooker to its initial state, thus forcing a reparse for
lib/bb/cooker.py +13 -0
@@ -167,6 +167,15 @@ class BBCooker: self.parser = None + def initConfigurationData(self): + self.configuration.data = bb.data.init() + + if not self.server_registration_cb: + bb.data.setVar("BB_WORKERCONTEXT", "1", self.configuration.data) + + filtered_keys = bb.utils.approved_variables() + bb.data.inheritFromOS(self.configuration.data, self.savedenv, filtered_keys) + def loadConfigurationData(self): self.configuration.data = bb.data.init()
@@ -1301,6 +1310,10 @@ class BBCooker: def reparseFiles(self): return + def initialize(self): + self.state = state.initial + self.initConfigurationData() + def reset(self): self.state = state.initial self.loadConfigurationData()
Signature · 1 line
1.7.0.4
PATCH 5/10 command.py: add parseConfigurationFiles API
In reply to Dongxiao XuThe parseConfigurationFiles API calls the related function in cooker.py to parse config files.
--- lib/bb/command.py | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-)
lib/bb/command.py +10 -0
@@ -326,3 +326,13 @@ class CommandsAsync: else: command.finishAsyncCommand() compareRevisions.needcache = True + + def parseConfigurationFiles(self, command, params): + """ + Parse the configuration files + """ + prefiles = params[0] + postfiles = params[1] + command.cooker.parseConfigurationFiles(prefiles, postfiles) + command.finishAsyncCommand() + parseConfigurationFiles.needcache = False
Signature · 1 line
1.7.0.4
Re: [PATCH 05/10] command.py: add parseConfigurationFiles API
Quoted 13 lines · Dongxiao Xu
The parseConfigurationFiles API calls the related function in
cooker.pyto parse config files.Signed-off-by Dongxiao Xu <[email protected]>--- lib/bb/command.py | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-)lib/bb/command.py +0
-0
lib/bb/command.py class CommandsAsync: · line 335
Quoted 10 lines
lib/bb/command.py +7
-0@@ -326,3 +326,13 @@ class CommandsAsync: else: command.finishAsyncCommand() compareRevisions.needcache = True + + def parseConfigurationFiles(self, command, params): + """ + Parse the configuration files + """ + prefiles = params[0] + postfiles = params[1]
Should we be able to use this method without having prefiles and postfiles set in the params container?
A common pattern throughout this module is to check the size of the params container [using len(params)] and set appropriate (empty) defaults if all required parameters aren't passed.
Cheers,
Joshua
Signature · 3 lines
Joshua Lock
Yocto Project "Johannes factotum"
Intel Open Source Technology CentreRe: [PATCH 05/10] command.py: add parseConfigurationFiles API
Hi Josh,
Quoted 37 lines
-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Joshua Lock
Sent: Wednesday, December 14, 2011 12:39 AM
To: [email protected]
Subject: Re: [bitbake-devel] [PATCH 05/10]command.py: add parseConfigurationFiles APIQuoted 13 lines · Dongxiao Xu
The parseConfigurationFiles API calls the related function in
cooker.pyto parse config files.Signed-off-by Dongxiao Xu <[email protected]>--- lib/bb/command.py | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-)lib/bb/command.py index renamed from lib/bb/command.py +0
-0701b286..6b4a598 100644
lib/bb/command.py class CommandsAsync: · line 335
Quoted 10 lines
lib/bb/command.py +7
-0@@ -326,3 +326,13 @@ class CommandsAsync: else: command.finishAsyncCommand() compareRevisions.needcache = True + + def parseConfigurationFiles(self, command, params): + """ + Parse the configuration files + """ + prefiles = params[0] + postfiles = params[1]Should we be able to use this method without having prefiles and postfiles set in the params container?
Quoted 3 lines
A common pattern throughout this module is to check the size of the params container [using len(params)] and set appropriate (empty) defaults if all required parameters aren't passed.
When calling parseConfigurationFiles(), we must pass the prefiles and postfiles as parameters, although they may be empty strings, for example,
self.server.runCommand(["parseConfigurationFiles", "", ""])
Therefore it is ensured that this API accepts two variables and it is not needed to check the length of parameters.
There are many functions in command.py doing the similar thing, like buildFile, generateDepTreeEvent, etc.
Thanks,
Dongxiao
Quoted 12 lines
Cheers,
Joshua
--
Joshua Lock
Yocto Project "Johannes factotum"
Intel Open Source Technology Centre_______________________________________________
bitbake-devel mailing list
[email protected]
http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/bitbake-devel
Re: [PATCH 05/10] command.py: add parseConfigurationFiles API
Quoted 39 lines · Dongxiao
Hi Josh,
Quoted 37 lines
-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Joshua Lock
Sent: Wednesday, December 14, 2011 12:39 AM
To: [email protected]
Subject: Re: [bitbake-devel] [PATCH 05/10]command.py: add parseConfigurationFiles APIQuoted 13 lines · Dongxiao Xu
The parseConfigurationFiles API calls the related function in
cooker.pyto parse config files.Signed-off-by Dongxiao Xu <[email protected]>--- lib/bb/command.py | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-)lib/bb/command.py index renamed from lib/bb/command.py +0
-0701b286..6b4a598 100644
lib/bb/command.py class CommandsAsync: · line 335
Quoted 10 lines
lib/bb/command.py +7
-0@@ -326,3 +326,13 @@ class CommandsAsync: else: command.finishAsyncCommand() compareRevisions.needcache = True + + def parseConfigurationFiles(self, command, params): + """ + Parse the configuration files + """ + prefiles = params[0] + postfiles = params[1]Should we be able to use this method without having prefiles and postfiles set in the params container?
Quoted 7 lines
Quoted 3 lines
A common pattern throughout this module is to check the size of the params container [using len(params)] and set appropriate (empty) defaults if all required parameters aren't passed.
When calling
parseConfigurationFiles(), we must pass the prefiles and postfiles as parameters, although they may be empty strings, for example,
self.server.runCommand(["parseConfigurationFiles", "", ""])
Ah yes, quite right. I remember writing that now.
Thanks,
Joshua
Signature · 3 lines
Joshua Lock
Yocto Project "Johannes factotum"
Intel Open Source Technology CentrePATCH 6/10 command.py: add generateTargetsTreePro API
In reply to Dongxiao XuCurrently we have generateTargetsTree API, which is used to get dependency information. However in that tree, there will be "virtual/xxx" in depends fields. Therefore we need to replace it with its real providers.
Besides, for packages that provided by multiple recipes, we will find their preverred provider.
To make the original Hob working, we still keep the generateTargetsTree API.
--- lib/bb/command.py | 14 +++++++ lib/bb/cooker.py | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 0 deletions(-)
lib/bb/command.py +14 -0
@@ -238,6 +238,20 @@ class CommandsAsync: command.finishAsyncCommand() generateTargetsTree.needcache = True + def generateTargetsTreePro(self, command, params): + """ + A pro version of generateTargetsTree + """ + klass = params[0] + if len(params) > 1: + pkg_list = params[1] + else: + pkg_list = [] + + command.cooker.generateTargetsTreePro(klass, pkg_list) + command.finishAsyncCommand() + generateTargetsTreePro.needcache = True + def findConfigFiles(self, command, params): """ Find config files which provide appropriate values
lib/bb/cooker.py +98 -0
110 lines not on this page · raw message
Signature · 1 line
1.7.0.4
Re: [PATCH 06/10] command.py: add generateTargetsTreePro API
Quoted 10 lines · Dongxiao Xu
Currently we have generateTargetsTree API, which is used to get dependency information. However in that tree, there will be "virtual/xxx" in depends fields. Therefore we need to replace it with its real providers.
Besides, for packages that provided by multiple recipes, we will find their preverred provider.
To make the original Hob working, we still keep the generateTargetsTree API.
I assume the 'original Hob' compatability will only be maintained until Hobv2 is merged?
Is there anything preventing us from having an optional parameter on generateTargetsTree to switch to pro mode rather than duplicating a bunch of code and functionality on a temporary basis?
something like:
Quoted 9 lines
lib/bb/command.py +9
-0+ def generateTargetsTree(self, command, params): + """ + A pro version of generateTargetsTree + """ + klass = params[0] + param_len = len(params) + if param_len 2: + pkg_list = params[1] + resolve = params[2]
lib/bb/command.py class CommandsAsync: · line 253
Quoted 10 lines
lib/bb/command.py +10
-0+ elif param_len > 1: + pkg_list = params[1] + resolve = false + else: + pkg_list = [] + resolve = false + + command.cooker.generateTargetsTree(klass, pkg_list, resolve) + command.finishAsyncCommand() + generateTargetsTreePro.needcache = True
Then you can use the resolve boolean to switch the generateBlahBlahBlah methods to operate in the old or new way. Method signature would be something like:
Review
Quoted 3 lines
(quoted diff) +2
-0+ def generateTargetsTree(self, klass=None, pkgs=[], resolve=false): + def generatePkgDepTreeDataPro(self, pkgs_to_build, task,
resolve=false):
Cheers,
Joshua
Quoted 148 lines
Signed-off-by Dongxiao Xu <[email protected]>--- lib/bb/command.py | 14 +++++++ lib/bb/cooker.py | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 0 deletions(-)lib/bb/command.py +14
-0@@ -238,6 +238,20 @@ class CommandsAsync: command.finishAsyncCommand() generateTargetsTree.needcache = True + def generateTargetsTreePro(self, command, params): + """ + A pro version of generateTargetsTree + """ + klass = params[0] + if len(params) > 1: + pkg_list = params[1] + else: + pkg_list = [] + + command.cooker.generateTargetsTreePro(klass, pkg_list) + command.finishAsyncCommand() + generateTargetsTreePro.needcache = True + def findConfigFiles(self, command, params): """ Find config files which provide appropriate valueslib/bb/cooker.py +98
-0@@ -498,6 +498,85 @@ class BBCooker: return depend_tree + def generatePkgDepTreeDataPro(self, pkgs_to_build, task): + """ + Create a dependency tree of pkgs_to_build, returning the data. + """ + _, taskdata = self.prepareTreeData(pkgs_to_build, task) + tasks_fnid = [] + if len(taskdata.tasks_name) != 0: + for task in xrange(len(taskdata.tasks_name)): + tasks_fnid.append(taskdata.tasks_fnid[task]) + + seen_fnids = [] + depend_tree = {} + depend_tree["depends"] = {} + depend_tree["pn"] = {} + depend_tree["rdepends-pn"] = {} + depend_tree["packages"] = {} + depend_tree["rdepends-pkg"] = {} + + for task in xrange(len(tasks_fnid)): + fnid = tasks_fnid[task] + fn = taskdata.fn_index[fnid] + pn = self.status.pkg_fn[fn] + version = "%s:%s-%s" % self.status.pkg_pepvpr[fn] + summary = self.status.summary[fn] + lic = self.status.license[fn] + section = self.status.section[fn] + rdepends = self.status.rundeps[fn] + if pn not in depend_tree["pn"]: + depend_tree["pn"][pn] = {} + depend_tree["pn"][pn]["filename"] = fn + depend_tree["pn"][pn]["version"] = version + depend_tree["pn"][pn]["summary"] = summary + depend_tree["pn"][pn]["license"] = lic + depend_tree["pn"][pn]["section"] = section + depend_tree["pn"][pn]["packages"] = rdepends.keys() + + if fnid not in seen_fnids: + seen_fnids.append(fnid) + packages = [] + + depend_tree["depends"][pn] = [] + for dep in taskdata.depids[fnid]: + item = taskdata.build_names_index[dep] + pn_provider = "" + targetid = taskdata.getbuild_id(item) + if targetid in taskdata.build_targets: + fnid = taskdata.build_targets[targetid][0] + fn_provider = taskdata.fn_index[fnid] + pn_provider = self.status.pkg_fn[fn_provider] + else: + pn_provider = item + depend_tree["depends"][pn].append(pn_provider) + + depend_tree["rdepends-pn"][pn] = [] + for rdep in taskdata.rdepids[fnid]: + depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep]) + + for package in rdepends: + depend_tree["rdepends-pkg"][package] = [] + for rdepend in rdepends[package]: + depend_tree["rdepends-pkg"][package].append(rdepend) + packages.append(package) + + for package in packages: + if package not in depend_tree["packages"]: + targetid = taskdata.getrun_id(package) + if targetid in taskdata.run_targets: + fnid = taskdata.run_targets[targetid][0] + fn = taskdata.fn_index[fnid] + pn = self.status.pkg_fn[fn] + version = "%s:%s-%s" % self.status.pkg_pepvpr[fn] + depend_tree["packages"][package] = {} + depend_tree["packages"][package]["pn"] = pn + depend_tree["packages"][package]["filename"] = fn + depend_tree["packages"][package]["version"] = version + + return depend_tree + + def generateDepTreeEvent(self, pkgs_to_build, task): """ Create a task dependency graph of pkgs_to_build.@@ -746,6 +825,25 @@ class BBCooker: tree = self.generatePkgDepTreeData(pkgs, 'build') bb.event.fire(bb.event.TargetsTreeGenerated(tree), self.configuration.data) + def generateTargetsTreePro(self, klass=None, pkgs=[]): + """ + Generate a dependency tree of buildable targets + Check and add providers for each dependencies + Generate an event with the result + """ + # if the caller hasn't specified a pkgs list default to universe + if not len(pkgs): + pkgs = ['universe'] + # if inherited_class passed ensure all recipes which inherit the + # specified class are included in pkgs + if klass: + extra_pkgs = self.findInheritsClass(klass) + pkgs = pkgs + extra_pkgs + + # generate a dependency tree for all our packages + tree = self.generatePkgDepTreeDataPro(pkgs, 'build') + bb.event.fire(bb.event.TargetsTreeGenerated(tree), self.configuration.data) + def buildWorldTargetList(self): """ Build package list for "bitbake world"
Signature · 3 lines
Joshua Lock
Yocto Project "Johannes factotum"
Intel Open Source Technology CentreRe: [PATCH 06/10] command.py: add generateTargetsTreePro API
Quoted 23 lines
-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Joshua Lock
Sent: Wednesday, December 14, 2011 12:47 AM
To: [email protected]
Subject: Re: [bitbake-devel] [PATCH 06/10]command.py: add generateTargetsTreePro APIQuoted 10 lines · Dongxiao Xu
Currently we have generateTargetsTree API, which is used to get dependency information. However in that tree, there will be "virtual/xxx" in depends fields. Therefore we need to replace it with its real providers.
Besides, for packages that provided by multiple recipes, we will find their preverred provider.
To make the original Hob working, we still keep the generateTargetsTree API.
Quoted 6 lines
I assume the 'original Hob' compatability will only be maintained until Hobv2 is merged?
Is there anything preventing us from having an optional parameter on generateTargetsTree to switch to pro mode rather than duplicating a bunch of code and functionality on a temporary basis?
Yes, thanks for your suggestion, I will modify it accordingly.
Thanks,
Dongxiao
Quoted 203 lines
something like:
Quoted 9 lines
lib/bb/command.py +9
-0+ def generateTargetsTree(self, command, params): + """ + A pro version of generateTargetsTree + """ + klass = params[0] + param_len = len(params) + if param_len 2: + pkg_list = params[1] + resolve = params[2]lib/bb/command.py class CommandsAsync: · line 253
Quoted 10 lines
lib/bb/command.py +10
-0+ elif param_len > 1: + pkg_list = params[1] + resolve = false + else: + pkg_list = [] + resolve = false + + command.cooker.generateTargetsTree(klass, pkg_list, resolve) + command.finishAsyncCommand() + generateTargetsTreePro.needcache = TrueThen you can use the resolve boolean to switch the generateBlahBlahBlah methods to operate in the old or new way. Method signature would be something like:
Review
Quoted 3 lines
(quoted diff) +2
-0+ def generateTargetsTree(self, klass=None, pkgs=[], resolve=false): + def generatePkgDepTreeDataPro(self, pkgs_to_build, task,resolve=false):
Cheers,
JoshuaQuoted 5 lines
Signed-off-by Dongxiao Xu <[email protected]>--- lib/bb/command.py | 14 +++++++ lib/bb/cooker.py | 98+++++++++++++++++++++++++++++++++++++++++++++++++++++
Quoted 87 lines
2 files changed, 112 insertions(+), 0 deletions(-)lib/bb/command.py index renamed from lib/bb/command.py +0
-06b4a598..15e7839 100644
lib/bb/command.py +14
-0@@ -238,6 +238,20 @@ class CommandsAsync: command.finishAsyncCommand() generateTargetsTree.needcache = True + def generateTargetsTreePro(self, command, params): + """ + A pro version of generateTargetsTree + """ + klass = params[0] + if len(params) > 1: + pkg_list = params[1] + else: + pkg_list = [] + + command.cooker.generateTargetsTreePro(klass, pkg_list) + command.finishAsyncCommand() + generateTargetsTreePro.needcache = True + def findConfigFiles(self, command, params): """ Find config files which provide appropriate values diff --git
a/lib/bb/cooker.pyb/lib/bb/cooker.pyindex 666242f..9f05d35 100644lib/bb/cooker.py +53
-0@@ -498,6 +498,85 @@ class BBCooker: return depend_tree + def generatePkgDepTreeDataPro(self, pkgs_to_build, task): + """ + Create a dependency tree of pkgs_to_build, returning the data. + """ + _, taskdata = self.prepareTreeData(pkgs_to_build, task) + tasks_fnid = [] + if len(taskdata.tasks_name) != 0: + for task in xrange(len(taskdata.tasks_name)): + tasks_fnid.append(taskdata.tasks_fnid[task]) + + seen_fnids = [] + depend_tree = {} + depend_tree["depends"] = {} + depend_tree["pn"] = {} + depend_tree["rdepends-pn"] = {} + depend_tree["packages"] = {} + depend_tree["rdepends-pkg"] = {} + + for task in xrange(len(tasks_fnid)): + fnid = tasks_fnid[task] + fn = taskdata.fn_index[fnid] + pn = self.status.pkg_fn[fn] + version = "%s:%s-%s" % self.status.pkg_pepvpr[fn] + summary = self.status.summary[fn] + lic = self.status.license[fn] + section = self.status.section[fn] + rdepends = self.status.rundeps[fn] + if pn not in depend_tree["pn"]: + depend_tree["pn"][pn] = {} + depend_tree["pn"][pn]["filename"] = fn + depend_tree["pn"][pn]["version"] = version + depend_tree["pn"][pn]["summary"] = summary + depend_tree["pn"][pn]["license"] = lic + depend_tree["pn"][pn]["section"] = section + depend_tree["pn"][pn]["packages"] = rdepends.keys() + + if fnid not in seen_fnids: + seen_fnids.append(fnid) + packages = [] + + depend_tree["depends"][pn] = [] + for dep in taskdata.depids[fnid]: + item = taskdata.build_names_index[dep] + pn_provider = "" + targetid = taskdata.getbuild_id(item) + if targetid in taskdata.build_targets: + fnid = taskdata.build_targets[targetid][0] + fn_provider = taskdata.fn_index[fnid] + pn_provider = self.status.pkg_fn[fn_provider] + else: + pn_provider = item + depend_tree["depends"][pn].append(pn_provider) +lib/bb/cooker.py class BBCooker: · line 563
Quoted 10 lines
lib/bb/cooker.py +10
-0+ depend_tree["rdepends-pn"][pn] = [] + for rdep in taskdata.rdepids[fnid]: + + depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep] + ) + + for package in rdepends: + depend_tree["rdepends-pkg"][package] = [] + for rdepend in rdepends[package]: +depend_tree["rdepends-pkg"][package].append(rdepend)
lib/bb/cooker.py class BBCooker:
Quoted 10 lines
lib/bb/cooker.py +10
-0+ packages.append(package) + + for package in packages: + if package not in depend_tree["packages"]: + targetid = taskdata.getrun_id(package) + if targetid in taskdata.run_targets: + fnid = taskdata.run_targets[targetid][0] + fn = taskdata.fn_index[fnid] + pn = self.status.pkg_fn[fn] + version = "%s:%s-%s" %self.status.pkg_pepvpr[fn]
lib/bb/cooker.py class BBCooker: · line 576
Quoted 3 lines
lib/bb/cooker.py +3
-0+ depend_tree["packages"][package] = {} + depend_tree["packages"][package]["pn"] = pn +depend_tree["packages"][package]["filename"] = fn
lib/bb/cooker.py class BBCooker: · line 577
Quoted 1 line
lib/bb/cooker.py +1
-0+ depend_tree["packages"][package]["version"]=
Quoted 26 lines
lib/bb/cooker.py +18
-0+ version + + return depend_tree + + def generateDepTreeEvent(self, pkgs_to_build, task): """ Create a task dependency graph of pkgs_to_build.@@ -746,6 +825,25 @@ class BBCooker: tree = self.generatePkgDepTreeData(pkgs, 'build') bb.event.fire(bb.event.TargetsTreeGenerated(tree), self.configuration.data) + def generateTargetsTreePro(self, klass=None, pkgs=[]): + """ + Generate a dependency tree of buildable targets + Check and add providers for each dependencies + Generate an event with the result + """ + # if the caller hasn't specified a pkgs list default to universe + if not len(pkgs): + pkgs = ['universe'] + # if inherited_class passed ensure all recipes which inherit the + # specified class are included in pkgs + if klass: + extra_pkgs = self.findInheritsClass(klass)lib/bb/cooker.py class BBCooker: · line 849
Quoted 10 lines
lib/bb/cooker.py +7
-0+ pkgs = pkgs + extra_pkgs + + # generate a dependency tree for all our packages + tree = self.generatePkgDepTreeDataPro(pkgs, 'build') + bb.event.fire(bb.event.TargetsTreeGenerated(tree), + self.configuration.data) + def buildWorldTargetList(self): """ Build package list for "bitbake world"--
Joshua Lock
Yocto Project "Johannes factotum"
Intel Open Source Technology Centre_______________________________________________
bitbake-devel mailing list
[email protected]
http://lists.linuxtogo.org/cgi-bin/mailman/listinfo/bitbake-devel
PATCH 7/10 event.py: Add a new event PackageInfo
In reply to Dongxiao XuThis event is to pass package information.
--- lib/bb/event.py | 9 +++++++++ 1 files changed, 9 insertions(+), 0 deletions(-)
lib/bb/event.py +9 -0
@@ -459,3 +459,12 @@ class LogHandler(logging.Handler): def filter(self, record): record.taskpid = worker_pid return True + +class PackageInfo(Event): + """ + Package information for GUI + """ + def __init__(self, recipe, pkginfolist): + Event.__init__(self) + self._recipe = recipe + self._pkginfolist = pkginfolist
Signature · 1 line
1.7.0.4
PATCH 8/10 xmlrpc: Change BitbakeServerInfo init function
In reply to Dongxiao XuPass host and port to BitbakeServerInfo class instead of the "server" instance. With this change, GUI can connect with server individually by host address and port.
--- lib/bb/server/xmlrpc.py | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-)
lib/bb/server/xmlrpc.py +4 -4
@@ -242,9 +242,9 @@ class BitBakeXMLRPCServer(SimpleXMLRPCServer): return class BitbakeServerInfo(): - def __init__(self, server): - self.host = server.host - self.port = server.port + def __init__(self, host, port): + self.host = host + self.port = port class BitBakeServerConnection(): def __init__(self, serverinfo):
@@ -278,7 +278,7 @@ class BitBakeServer(object): return self.server.register_idle_function def saveConnectionDetails(self): - self.serverinfo = BitbakeServerInfo(self.server) + self.serverinfo = BitbakeServerInfo(self.server.host, self.server.port) def detach(self, cooker_logfile): daemonize.createDaemon(self.server.serve_forever, cooker_logfile)
Signature · 1 line
1.7.0.4
PATCH 9/10 cooker: remove command import in cooker.py
In reply to Dongxiao XuThere is no direct use of command in cooker.py, and it is using bb.command instead. Remove command in the import list.
This fixes a problem of embedded import between command.py and cooker.py.
--- lib/bb/cooker.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
lib/bb/cooker.py +2 -2
@@ -34,8 +34,8 @@ from cStringIO import StringIO from contextlib import closing from functools import wraps from collections import defaultdict -import bb, bb.exceptions -from bb import utils, data, parse, event, cache, providers, taskdata, command, runqueue +import bb, bb.exceptions, bb.command +from bb import utils, data, parse, event, cache, providers, taskdata, runqueue logger = logging.getLogger("BitBake") collectlog = logging.getLogger("BitBake.Collection")
Signature · 1 line
1.7.0.4
PATCH 10/10 bitbake: add a new option "--server-only"
In reply to Dongxiao XuCreate a new option "--server-only" for bitbake command, which allows bitbake runs as a server, and let frontend connect the server itself.
"--server-only" should work with "-t xmlrpc", or bitbake will exit.
bitbake --server-only -t xmlrpc will print out the server address and port information.
--- bin/bitbake | 23 ++++++++++++++++------- 1 files changed, 16 insertions(+), 7 deletions(-)
bin/bitbake +16 -7
@@ -165,6 +165,9 @@ Default BBFILES are the .bb files in the current directory.""") parser.add_option("", "--revisions-changed", help = "Set the exit code depending on whether upstream floating revisions have changed or not", action = "store_true", dest = "revisions_changed", default = False) + parser.add_option("", "--server-only", help = "Run bitbake without UI, the frontend can connect with bitbake server itself", + action = "store_true", dest = "server_only", default = False) + options, args = parser.parse_args(sys.argv) configuration = BBConfiguration(options)
@@ -186,6 +189,9 @@ Default BBFILES are the .bb files in the current directory.""") sys.exit("FATAL: Invalid server type '%s' specified.\n" "Valid interfaces: xmlrpc, process [default], none." % servertype) + if configuration.server_only and configuration.servertype != "xmlrpc": + sys.exit("FATAL: If '--server-only' is defined, we must set the servertype as 'xmlrpc'.\n") + # Save a logfile for cooker into the current working directory. When the # server is daemonized this logfile will be truncated. cooker_logfile = os.path.join(os.getcwd(), "cooker.log")
@@ -222,14 +228,17 @@ Default BBFILES are the .bb files in the current directory.""") logger.removeHandler(handler) - # Setup a connection to the server (cooker) - server_connection = server.establishConnection() + if not configuration.server_only: + # Setup a connection to the server (cooker) + server_connection = server.establishConnection() - try: - return server.launchUI(ui_main, server_connection.connection, server_connection.events) - finally: - bb.event.ui_queue = [] - server_connection.terminate() + try: + return server.launchUI(ui_main, server_connection.connection, server_connection.events) + finally: + bb.event.ui_queue = [] + server_connection.terminate() + else: + print("server address: %s, server port: %s" % (server.serverinfo.host, server.serverinfo.port)) return 1
Signature · 1 line
1.7.0.4