Add support for per-multiconfig BBMASK
cover Add support for per-multiconfig BBMASK
Splits apart the bitbake configuration data and caches so that each defined multiconfig can specify as separate BBMASK independently of the other multiconfigs. This can take multiple forms, such as a BBMASK in a multiconfig .conf file, or BBMASK assignment based on an override (see linked bugzilla for an example).
The most noticeable change is that bitbake now keeps a parsing cache file for each multiconfig instead of a single parsing cache file that caches all multiconfigs.
[YOCTO #13721]
Joshua Watt (8):
bitbake: cooker: Split file collections per multiconfig
bitbake: cache: Use multiconfig aware caches
bitbake: lib: Add support for Logging Adapters
bitbake: lib: Add PrefixLoggerAdapter helper
bitbake: cache: Improve logging
bitbake: cache: Cache size optimization
bitbake: tests: Add tests for BBMASK in multiconfig
bitbake: command: Move split_mc_pn to runqueue
bitbake/lib/bb/__init__.py | 30 ++- bitbake/lib/bb/cache.py | 219 ++++++++++++------ bitbake/lib/bb/command.py | 39 ++-- bitbake/lib/bb/cooker.py | 165 +++++++------ bitbake/lib/bb/runqueue.py | 17 +- .../bb/tests/runqueue-tests/conf/bitbake.conf | 3 +- .../runqueue-tests/conf/multiconfig/mc1.conf | 1 + .../runqueue-tests/conf/multiconfig/mc2.conf | 1 + .../recipes/fails-mc/fails-mc1.bb | 5 + .../recipes/fails-mc/fails-mc2.bb | 4 + bitbake/lib/bb/tests/runqueue.py | 15 ++ bitbake/lib/bb/tinfoil.py | 26 ++- bitbake/lib/bblayers/query.py | 4 +- 13 files changed, 356 insertions(+), 173 deletions(-) create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb
Signature · 1 line
2.17.1
PATCH 1/8 bitbake: cooker: Split file collections per multiconfig
Splits the cooker to track a collection per multiconfig instead of a single collection for all multiconfigs. Practically speaking, this allows each multiconfigs to each have different BBMASKs that apply to it instead of each one using the mask specified in the base configuration.
--- bitbake/lib/bb/cache.py | 2 +- bitbake/lib/bb/command.py | 31 ++++++-- bitbake/lib/bb/cooker.py | 139 ++++++++++++++++++++-------------- bitbake/lib/bb/runqueue.py | 11 +-- bitbake/lib/bb/tinfoil.py | 26 ++++--- bitbake/lib/bblayers/query.py | 4 +- 6 files changed, 129 insertions(+), 84 deletions(-)
bitbake/lib/bb/cache.py +1 -1
@@ -623,7 +623,7 @@ class Cache(NoCache): self.remove(fn) return False - if appends != info_array[0].appends: + if tuple(appends) != tuple(info_array[0].appends): logger.debug(2, "Cache: appends for %s changed", fn) logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) self.remove(fn)
bitbake/lib/bb/command.py +24 -7
@@ -232,7 +232,11 @@ class CommandsSync: def matchFile(self, command, params): fMatch = params[0] - return command.cooker.matchFile(fMatch) + try: + mc = params[0] + except IndexError: + mc = '' + return command.cooker.matchFile(fMatch, mc) matchFile.needconfig = False def getUIHandlerNum(self, command, params):
@@ -396,21 +400,33 @@ class CommandsSync: # Return list sorted by reverse priority order import bb.cache skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), - key=lambda x: (-command.cooker.collection.calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0]))) + key=lambda x: (-command.cooker.collections[mc].calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0]))) return list(skipdict.items()) getSkippedRecipes.readonly = True def getOverlayedRecipes(self, command, params): - return list(command.cooker.collection.overlayed.items()) + try: + mc = params[0] + except IndexError: + mc = '' + return list(command.cooker.collections[mc].overlayed.items()) getOverlayedRecipes.readonly = True def getFileAppends(self, command, params): fn = params[0] - return command.cooker.collection.get_file_appends(fn) + try: + mc = params[1] + except IndexError: + mc = '' + return command.cooker.collections[mc].get_file_appends(fn) getFileAppends.readonly = True def getAllAppends(self, command, params): - return command.cooker.collection.bbappends + try: + mc = params[0] + except IndexError: + mc = '' + return command.cooker.collections[mc].bbappends getAllAppends.readonly = True def findProviders(self, command, params):
@@ -496,6 +512,7 @@ class CommandsSync: for the recipe. """ fn = params[0] + mc = bb.runqueue.mc_from_tid(fn) appends = params[1] appendlist = params[2] if len(params) > 3:
@@ -507,7 +524,7 @@ class CommandsSync: if appendlist is not None: appendfiles = appendlist else: - appendfiles = command.cooker.collection.get_file_appends(fn) + appendfiles = command.cooker.collections[mc].get_file_appends(fn) else: appendfiles = [] # We are calling bb.cache locally here rather than on the server,
@@ -517,7 +534,7 @@ class CommandsSync: if config_data: # We have to use a different function here if we're passing in a datastore # NOTE: we took a copy above, so we don't do it here again - envdata = bb.cache.parse_recipe(config_data, fn, appendfiles)[''] + envdata = bb.cache.parse_recipe(config_data, fn, appendfiles, mc)[''] else: # Use the standard path parser = bb.cache.NoCache(command.cooker.databuilder)
bitbake/lib/bb/cooker.py +82 -57
268 lines not on this page · raw message
bitbake/lib/bb/runqueue.py +6 -5
@@ -1557,7 +1557,8 @@ class RunQueue: def rq_dump_sigfn(self, fn, options): bb_cache = bb.cache.NoCache(self.cooker.databuilder) - the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn)) + mc = bb.runqueue.mc_from_tid(fn) + the_data = bb_cache.loadDataFull(fn, self.cooker.collections[mc].get_file_appends(fn)) siggen = bb.parse.siggen dataCaches = self.rqdata.dataCaches siggen.dump_sigfn(fn, dataCaches, options)
@@ -2042,10 +2043,10 @@ class RunQueueExecute: if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: if not mc in self.rq.fakeworker: self.rq.start_fakeworker(self, mc) - self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") + self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") self.rq.fakeworker[mc].process.stdin.flush() else: - self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") + self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") self.rq.worker[mc].process.stdin.flush() self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
@@ -2129,10 +2130,10 @@ class RunQueueExecute: self.rq.state = runQueueFailed self.stats.taskFailed() return True - self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") + self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") self.rq.fakeworker[mc].process.stdin.flush() else: - self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") + self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") self.rq.worker[mc].process.stdin.flush() self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
bitbake/lib/bb/tinfoil.py +14 -12
@@ -117,15 +117,16 @@ class TinfoilCookerAdapter: class TinfoilCookerCollectionAdapter: """ cooker.collection adapter """ - def __init__(self, tinfoil): + def __init__(self, tinfoil, mc=''): self.tinfoil = tinfoil + self.mc = mc def get_file_appends(self, fn): - return self.tinfoil.get_file_appends(fn) + return self.tinfoil.get_file_appends(fn, self.mc) def __getattr__(self, name): if name == 'overlayed': - return self.tinfoil.get_overlayed_recipes() + return self.tinfoil.get_overlayed_recipes(self.mc) elif name == 'bbappends': - return self.tinfoil.run_command('getAllAppends') + return self.tinfoil.run_command('getAllAppends', self.mc) else: raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
@@ -185,10 +186,11 @@ class TinfoilCookerAdapter: def __init__(self, tinfoil): self.tinfoil = tinfoil - self.collection = self.TinfoilCookerCollectionAdapter(tinfoil) + self.collections = {} self.recipecaches = {} self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil) for mc in (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split(): + self.collections[mc] = self.TinfoilCookerCollectionAdapter(tinfoil) self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc) self._cache = {} def __getattr__(self, name):
@@ -196,7 +198,7 @@ class TinfoilCookerAdapter: if name in self._cache: return self._cache[name] elif name == 'skiplist': - attrvalue = self.tinfoil.get_skipped_recipes() + attrvalue = self.tinfoil.get_skipped_recipes(self.mc) elif name == 'bbfile_config_priorities': ret = self.tinfoil.run_command('getLayerPriorities') bbfile_config_priorities = []
@@ -492,18 +494,18 @@ class Tinfoil: raise Exception('Not connected to server (did you call .prepare()?)') return self.server_connection.events.waitEvent(timeout) - def get_overlayed_recipes(self): + def get_overlayed_recipes(self, mc='') """ Find recipes which are overlayed (i.e. where recipes exist in multiple layers) """ - return defaultdict(list, self.run_command('getOverlayedRecipes')) + return defaultdict(list, self.run_command('getOverlayedRecipes', mc)) - def get_skipped_recipes(self): + def get_skipped_recipes(self, mc=''): """ Find recipes which were skipped (i.e. SkipRecipe was raised during parsing). """ - return OrderedDict(self.run_command('getSkippedRecipes')) + return OrderedDict(self.run_command('getSkippedRecipes', mc)) def get_all_providers(self, mc=''): return defaultdict(list, self.run_command('allProviders', mc))
@@ -534,11 +536,11 @@ class Tinfoil: raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn) return best[3] - def get_file_appends(self, fn): + def get_file_appends(self, fn, mc='') """ Find the bbappends for a recipe file """ - return self.run_command('getFileAppends', fn) + return self.run_command('getFileAppends', fn, mc) def all_recipes(self, mc='', sort=True): """
bitbake/lib/bblayers/query.py +2 -2
@@ -320,12 +320,12 @@ Lists recipes with the bbappends that apply to them as subitems. def get_appends_for_files(self, filenames): appended, notappended = [], [] for filename in filenames: - _, cls, _ = bb.cache.virtualfn2realfn(filename) + _, cls, mc = bb.cache.virtualfn2realfn(filename) if cls: continue basename = os.path.basename(filename) - appends = self.tinfoil.cooker.collection.get_file_appends(basename) + appends = self.tinfoil.cooker.collections[mc].get_file_appends(basename) if appends: appended.append((basename, list(appends))) else:
Signature · 1 line
2.17.1
PATCH 2/8 bitbake: cache: Use multiconfig aware caches
In reply to Joshua WattSplits the parsing cache to maintain one cache per multiconfig instead of one global cache. This is necessary now that the files and appends can vary for each multiconfig. A bb.cache.MulticonfigCache dictionary-like proxy object is created instead of a single bb.cache.Cache object. This object will create and properly initialize bb.cache.Cache object for each multiconfig, and each of these caches has a dedicated cache file with a name based on the multiconfig.
--- bitbake/lib/bb/cache.py | 143 +++++++++++++++++++++++++++++---------- bitbake/lib/bb/cooker.py | 42 +++++++----- 2 files changed, 133 insertions(+), 52 deletions(-)
bitbake/lib/bb/cache.py +108 -35
@@ -19,7 +19,7 @@ import os import logging import pickle -from collections import defaultdict +from collections import defaultdict, Mapping import bb.utils import re
@@ -27,8 +27,11 @@ logger = logging.getLogger("BitBake.Cache") __cache_version__ = "152" -def getCacheFile(path, filename, data_hash): - return os.path.join(path, filename + "." + data_hash) +def getCacheFile(path, filename, mc, data_hash): + mcspec = '' + if mc: + mcspec = ".%s" % mc + return os.path.join(path, filename + mcspec + "." + data_hash) # RecipeInfoCommon defines common data retrieving methods # from meta data for caches. CoreRecipeInfo as well as other
@@ -354,14 +357,14 @@ class Cache(NoCache): """ BitBake Cache implementation """ - - def __init__(self, databuilder, data_hash, caches_array): + def __init__(self, databuilder, mc, data_hash, caches_array): super().__init__(databuilder) data = databuilder.data # Pass caches_array information into Cache Constructor # It will be used later for deciding whether we # need extra cache file dump/load support + self.mc = mc self.caches_array = caches_array self.cachedir = data.getVar("CACHE") self.clean = set()
@@ -379,7 +382,17 @@ class Cache(NoCache): return self.has_cache = True - self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash) + + def getCacheFile(self, cachefile): + return getCacheFile(self.cachedir, cachefile, self.mc, self.data_hash) + + def prepare_cache(self, progress): + if not self.has_cache: + return 0 + + loaded = 0 + + self.cachefile = self.getCacheFile("bb_cache.dat") logger.debug(1, "Cache dir: %s", self.cachedir) bb.utils.mkdirhier(self.cachedir)
@@ -387,18 +400,22 @@ class Cache(NoCache): cache_ok = True if self.caches_array: for cache_class in self.caches_array: - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + cachefile = self.getCacheFile(cache_class.cachefile) cache_ok = cache_ok and os.path.exists(cachefile) cache_class.init_cacheData(self) if cache_ok: - self.load_cachefile() + loaded = self.load_cachefile(progress) elif os.path.isfile(self.cachefile): logger.info("Out of date cache found, rebuilding...") else: logger.debug(1, "Cache file %s not found, building..." % self.cachefile) # We don't use the symlink, its just for debugging convinience - symlink = os.path.join(self.cachedir, "bb_cache.dat") + if self.mc: + symlink = os.path.join(self.cachedir, "bb_cache.dat.%s" % self.mc) + else: + symlink = os.path.join(self.cachedir, "bb_cache.dat") + if os.path.exists(symlink): bb.utils.remove(symlink) try:
@@ -406,21 +423,30 @@ class Cache(NoCache): except OSError: pass - def load_cachefile(self): - cachesize = 0 - previous_progress = 0 - previous_percent = 0 + return loaded + + def cachesize(self): + if not self.has_cache: + return 0 - # Calculate the correct cachesize of all those cache files + cachesize = 0 for cache_class in self.caches_array: - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) - with open(cachefile, "rb") as cachefile: - cachesize += os.fstat(cachefile.fileno()).st_size + cachefile = self.getCacheFile(cache_class.cachefile) + try: + with open(cachefile, "rb") as cachefile: + cachesize += os.fstat(cachefile.fileno()).st_size + except FileNotFoundError: + pass - bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data) + return cachesize + + def load_cachefile(self, progress): + cachesize = self.cachesize() + previous_progress = 0 + previous_percent = 0 for cache_class in self.caches_array: - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + cachefile = self.getCacheFile(cache_class.cachefile) logger.debug(1, 'Loading cache file: %s' % cachefile) with open(cachefile, "rb") as cachefile: pickled = pickle.Unpickler(cachefile)
@@ -460,23 +486,11 @@ class Cache(NoCache): self.depends_cache[key] = [value] # only fire events on even percentage boundaries current_progress = cachefile.tell() + previous_progress - if current_progress > cachesize: - # we might have calculated incorrect total size because a file - # might've been written out just after we checked its size - cachesize = current_progress - current_percent = 100 * current_progress / cachesize - if current_percent > previous_percent: - previous_percent = current_percent - bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize), - self.data) + progress(cachefile.tell() + previous_progress) previous_progress += current_progress - # Note: depends cache number is corresponding to the parsing file numbers. - # The same file has several caches, still regarded as one item in the cache - bb.event.fire(bb.event.CacheLoadCompleted(cachesize, - len(self.depends_cache)), - self.data) + return len(self.depends_cache) def parse(self, filename, appends): """Parse the specified filename, returning the recipe information"""
@@ -682,7 +696,7 @@ class Cache(NoCache): for cache_class in self.caches_array: cache_class_name = cache_class.__name__ - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + cachefile = self.getCacheFile(cache_class.cachefile) with open(cachefile, "wb") as f: p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL) p.dump(__cache_version__)
@@ -701,7 +715,7 @@ class Cache(NoCache): return bb.parse.cached_mtime_noerror(cachefile) def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None): - if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): + if cacheData is not None and isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): cacheData.add_from_recipeinfo(filename, info_array) if watcher:
@@ -727,6 +741,65 @@ class Cache(NoCache): info_array.append(cache_class(realfn, data)) self.add_info(file_name, info_array, cacheData, parsed) +class MulticonfigCache(Mapping): + def __init__(self, databuilder, data_hash, caches_array): + def progress(p): + nonlocal current_progress + nonlocal previous_progress + nonlocal previous_percent + nonlocal cachesize + + current_progress = previous_progress + p + + if current_progress > cachesize: + # we might have calculated incorrect total size because a file + # might've been written out just after we checked its size + cachesize = current_progress + current_percent = 100 * current_progress / cachesize + if current_percent > previous_percent: + previous_percent = current_percent + bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize), + databuilder.data) + + + cachesize = 0 + current_progress = 0 + previous_progress = 0 + previous_percent = 0 + self.__caches = {} + + for mc, mcdata in databuilder.mcdata.items(): + self.__caches[mc] = Cache(databuilder, mc, data_hash, caches_array) + + cachesize += self.__caches[mc].cachesize() + + bb.event.fire(bb.event.CacheLoadStarted(cachesize), databuilder.data) + loaded = 0 + + for c in self.__caches.values(): + loaded += c.prepare_cache(progress) + previous_progress = current_progress + + # Note: depends cache number is corresponding to the parsing file numbers. + # The same file has several caches, still regarded as one item in the cache + bb.event.fire(bb.event.CacheLoadCompleted(cachesize, loaded), databuilder.data) + + def __len__(self): + return len(self.__caches) + + def __getitem__(self, key): + return self.__caches[key] + + def __contains__(self, key): + return key in self.__caches + + def __iter__(self): + for k in self.__caches: + yield k + + def keys(self): + return self.__caches[key] + def init(cooker): """
bitbake/lib/bb/cooker.py +25 -17
108 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH 3/8 bitbake: lib: Add support for Logging Adapters
In reply to Joshua WattCreates a BBLoggingAdapter class that is monkey patched in place of the logginer.LoggingAdapter. The new adapter is compatible with the BBLogger class API, allowing adapters to be created for bitbake loggers. A new BBLoggerMixin class is used to reduce code duplication between the BBLogger and BBLoggerAdapter classes.
--- bitbake/lib/bb/__init__.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-)
bitbake/lib/bb/__init__.py +18 -4
@@ -35,12 +35,14 @@ class NullHandler(logging.Handler): def emit(self, record): pass -Logger = logging.getLoggerClass() -class BBLogger(Logger): - def __init__(self, name): +class BBLoggerMixin(object): + def __init__(self, *args, **kwargs): + # Does nothing to allow calling super() from derived classes + pass + + def setup_bblogger(self, name): if name.split(".")[0] == "BitBake": self.debug = self.bbdebug - Logger.__init__(self, name) def bbdebug(self, level, msg, *args, **kwargs): loglevel = logging.DEBUG - level + 1
@@ -60,10 +62,22 @@ class BBLogger(Logger): def verbnote(self, msg, *args, **kwargs): return self.log(logging.INFO + 2, msg, *args, **kwargs) +Logger = logging.getLoggerClass() +class BBLogger(Logger, BBLoggerMixin): + def __init__(self, name, *args, **kwargs): + self.setup_bblogger(name) + super().__init__(name, *args, **kwargs) logging.raiseExceptions = False logging.setLoggerClass(BBLogger) +class BBLoggerAdapter(logging.LoggerAdapter, BBLoggerMixin): + def __init__(self, logger, *args, **kwargs): + self.setup_bblogger(logger.name) + super().__init__(logger, *args, **kwargs) + +logging.LoggerAdapter = BBLoggerAdapter + logger = logging.getLogger("BitBake") logger.addHandler(NullHandler()) logger.setLevel(logging.DEBUG - 2)
Signature · 1 line
2.17.1
PATCH 4/8 bitbake: lib: Add PrefixLoggerAdapter helper
In reply to Joshua WattAdds a helper logger adapter to add a prefix to all log messages. This is useful to distinguish log messages between multiple instances of a object.
--- bitbake/lib/bb/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+)
bitbake/lib/bb/__init__.py +8 -0
@@ -84,6 +84,14 @@ logger.setLevel(logging.DEBUG - 2) mainlogger = logging.getLogger("BitBake.Main") +class PrefixLoggerAdapter(logging.LoggerAdapter): + def __init__(self, prefix, logger): + super().__init__(logger, {}) + self.__msg_prefix = prefix + + def process(self, msg, kwargs): + return "%s%s" %(self.__msg_prefix, msg), kwargs + # This has to be imported after the setLoggerClass, as the import of bb.msg # can result in construction of the various loggers. import bb.msg
Signature · 1 line
2.17.1
PATCH 5/8 bitbake: cache: Improve logging
In reply to Joshua WattImproves the logging of Cache objects by prefixing the log messages with the multiconfig name of the cache, so as to distinguish between multiple instances of the class. Also adds a more log messages.
--- bitbake/lib/bb/cache.py | 65 ++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 30 deletions(-)
bitbake/lib/bb/cache.py +35 -30
@@ -21,6 +21,7 @@ import logging import pickle from collections import defaultdict, Mapping import bb.utils +from bb import PrefixLoggerAdapter import re logger = logging.getLogger("BitBake.Cache")
@@ -365,6 +366,7 @@ class Cache(NoCache): # It will be used later for deciding whether we # need extra cache file dump/load support self.mc = mc + self.logger = PrefixLoggerAdapter("Cache: %s: " % (mc if mc else "default"), logger) self.caches_array = caches_array self.cachedir = data.getVar("CACHE") self.clean = set()
@@ -377,8 +379,8 @@ class Cache(NoCache): if self.cachedir in [None, '']: self.has_cache = False - logger.info("Not using a cache. " - "Set CACHE = <directory> to enable.") + self.logger.info("Not using a cache. " + "Set CACHE = <directory> to enable.") return self.has_cache = True
@@ -394,21 +396,23 @@ class Cache(NoCache): self.cachefile = self.getCacheFile("bb_cache.dat") - logger.debug(1, "Cache dir: %s", self.cachedir) + self.logger.debug(1, "Cache dir: %s", self.cachedir) bb.utils.mkdirhier(self.cachedir) cache_ok = True if self.caches_array: for cache_class in self.caches_array: cachefile = self.getCacheFile(cache_class.cachefile) - cache_ok = cache_ok and os.path.exists(cachefile) + cache_exists = os.path.exists(cachefile) + self.logger.debug(2, "Checking if %s exists: %r", cachefile, cache_exists) + cache_ok = cache_ok and cache_exists cache_class.init_cacheData(self) if cache_ok: loaded = self.load_cachefile(progress) elif os.path.isfile(self.cachefile): - logger.info("Out of date cache found, rebuilding...") + self.logger.info("Out of date cache found, rebuilding...") else: - logger.debug(1, "Cache file %s not found, building..." % self.cachefile) + self.logger.debug(1, "Cache file %s not found, building..." % self.cachefile) # We don't use the symlink, its just for debugging convinience if self.mc:
@@ -447,7 +451,7 @@ class Cache(NoCache): for cache_class in self.caches_array: cachefile = self.getCacheFile(cache_class.cachefile) - logger.debug(1, 'Loading cache file: %s' % cachefile) + self.logger.debug(1, 'Loading cache file: %s' % cachefile) with open(cachefile, "rb") as cachefile: pickled = pickle.Unpickler(cachefile) # Check cache version information
@@ -455,14 +459,14 @@ class Cache(NoCache): cache_ver = pickled.load() bitbake_ver = pickled.load() except Exception: - logger.info('Invalid cache, rebuilding...') + self.logger.info('Invalid cache, rebuilding...') return if cache_ver != __cache_version__: - logger.info('Cache version mismatch, rebuilding...') + self.logger.info('Cache version mismatch, rebuilding...') return elif bitbake_ver != bb.__version__: - logger.info('Bitbake version mismatch, rebuilding...') + self.logger.info('Bitbake version mismatch, rebuilding...') return # Load the rest of the cache file
@@ -494,7 +498,7 @@ class Cache(NoCache): def parse(self, filename, appends): """Parse the specified filename, returning the recipe information""" - logger.debug(1, "Parsing %s", filename) + self.logger.debug(1, "Parsing %s", filename) infos = [] datastores = self.load_bbfile(filename, appends) depends = []
@@ -548,7 +552,7 @@ class Cache(NoCache): cached, infos = self.load(fn, appends) for virtualfn, info_array in infos: if info_array[0].skipped: - logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason) + self.logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason) skipped += 1 else: self.add_info(virtualfn, info_array, cacheData, not cached)
@@ -584,21 +588,21 @@ class Cache(NoCache): # File isn't in depends_cache if not fn in self.depends_cache: - logger.debug(2, "Cache: %s is not cached", fn) + self.logger.debug(2, "%s is not cached", fn) return False mtime = bb.parse.cached_mtime_noerror(fn) # Check file still exists if mtime == 0: - logger.debug(2, "Cache: %s no longer exists", fn) + self.logger.debug(2, "%s no longer exists", fn) self.remove(fn) return False info_array = self.depends_cache[fn] # Check the file's timestamp if mtime != info_array[0].timestamp: - logger.debug(2, "Cache: %s changed", fn) + self.logger.debug(2, "%s changed", fn) self.remove(fn) return False
@@ -609,14 +613,14 @@ class Cache(NoCache): fmtime = bb.parse.cached_mtime_noerror(f) # Check if file still exists if old_mtime != 0 and fmtime == 0: - logger.debug(2, "Cache: %s's dependency %s was removed", - fn, f) + self.logger.debug(2, "%s's dependency %s was removed", + fn, f) self.remove(fn) return False if (fmtime != old_mtime): - logger.debug(2, "Cache: %s's dependency %s changed", - fn, f) + self.logger.debug(2, "%s's dependency %s changed", + fn, f) self.remove(fn) return False
@@ -632,14 +636,14 @@ class Cache(NoCache): continue f, exist = f.split(":") if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)): - logger.debug(2, "Cache: %s's file checksum list file %s changed", - fn, f) + self.logger.debug(2, "%s's file checksum list file %s changed", + fn, f) self.remove(fn) return False if tuple(appends) != tuple(info_array[0].appends): - logger.debug(2, "Cache: appends for %s changed", fn) - logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) + self.logger.debug(2, "appends for %s changed", fn) + self.logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) self.remove(fn) return False
@@ -648,10 +652,10 @@ class Cache(NoCache): virtualfn = variant2virtual(fn, cls) self.clean.add(virtualfn) if virtualfn not in self.depends_cache: - logger.debug(2, "Cache: %s is not cached", virtualfn) + self.logger.debug(2, "%s is not cached", virtualfn) invalid = True elif len(self.depends_cache[virtualfn]) != len(self.caches_array): - logger.debug(2, "Cache: Extra caches missing for %s?" % virtualfn) + self.logger.debug(2, "Extra caches missing for %s?" % virtualfn) invalid = True # If any one of the variants is not present, mark as invalid for all
@@ -659,10 +663,10 @@ class Cache(NoCache): for cls in info_array[0].variants: virtualfn = variant2virtual(fn, cls) if virtualfn in self.clean: - logger.debug(2, "Cache: Removing %s from cache", virtualfn) + self.logger.debug(2, "Removing %s from cache", virtualfn) self.clean.remove(virtualfn) if fn in self.clean: - logger.debug(2, "Cache: Marking %s as not clean", fn) + self.logger.debug(2, "Marking %s as not clean", fn) self.clean.remove(fn) return False
@@ -675,10 +679,10 @@ class Cache(NoCache): Called from the parser in error cases """ if fn in self.depends_cache: - logger.debug(1, "Removing %s from cache", fn) + self.logger.debug(1, "Removing %s from cache", fn) del self.depends_cache[fn] if fn in self.clean: - logger.debug(1, "Marking %s as unclean", fn) + self.logger.debug(1, "Marking %s as unclean", fn) self.clean.remove(fn) def sync(self):
@@ -691,12 +695,13 @@ class Cache(NoCache): return if self.cacheclean: - logger.debug(2, "Cache is clean, not saving.") + self.logger.debug(2, "Cache is clean, not saving.") return for cache_class in self.caches_array: cache_class_name = cache_class.__name__ cachefile = self.getCacheFile(cache_class.cachefile) + self.logger.debug(2, "Writing %s", cachefile) with open(cachefile, "wb") as f: p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL) p.dump(__cache_version__)
Signature · 1 line
2.17.1
PATCH 6/8 bitbake: cache: Cache size optimization
In reply to Joshua WattNow that there is a cache object per multiconfig, it is not necessary for each cache object to parse all other multiconfigs. Only the base configuration and the specific multiconfig that the cache is tracking is required. This significantly reduces the number of files parsed and the size of each cache when there are multiple multiconfigs enabled.
This still doesn't get quite to the ideal case, since each multiconfig cache still has a independent copy of the base cache, but removing this is more difficult and invasive.
--- bitbake/lib/bb/cache.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-)
bitbake/lib/bb/cache.py +6 -3
@@ -328,7 +328,7 @@ class NoCache(object): bb_data = self.load_bbfile(virtualfn, appends, virtonly=True) return bb_data[virtual] - def load_bbfile(self, bbfile, appends, virtonly = False): + def load_bbfile(self, bbfile, appends, virtonly = False, mclist = None): """ Load and parse one .bb build file Return the data and whether parsing resulted in the file being skipped
@@ -344,7 +344,10 @@ class NoCache(object): bb_data = self.data.createCopy() datastores = parse_recipe(bb_data, bbfile, appends) - for mc in self.databuilder.mcdata: + if mclist is None: + mclist = self.databuilder.mcdata.keys() + + for mc in mclist: if not mc: continue bb_data = self.databuilder.mcdata[mc].createCopy()
@@ -500,7 +503,7 @@ class Cache(NoCache): """Parse the specified filename, returning the recipe information""" self.logger.debug(1, "Parsing %s", filename) infos = [] - datastores = self.load_bbfile(filename, appends) + datastores = self.load_bbfile(filename, appends, mclist=(self.mc,)) depends = [] variants = [] # Process the "real" fn last so we can store variants list
Signature · 1 line
2.17.1
PATCH 7/8 bitbake: tests: Add tests for BBMASK in multiconfig
In reply to Joshua WattAdds a test to validate that multiconfigs can independently mask off recipes by setting BBMASK. See the test description for further information about how the test works.
--- .../lib/bb/tests/runqueue-tests/conf/bitbake.conf | 3 ++- .../runqueue-tests/conf/multiconfig/mc1.conf | 1 + .../runqueue-tests/conf/multiconfig/mc2.conf | 1 + .../runqueue-tests/recipes/fails-mc/fails-mc1.bb | 5 +++++ .../runqueue-tests/recipes/fails-mc/fails-mc2.bb | 4 ++++ bitbake/lib/bb/tests/runqueue.py | 15 +++++++++++++++ 6 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb
bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf +2 -1
@@ -1,7 +1,8 @@ CACHE = "${TOPDIR}/cache" THISDIR = "${@os.path.dirname(d.getVar('FILE'))}" COREBASE := "${@os.path.normpath(os.path.dirname(d.getVar('FILE')+'/../../'))}" -BBFILES = "${COREBASE}/recipes/*.bb" +EXTRA_BBFILES ?= "" +BBFILES = "${COREBASE}/recipes/*.bb ${EXTRA_BBFILES}" PROVIDES = "${PN}" PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}" PF = "${BB_CURRENT_MC}:${PN}"
bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf +1 -0
@@ -1 +1,2 @@ TMPDIR = "${TOPDIR}/mc1/" +BBMASK += "recipes/fails-mc/fails-mc1.bb"
bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf +1 -0
@@ -1 +1,2 @@ TMPDIR = "${TOPDIR}/mc2/" +BBMASK += "recipes/fails-mc/fails-mc2.bb"
bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb new file +5 -0
@@ -0,0 +1,5 @@ +python () { + if d.getVar("BB_CURRENT_MC") == "mc1": + bb.fatal("Multiconfig is mc1") +} +
bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb new file +4 -0
@@ -0,0 +1,4 @@ +python () { + if d.getVar("BB_CURRENT_MC") == "mc2": + bb.fatal("Multiconfig is mc2") +}
bitbake/lib/bb/tests/runqueue.py +15 -0
@@ -232,6 +232,21 @@ class RunQueueTests(unittest.TestCase): expected.remove(x) self.assertEqual(set(tasks), set(expected)) + def test_multiconfig_bbmask(self): + # This test validates that multiconfigs can independently mask off + # recipes they do not want with BBMASK. It works by having recipes + # that will fail to parse for mc1 and mc2, then making each multiconfig + # build the one that does parse. This ensures that the recipes are in + # each multiconfigs BBFILES, but each is masking only the one that + # doesn't parse + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + extraenv = { + "BBMULTICONFIG" : "mc1 mc2", + "BB_SIGNATURE_HANDLER" : "basic", + "EXTRA_BBFILES": "${COREBASE}/recipes/fails-mc/*.bb", + } + cmd = ["bitbake", "mc:mc1:fails-mc2", "mc:mc2:fails-mc1"] + self.run_bitbakecmd(cmd, tempdir, "", extraenv=extraenv) @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required') def test_hashserv_single(self):
Signature · 1 line
2.17.1
PATCH 8/8 bitbake: command: Move split_mc_pn to runqueue
In reply to Joshua WattAll of the other multiconfig splitting functions are located in runqueue so move the function to split a pn/fn there also so that its easier to see them all together. Fixes a case where the findBestProvider() command wasn't working for multiconfig because it was looking for a prefix of "multiconfig:" instead of the newer "mc:"
--- bitbake/lib/bb/command.py | 8 +------- bitbake/lib/bb/runqueue.py | 6 ++++++ 2 files changed, 7 insertions(+), 7 deletions(-)
bitbake/lib/bb/command.py +1 -7
@@ -138,12 +138,6 @@ class Command: def reset(self): self.remotedatastores = bb.remotedata.RemoteDatastores(self.cooker) -def split_mc_pn(pn): - if pn.startswith("multiconfig:"): - _, mc, pn = pn.split(":", 2) - return (mc, pn) - return ('', pn) - class CommandsSync: """ A class of synchronous commands
@@ -438,7 +432,7 @@ class CommandsSync: findProviders.readonly = True def findBestProvider(self, command, params): - (mc, pn) = split_mc_pn(params[0]) + (mc, pn) = bb.runqueue.split_pn(params[0]) return command.cooker.findBestProvider(pn, mc) findBestProvider.readonly = True
bitbake/lib/bb/runqueue.py +6 -0
@@ -46,6 +46,12 @@ def split_tid(tid): (mc, fn, taskname, _) = split_tid_mcfn(tid) return (mc, fn, taskname) +def split_pn(pn): + if pn.startswith("mc:"): + _, mc, pn = pn.split(":", 2) + return (mc, pn) + return ('', pn) + def split_tid_mcfn(tid): if tid.startswith('mc:'): elems = tid.split(':')
Signature · 1 line
2.17.1
v2 cover Add support for per-multiconfig BBMASK
In reply to Joshua WattSplits apart the bitbake configuration data and caches so that each defined multiconfig can specify as separate BBMASK independently of the other multiconfigs. This can take multiple forms, such as a BBMASK in a multiconfig .conf file, or BBMASK assignment based on an override (see linked bugzilla for an example).
The most noticeable change is that bitbake now keeps a parsing cache file for each multiconfig instead of a single parsing cache file that caches all multiconfigs.
V2: Fixes some syntax errors in tinfoil
Joshua Watt (8):
bitbake: cooker: Split file collections per multiconfig
bitbake: cache: Use multiconfig aware caches
bitbake: lib: Add support for Logging Adapters
bitbake: lib: Add PrefixLoggerAdapter helper
bitbake: cache: Improve logging
bitbake: cache: Cache size optimization
bitbake: tests: Add tests for BBMASK in multiconfig
bitbake: command: Move split_mc_pn to runqueue
bitbake/lib/bb/__init__.py | 30 ++- bitbake/lib/bb/cache.py | 219 ++++++++++++------ bitbake/lib/bb/command.py | 39 ++-- bitbake/lib/bb/cooker.py | 165 +++++++------ bitbake/lib/bb/runqueue.py | 17 +- .../bb/tests/runqueue-tests/conf/bitbake.conf | 3 +- .../runqueue-tests/conf/multiconfig/mc1.conf | 1 + .../runqueue-tests/conf/multiconfig/mc2.conf | 1 + .../recipes/fails-mc/fails-mc1.bb | 5 + .../recipes/fails-mc/fails-mc2.bb | 4 + bitbake/lib/bb/tests/runqueue.py | 15 ++ bitbake/lib/bb/tinfoil.py | 26 ++- bitbake/lib/bblayers/query.py | 4 +- 13 files changed, 356 insertions(+), 173 deletions(-) create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb
Signature · 1 line
2.26.2
PATCH v2 1/8 bitbake: cooker: Split file collections per multiconfig
Splits the cooker to track a collection per multiconfig instead of a single collection for all multiconfigs. Practically speaking, this allows each multiconfigs to each have different BBMASKs that apply to it instead of each one using the mask specified in the base configuration.
--- bitbake/lib/bb/cache.py | 2 +- bitbake/lib/bb/command.py | 31 ++++++-- bitbake/lib/bb/cooker.py | 139 ++++++++++++++++++++-------------- bitbake/lib/bb/runqueue.py | 11 +-- bitbake/lib/bb/tinfoil.py | 26 ++++--- bitbake/lib/bblayers/query.py | 4 +- 6 files changed, 129 insertions(+), 84 deletions(-)
bitbake/lib/bb/cache.py +1 -1
@@ -623,7 +623,7 @@ class Cache(NoCache): self.remove(fn) return False - if appends != info_array[0].appends: + if tuple(appends) != tuple(info_array[0].appends): logger.debug(2, "Cache: appends for %s changed", fn) logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) self.remove(fn)
bitbake/lib/bb/command.py +24 -7
@@ -232,7 +232,11 @@ class CommandsSync: def matchFile(self, command, params): fMatch = params[0] - return command.cooker.matchFile(fMatch) + try: + mc = params[0] + except IndexError: + mc = '' + return command.cooker.matchFile(fMatch, mc) matchFile.needconfig = False def getUIHandlerNum(self, command, params):
@@ -396,21 +400,33 @@ class CommandsSync: # Return list sorted by reverse priority order import bb.cache skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), - key=lambda x: (-command.cooker.collection.calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0]))) + key=lambda x: (-command.cooker.collections[mc].calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0]))) return list(skipdict.items()) getSkippedRecipes.readonly = True def getOverlayedRecipes(self, command, params): - return list(command.cooker.collection.overlayed.items()) + try: + mc = params[0] + except IndexError: + mc = '' + return list(command.cooker.collections[mc].overlayed.items()) getOverlayedRecipes.readonly = True def getFileAppends(self, command, params): fn = params[0] - return command.cooker.collection.get_file_appends(fn) + try: + mc = params[1] + except IndexError: + mc = '' + return command.cooker.collections[mc].get_file_appends(fn) getFileAppends.readonly = True def getAllAppends(self, command, params): - return command.cooker.collection.bbappends + try: + mc = params[0] + except IndexError: + mc = '' + return command.cooker.collections[mc].bbappends getAllAppends.readonly = True def findProviders(self, command, params):
@@ -496,6 +512,7 @@ class CommandsSync: for the recipe. """ fn = params[0] + mc = bb.runqueue.mc_from_tid(fn) appends = params[1] appendlist = params[2] if len(params) > 3:
@@ -507,7 +524,7 @@ class CommandsSync: if appendlist is not None: appendfiles = appendlist else: - appendfiles = command.cooker.collection.get_file_appends(fn) + appendfiles = command.cooker.collections[mc].get_file_appends(fn) else: appendfiles = [] # We are calling bb.cache locally here rather than on the server,
@@ -517,7 +534,7 @@ class CommandsSync: if config_data: # We have to use a different function here if we're passing in a datastore # NOTE: we took a copy above, so we don't do it here again - envdata = bb.cache.parse_recipe(config_data, fn, appendfiles)[''] + envdata = bb.cache.parse_recipe(config_data, fn, appendfiles, mc)[''] else: # Use the standard path parser = bb.cache.NoCache(command.cooker.databuilder)
bitbake/lib/bb/cooker.py +82 -57
268 lines not on this page · raw message
bitbake/lib/bb/runqueue.py +6 -5
@@ -1557,7 +1557,8 @@ class RunQueue: def rq_dump_sigfn(self, fn, options): bb_cache = bb.cache.NoCache(self.cooker.databuilder) - the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn)) + mc = bb.runqueue.mc_from_tid(fn) + the_data = bb_cache.loadDataFull(fn, self.cooker.collections[mc].get_file_appends(fn)) siggen = bb.parse.siggen dataCaches = self.rqdata.dataCaches siggen.dump_sigfn(fn, dataCaches, options)
@@ -2042,10 +2043,10 @@ class RunQueueExecute: if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: if not mc in self.rq.fakeworker: self.rq.start_fakeworker(self, mc) - self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") + self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") self.rq.fakeworker[mc].process.stdin.flush() else: - self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") + self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") self.rq.worker[mc].process.stdin.flush() self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
@@ -2129,10 +2130,10 @@ class RunQueueExecute: self.rq.state = runQueueFailed self.stats.taskFailed() return True - self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") + self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") self.rq.fakeworker[mc].process.stdin.flush() else: - self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") + self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") self.rq.worker[mc].process.stdin.flush() self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
bitbake/lib/bb/tinfoil.py +14 -12
@@ -117,15 +117,16 @@ class TinfoilCookerAdapter: class TinfoilCookerCollectionAdapter: """ cooker.collection adapter """ - def __init__(self, tinfoil): + def __init__(self, tinfoil, mc=''): self.tinfoil = tinfoil + self.mc = mc def get_file_appends(self, fn): - return self.tinfoil.get_file_appends(fn) + return self.tinfoil.get_file_appends(fn, self.mc) def __getattr__(self, name): if name == 'overlayed': - return self.tinfoil.get_overlayed_recipes() + return self.tinfoil.get_overlayed_recipes(self.mc) elif name == 'bbappends': - return self.tinfoil.run_command('getAllAppends') + return self.tinfoil.run_command('getAllAppends', self.mc) else: raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
@@ -185,10 +186,11 @@ class TinfoilCookerAdapter: def __init__(self, tinfoil): self.tinfoil = tinfoil - self.collection = self.TinfoilCookerCollectionAdapter(tinfoil) + self.collections = {} self.recipecaches = {} self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil) for mc in (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split(): + self.collections[mc] = self.TinfoilCookerCollectionAdapter(tinfoil) self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc) self._cache = {} def __getattr__(self, name):
@@ -196,7 +198,7 @@ class TinfoilCookerAdapter: if name in self._cache: return self._cache[name] elif name == 'skiplist': - attrvalue = self.tinfoil.get_skipped_recipes() + attrvalue = self.tinfoil.get_skipped_recipes(self.mc) elif name == 'bbfile_config_priorities': ret = self.tinfoil.run_command('getLayerPriorities') bbfile_config_priorities = []
@@ -492,18 +494,18 @@ class Tinfoil: raise Exception('Not connected to server (did you call .prepare()?)') return self.server_connection.events.waitEvent(timeout) - def get_overlayed_recipes(self): + def get_overlayed_recipes(self, mc=''): """ Find recipes which are overlayed (i.e. where recipes exist in multiple layers) """ - return defaultdict(list, self.run_command('getOverlayedRecipes')) + return defaultdict(list, self.run_command('getOverlayedRecipes', mc)) - def get_skipped_recipes(self): + def get_skipped_recipes(self, mc=''): """ Find recipes which were skipped (i.e. SkipRecipe was raised during parsing). """ - return OrderedDict(self.run_command('getSkippedRecipes')) + return OrderedDict(self.run_command('getSkippedRecipes', mc)) def get_all_providers(self, mc=''): return defaultdict(list, self.run_command('allProviders', mc))
@@ -534,11 +536,11 @@ class Tinfoil: raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn) return best[3] - def get_file_appends(self, fn): + def get_file_appends(self, fn, mc=''): """ Find the bbappends for a recipe file """ - return self.run_command('getFileAppends', fn) + return self.run_command('getFileAppends', fn, mc) def all_recipes(self, mc='', sort=True): """
bitbake/lib/bblayers/query.py +2 -2
@@ -320,12 +320,12 @@ Lists recipes with the bbappends that apply to them as subitems. def get_appends_for_files(self, filenames): appended, notappended = [], [] for filename in filenames: - _, cls, _ = bb.cache.virtualfn2realfn(filename) + _, cls, mc = bb.cache.virtualfn2realfn(filename) if cls: continue basename = os.path.basename(filename) - appends = self.tinfoil.cooker.collection.get_file_appends(basename) + appends = self.tinfoil.cooker.collections[mc].get_file_appends(basename) if appends: appended.append((basename, list(appends))) else:
Signature · 1 line
2.26.2
PATCH v2 2/8 bitbake: cache: Use multiconfig aware caches
In reply to Joshua WattSplits the parsing cache to maintain one cache per multiconfig instead of one global cache. This is necessary now that the files and appends can vary for each multiconfig. A bb.cache.MulticonfigCache dictionary-like proxy object is created instead of a single bb.cache.Cache object. This object will create and properly initialize bb.cache.Cache object for each multiconfig, and each of these caches has a dedicated cache file with a name based on the multiconfig.
--- bitbake/lib/bb/cache.py | 143 +++++++++++++++++++++++++++++---------- bitbake/lib/bb/cooker.py | 42 +++++++----- 2 files changed, 133 insertions(+), 52 deletions(-)
bitbake/lib/bb/cache.py +108 -35
@@ -19,7 +19,7 @@ import os import logging import pickle -from collections import defaultdict +from collections import defaultdict, Mapping import bb.utils import re
@@ -27,8 +27,11 @@ logger = logging.getLogger("BitBake.Cache") __cache_version__ = "152" -def getCacheFile(path, filename, data_hash): - return os.path.join(path, filename + "." + data_hash) +def getCacheFile(path, filename, mc, data_hash): + mcspec = '' + if mc: + mcspec = ".%s" % mc + return os.path.join(path, filename + mcspec + "." + data_hash) # RecipeInfoCommon defines common data retrieving methods # from meta data for caches. CoreRecipeInfo as well as other
@@ -354,14 +357,14 @@ class Cache(NoCache): """ BitBake Cache implementation """ - - def __init__(self, databuilder, data_hash, caches_array): + def __init__(self, databuilder, mc, data_hash, caches_array): super().__init__(databuilder) data = databuilder.data # Pass caches_array information into Cache Constructor # It will be used later for deciding whether we # need extra cache file dump/load support + self.mc = mc self.caches_array = caches_array self.cachedir = data.getVar("CACHE") self.clean = set()
@@ -379,7 +382,17 @@ class Cache(NoCache): return self.has_cache = True - self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash) + + def getCacheFile(self, cachefile): + return getCacheFile(self.cachedir, cachefile, self.mc, self.data_hash) + + def prepare_cache(self, progress): + if not self.has_cache: + return 0 + + loaded = 0 + + self.cachefile = self.getCacheFile("bb_cache.dat") logger.debug(1, "Cache dir: %s", self.cachedir) bb.utils.mkdirhier(self.cachedir)
@@ -387,18 +400,22 @@ class Cache(NoCache): cache_ok = True if self.caches_array: for cache_class in self.caches_array: - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + cachefile = self.getCacheFile(cache_class.cachefile) cache_ok = cache_ok and os.path.exists(cachefile) cache_class.init_cacheData(self) if cache_ok: - self.load_cachefile() + loaded = self.load_cachefile(progress) elif os.path.isfile(self.cachefile): logger.info("Out of date cache found, rebuilding...") else: logger.debug(1, "Cache file %s not found, building..." % self.cachefile) # We don't use the symlink, its just for debugging convinience - symlink = os.path.join(self.cachedir, "bb_cache.dat") + if self.mc: + symlink = os.path.join(self.cachedir, "bb_cache.dat.%s" % self.mc) + else: + symlink = os.path.join(self.cachedir, "bb_cache.dat") + if os.path.exists(symlink): bb.utils.remove(symlink) try:
@@ -406,21 +423,30 @@ class Cache(NoCache): except OSError: pass - def load_cachefile(self): - cachesize = 0 - previous_progress = 0 - previous_percent = 0 + return loaded + + def cachesize(self): + if not self.has_cache: + return 0 - # Calculate the correct cachesize of all those cache files + cachesize = 0 for cache_class in self.caches_array: - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) - with open(cachefile, "rb") as cachefile: - cachesize += os.fstat(cachefile.fileno()).st_size + cachefile = self.getCacheFile(cache_class.cachefile) + try: + with open(cachefile, "rb") as cachefile: + cachesize += os.fstat(cachefile.fileno()).st_size + except FileNotFoundError: + pass - bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data) + return cachesize + + def load_cachefile(self, progress): + cachesize = self.cachesize() + previous_progress = 0 + previous_percent = 0 for cache_class in self.caches_array: - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + cachefile = self.getCacheFile(cache_class.cachefile) logger.debug(1, 'Loading cache file: %s' % cachefile) with open(cachefile, "rb") as cachefile: pickled = pickle.Unpickler(cachefile)
@@ -460,23 +486,11 @@ class Cache(NoCache): self.depends_cache[key] = [value] # only fire events on even percentage boundaries current_progress = cachefile.tell() + previous_progress - if current_progress > cachesize: - # we might have calculated incorrect total size because a file - # might've been written out just after we checked its size - cachesize = current_progress - current_percent = 100 * current_progress / cachesize - if current_percent > previous_percent: - previous_percent = current_percent - bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize), - self.data) + progress(cachefile.tell() + previous_progress) previous_progress += current_progress - # Note: depends cache number is corresponding to the parsing file numbers. - # The same file has several caches, still regarded as one item in the cache - bb.event.fire(bb.event.CacheLoadCompleted(cachesize, - len(self.depends_cache)), - self.data) + return len(self.depends_cache) def parse(self, filename, appends): """Parse the specified filename, returning the recipe information"""
@@ -682,7 +696,7 @@ class Cache(NoCache): for cache_class in self.caches_array: cache_class_name = cache_class.__name__ - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + cachefile = self.getCacheFile(cache_class.cachefile) with open(cachefile, "wb") as f: p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL) p.dump(__cache_version__)
@@ -701,7 +715,7 @@ class Cache(NoCache): return bb.parse.cached_mtime_noerror(cachefile) def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None): - if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): + if cacheData is not None and isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): cacheData.add_from_recipeinfo(filename, info_array) if watcher:
@@ -727,6 +741,65 @@ class Cache(NoCache): info_array.append(cache_class(realfn, data)) self.add_info(file_name, info_array, cacheData, parsed) +class MulticonfigCache(Mapping): + def __init__(self, databuilder, data_hash, caches_array): + def progress(p): + nonlocal current_progress + nonlocal previous_progress + nonlocal previous_percent + nonlocal cachesize + + current_progress = previous_progress + p + + if current_progress > cachesize: + # we might have calculated incorrect total size because a file + # might've been written out just after we checked its size + cachesize = current_progress + current_percent = 100 * current_progress / cachesize + if current_percent > previous_percent: + previous_percent = current_percent + bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize), + databuilder.data) + + + cachesize = 0 + current_progress = 0 + previous_progress = 0 + previous_percent = 0 + self.__caches = {} + + for mc, mcdata in databuilder.mcdata.items(): + self.__caches[mc] = Cache(databuilder, mc, data_hash, caches_array) + + cachesize += self.__caches[mc].cachesize() + + bb.event.fire(bb.event.CacheLoadStarted(cachesize), databuilder.data) + loaded = 0 + + for c in self.__caches.values(): + loaded += c.prepare_cache(progress) + previous_progress = current_progress + + # Note: depends cache number is corresponding to the parsing file numbers. + # The same file has several caches, still regarded as one item in the cache + bb.event.fire(bb.event.CacheLoadCompleted(cachesize, loaded), databuilder.data) + + def __len__(self): + return len(self.__caches) + + def __getitem__(self, key): + return self.__caches[key] + + def __contains__(self, key): + return key in self.__caches + + def __iter__(self): + for k in self.__caches: + yield k + + def keys(self): + return self.__caches[key] + def init(cooker): """
bitbake/lib/bb/cooker.py +25 -17
108 lines not on this page · raw message
Signature · 1 line
2.26.2
PATCH v2 3/8 bitbake: lib: Add support for Logging Adapters
In reply to Joshua WattCreates a BBLoggingAdapter class that is monkey patched in place of the logginer.LoggingAdapter. The new adapter is compatible with the BBLogger class API, allowing adapters to be created for bitbake loggers. A new BBLoggerMixin class is used to reduce code duplication between the BBLogger and BBLoggerAdapter classes.
--- bitbake/lib/bb/__init__.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-)
bitbake/lib/bb/__init__.py +18 -4
@@ -35,12 +35,14 @@ class NullHandler(logging.Handler): def emit(self, record): pass -Logger = logging.getLoggerClass() -class BBLogger(Logger): - def __init__(self, name): +class BBLoggerMixin(object): + def __init__(self, *args, **kwargs): + # Does nothing to allow calling super() from derived classes + pass + + def setup_bblogger(self, name): if name.split(".")[0] == "BitBake": self.debug = self.bbdebug - Logger.__init__(self, name) def bbdebug(self, level, msg, *args, **kwargs): loglevel = logging.DEBUG - level + 1
@@ -60,10 +62,22 @@ class BBLogger(Logger): def verbnote(self, msg, *args, **kwargs): return self.log(logging.INFO + 2, msg, *args, **kwargs) +Logger = logging.getLoggerClass() +class BBLogger(Logger, BBLoggerMixin): + def __init__(self, name, *args, **kwargs): + self.setup_bblogger(name) + super().__init__(name, *args, **kwargs) logging.raiseExceptions = False logging.setLoggerClass(BBLogger) +class BBLoggerAdapter(logging.LoggerAdapter, BBLoggerMixin): + def __init__(self, logger, *args, **kwargs): + self.setup_bblogger(logger.name) + super().__init__(logger, *args, **kwargs) + +logging.LoggerAdapter = BBLoggerAdapter + logger = logging.getLogger("BitBake") logger.addHandler(NullHandler()) logger.setLevel(logging.DEBUG - 2)
Signature · 1 line
2.26.2
PATCH v2 4/8 bitbake: lib: Add PrefixLoggerAdapter helper
In reply to Joshua WattAdds a helper logger adapter to add a prefix to all log messages. This is useful to distinguish log messages between multiple instances of a object.
--- bitbake/lib/bb/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+)
bitbake/lib/bb/__init__.py +8 -0
@@ -84,6 +84,14 @@ logger.setLevel(logging.DEBUG - 2) mainlogger = logging.getLogger("BitBake.Main") +class PrefixLoggerAdapter(logging.LoggerAdapter): + def __init__(self, prefix, logger): + super().__init__(logger, {}) + self.__msg_prefix = prefix + + def process(self, msg, kwargs): + return "%s%s" %(self.__msg_prefix, msg), kwargs + # This has to be imported after the setLoggerClass, as the import of bb.msg # can result in construction of the various loggers. import bb.msg
Signature · 1 line
2.26.2
PATCH v2 5/8 bitbake: cache: Improve logging
In reply to Joshua WattImproves the logging of Cache objects by prefixing the log messages with the multiconfig name of the cache, so as to distinguish between multiple instances of the class. Also adds a more log messages.
--- bitbake/lib/bb/cache.py | 65 ++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 30 deletions(-)
bitbake/lib/bb/cache.py +35 -30
@@ -21,6 +21,7 @@ import logging import pickle from collections import defaultdict, Mapping import bb.utils +from bb import PrefixLoggerAdapter import re logger = logging.getLogger("BitBake.Cache")
@@ -365,6 +366,7 @@ class Cache(NoCache): # It will be used later for deciding whether we # need extra cache file dump/load support self.mc = mc + self.logger = PrefixLoggerAdapter("Cache: %s: " % (mc if mc else "default"), logger) self.caches_array = caches_array self.cachedir = data.getVar("CACHE") self.clean = set()
@@ -377,8 +379,8 @@ class Cache(NoCache): if self.cachedir in [None, '']: self.has_cache = False - logger.info("Not using a cache. " - "Set CACHE = <directory> to enable.") + self.logger.info("Not using a cache. " + "Set CACHE = <directory> to enable.") return self.has_cache = True
@@ -394,21 +396,23 @@ class Cache(NoCache): self.cachefile = self.getCacheFile("bb_cache.dat") - logger.debug(1, "Cache dir: %s", self.cachedir) + self.logger.debug(1, "Cache dir: %s", self.cachedir) bb.utils.mkdirhier(self.cachedir) cache_ok = True if self.caches_array: for cache_class in self.caches_array: cachefile = self.getCacheFile(cache_class.cachefile) - cache_ok = cache_ok and os.path.exists(cachefile) + cache_exists = os.path.exists(cachefile) + self.logger.debug(2, "Checking if %s exists: %r", cachefile, cache_exists) + cache_ok = cache_ok and cache_exists cache_class.init_cacheData(self) if cache_ok: loaded = self.load_cachefile(progress) elif os.path.isfile(self.cachefile): - logger.info("Out of date cache found, rebuilding...") + self.logger.info("Out of date cache found, rebuilding...") else: - logger.debug(1, "Cache file %s not found, building..." % self.cachefile) + self.logger.debug(1, "Cache file %s not found, building..." % self.cachefile) # We don't use the symlink, its just for debugging convinience if self.mc:
@@ -447,7 +451,7 @@ class Cache(NoCache): for cache_class in self.caches_array: cachefile = self.getCacheFile(cache_class.cachefile) - logger.debug(1, 'Loading cache file: %s' % cachefile) + self.logger.debug(1, 'Loading cache file: %s' % cachefile) with open(cachefile, "rb") as cachefile: pickled = pickle.Unpickler(cachefile) # Check cache version information
@@ -455,14 +459,14 @@ class Cache(NoCache): cache_ver = pickled.load() bitbake_ver = pickled.load() except Exception: - logger.info('Invalid cache, rebuilding...') + self.logger.info('Invalid cache, rebuilding...') return if cache_ver != __cache_version__: - logger.info('Cache version mismatch, rebuilding...') + self.logger.info('Cache version mismatch, rebuilding...') return elif bitbake_ver != bb.__version__: - logger.info('Bitbake version mismatch, rebuilding...') + self.logger.info('Bitbake version mismatch, rebuilding...') return # Load the rest of the cache file
@@ -494,7 +498,7 @@ class Cache(NoCache): def parse(self, filename, appends): """Parse the specified filename, returning the recipe information""" - logger.debug(1, "Parsing %s", filename) + self.logger.debug(1, "Parsing %s", filename) infos = [] datastores = self.load_bbfile(filename, appends) depends = []
@@ -548,7 +552,7 @@ class Cache(NoCache): cached, infos = self.load(fn, appends) for virtualfn, info_array in infos: if info_array[0].skipped: - logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason) + self.logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason) skipped += 1 else: self.add_info(virtualfn, info_array, cacheData, not cached)
@@ -584,21 +588,21 @@ class Cache(NoCache): # File isn't in depends_cache if not fn in self.depends_cache: - logger.debug(2, "Cache: %s is not cached", fn) + self.logger.debug(2, "%s is not cached", fn) return False mtime = bb.parse.cached_mtime_noerror(fn) # Check file still exists if mtime == 0: - logger.debug(2, "Cache: %s no longer exists", fn) + self.logger.debug(2, "%s no longer exists", fn) self.remove(fn) return False info_array = self.depends_cache[fn] # Check the file's timestamp if mtime != info_array[0].timestamp: - logger.debug(2, "Cache: %s changed", fn) + self.logger.debug(2, "%s changed", fn) self.remove(fn) return False
@@ -609,14 +613,14 @@ class Cache(NoCache): fmtime = bb.parse.cached_mtime_noerror(f) # Check if file still exists if old_mtime != 0 and fmtime == 0: - logger.debug(2, "Cache: %s's dependency %s was removed", - fn, f) + self.logger.debug(2, "%s's dependency %s was removed", + fn, f) self.remove(fn) return False if (fmtime != old_mtime): - logger.debug(2, "Cache: %s's dependency %s changed", - fn, f) + self.logger.debug(2, "%s's dependency %s changed", + fn, f) self.remove(fn) return False
@@ -632,14 +636,14 @@ class Cache(NoCache): continue f, exist = f.split(":") if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)): - logger.debug(2, "Cache: %s's file checksum list file %s changed", - fn, f) + self.logger.debug(2, "%s's file checksum list file %s changed", + fn, f) self.remove(fn) return False if tuple(appends) != tuple(info_array[0].appends): - logger.debug(2, "Cache: appends for %s changed", fn) - logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) + self.logger.debug(2, "appends for %s changed", fn) + self.logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) self.remove(fn) return False
@@ -648,10 +652,10 @@ class Cache(NoCache): virtualfn = variant2virtual(fn, cls) self.clean.add(virtualfn) if virtualfn not in self.depends_cache: - logger.debug(2, "Cache: %s is not cached", virtualfn) + self.logger.debug(2, "%s is not cached", virtualfn) invalid = True elif len(self.depends_cache[virtualfn]) != len(self.caches_array): - logger.debug(2, "Cache: Extra caches missing for %s?" % virtualfn) + self.logger.debug(2, "Extra caches missing for %s?" % virtualfn) invalid = True # If any one of the variants is not present, mark as invalid for all
@@ -659,10 +663,10 @@ class Cache(NoCache): for cls in info_array[0].variants: virtualfn = variant2virtual(fn, cls) if virtualfn in self.clean: - logger.debug(2, "Cache: Removing %s from cache", virtualfn) + self.logger.debug(2, "Removing %s from cache", virtualfn) self.clean.remove(virtualfn) if fn in self.clean: - logger.debug(2, "Cache: Marking %s as not clean", fn) + self.logger.debug(2, "Marking %s as not clean", fn) self.clean.remove(fn) return False
@@ -675,10 +679,10 @@ class Cache(NoCache): Called from the parser in error cases """ if fn in self.depends_cache: - logger.debug(1, "Removing %s from cache", fn) + self.logger.debug(1, "Removing %s from cache", fn) del self.depends_cache[fn] if fn in self.clean: - logger.debug(1, "Marking %s as unclean", fn) + self.logger.debug(1, "Marking %s as unclean", fn) self.clean.remove(fn) def sync(self):
@@ -691,12 +695,13 @@ class Cache(NoCache): return if self.cacheclean: - logger.debug(2, "Cache is clean, not saving.") + self.logger.debug(2, "Cache is clean, not saving.") return for cache_class in self.caches_array: cache_class_name = cache_class.__name__ cachefile = self.getCacheFile(cache_class.cachefile) + self.logger.debug(2, "Writing %s", cachefile) with open(cachefile, "wb") as f: p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL) p.dump(__cache_version__)
Signature · 1 line
2.26.2
PATCH v2 6/8 bitbake: cache: Cache size optimization
In reply to Joshua WattNow that there is a cache object per multiconfig, it is not necessary for each cache object to parse all other multiconfigs. Only the base configuration and the specific multiconfig that the cache is tracking is required. This significantly reduces the number of files parsed and the size of each cache when there are multiple multiconfigs enabled.
This still doesn't get quite to the ideal case, since each multiconfig cache still has a independent copy of the base cache, but removing this is more difficult and invasive.
--- bitbake/lib/bb/cache.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-)
bitbake/lib/bb/cache.py +6 -3
@@ -328,7 +328,7 @@ class NoCache(object): bb_data = self.load_bbfile(virtualfn, appends, virtonly=True) return bb_data[virtual] - def load_bbfile(self, bbfile, appends, virtonly = False): + def load_bbfile(self, bbfile, appends, virtonly = False, mclist = None): """ Load and parse one .bb build file Return the data and whether parsing resulted in the file being skipped
@@ -344,7 +344,10 @@ class NoCache(object): bb_data = self.data.createCopy() datastores = parse_recipe(bb_data, bbfile, appends) - for mc in self.databuilder.mcdata: + if mclist is None: + mclist = self.databuilder.mcdata.keys() + + for mc in mclist: if not mc: continue bb_data = self.databuilder.mcdata[mc].createCopy()
@@ -500,7 +503,7 @@ class Cache(NoCache): """Parse the specified filename, returning the recipe information""" self.logger.debug(1, "Parsing %s", filename) infos = [] - datastores = self.load_bbfile(filename, appends) + datastores = self.load_bbfile(filename, appends, mclist=(self.mc,)) depends = [] variants = [] # Process the "real" fn last so we can store variants list
Signature · 1 line
2.26.2
PATCH v2 7/8 bitbake: tests: Add tests for BBMASK in multiconfig
In reply to Joshua WattAdds a test to validate that multiconfigs can independently mask off recipes by setting BBMASK. See the test description for further information about how the test works.
--- .../lib/bb/tests/runqueue-tests/conf/bitbake.conf | 3 ++- .../runqueue-tests/conf/multiconfig/mc1.conf | 1 + .../runqueue-tests/conf/multiconfig/mc2.conf | 1 + .../runqueue-tests/recipes/fails-mc/fails-mc1.bb | 5 +++++ .../runqueue-tests/recipes/fails-mc/fails-mc2.bb | 4 ++++ bitbake/lib/bb/tests/runqueue.py | 15 +++++++++++++++ 6 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb
bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf +2 -1
@@ -1,7 +1,8 @@ CACHE = "${TOPDIR}/cache" THISDIR = "${@os.path.dirname(d.getVar('FILE'))}" COREBASE := "${@os.path.normpath(os.path.dirname(d.getVar('FILE')+'/../../'))}" -BBFILES = "${COREBASE}/recipes/*.bb" +EXTRA_BBFILES ?= "" +BBFILES = "${COREBASE}/recipes/*.bb ${EXTRA_BBFILES}" PROVIDES = "${PN}" PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}" PF = "${BB_CURRENT_MC}:${PN}"
bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf +1 -0
@@ -1 +1,2 @@ TMPDIR = "${TOPDIR}/mc1/" +BBMASK += "recipes/fails-mc/fails-mc1.bb"
bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf +1 -0
@@ -1 +1,2 @@ TMPDIR = "${TOPDIR}/mc2/" +BBMASK += "recipes/fails-mc/fails-mc2.bb"
bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb new file +5 -0
@@ -0,0 +1,5 @@ +python () { + if d.getVar("BB_CURRENT_MC") == "mc1": + bb.fatal("Multiconfig is mc1") +} +
bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb new file +4 -0
@@ -0,0 +1,4 @@ +python () { + if d.getVar("BB_CURRENT_MC") == "mc2": + bb.fatal("Multiconfig is mc2") +}
bitbake/lib/bb/tests/runqueue.py +15 -0
@@ -232,6 +232,21 @@ class RunQueueTests(unittest.TestCase): expected.remove(x) self.assertEqual(set(tasks), set(expected)) + def test_multiconfig_bbmask(self): + # This test validates that multiconfigs can independently mask off + # recipes they do not want with BBMASK. It works by having recipes + # that will fail to parse for mc1 and mc2, then making each multiconfig + # build the one that does parse. This ensures that the recipes are in + # each multiconfigs BBFILES, but each is masking only the one that + # doesn't parse + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + extraenv = { + "BBMULTICONFIG" : "mc1 mc2", + "BB_SIGNATURE_HANDLER" : "basic", + "EXTRA_BBFILES": "${COREBASE}/recipes/fails-mc/*.bb", + } + cmd = ["bitbake", "mc:mc1:fails-mc2", "mc:mc2:fails-mc1"] + self.run_bitbakecmd(cmd, tempdir, "", extraenv=extraenv) @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required') def test_hashserv_single(self):
Signature · 1 line
2.26.2
PATCH v2 8/8 bitbake: command: Move split_mc_pn to runqueue
In reply to Joshua WattAll of the other multiconfig splitting functions are located in runqueue so move the function to split a pn/fn there also so that its easier to see them all together. Fixes a case where the findBestProvider() command wasn't working for multiconfig because it was looking for a prefix of "multiconfig:" instead of the newer "mc:"
--- bitbake/lib/bb/command.py | 8 +------- bitbake/lib/bb/runqueue.py | 6 ++++++ 2 files changed, 7 insertions(+), 7 deletions(-)
bitbake/lib/bb/command.py +1 -7
@@ -138,12 +138,6 @@ class Command: def reset(self): self.remotedatastores = bb.remotedata.RemoteDatastores(self.cooker) -def split_mc_pn(pn): - if pn.startswith("multiconfig:"): - _, mc, pn = pn.split(":", 2) - return (mc, pn) - return ('', pn) - class CommandsSync: """ A class of synchronous commands
@@ -438,7 +432,7 @@ class CommandsSync: findProviders.readonly = True def findBestProvider(self, command, params): - (mc, pn) = split_mc_pn(params[0]) + (mc, pn) = bb.runqueue.split_pn(params[0]) return command.cooker.findBestProvider(pn, mc) findBestProvider.readonly = True
bitbake/lib/bb/runqueue.py +6 -0
@@ -46,6 +46,12 @@ def split_tid(tid): (mc, fn, taskname, _) = split_tid_mcfn(tid) return (mc, fn, taskname) +def split_pn(pn): + if pn.startswith("mc:"): + _, mc, pn = pn.split(":", 2) + return (mc, pn) + return ('', pn) + def split_tid_mcfn(tid): if tid.startswith('mc:'): elems = tid.split(':')
Signature · 1 line
2.26.2
v3 cover Add support for per-multiconfig BBMASK
In reply to Joshua WattSplits apart the bitbake configuration data and caches so that each defined multiconfig can specify as separate BBMASK independently of the other multiconfigs. This can take multiple forms, such as a BBMASK in a multiconfig .conf file, or BBMASK assignment based on an override (see linked bugzilla for an example).
The most noticeable change is that bitbake now keeps a parsing cache file for each multiconfig instead of a single parsing cache file that caches all multiconfigs.
V2: Fixes some syntax errors in tinfoil
V3: Optimizes caches so that each one only caches the files for it's own multiconfig
Joshua Watt (8):
bitbake: cooker: Split file collections per multiconfig
bitbake: cache: Use multiconfig aware caches
bitbake: lib: Add support for Logging Adapters
bitbake: lib: Add PrefixLoggerAdapter helper
bitbake: cache: Improve logging
bitbake: cache: Cache size optimization
bitbake: tests: Add tests for BBMASK in multiconfig
bitbake: command: Move split_mc_pn to runqueue
bitbake/lib/bb/__init__.py | 30 ++- bitbake/lib/bb/cache.py | 228 ++++++++++++------ bitbake/lib/bb/command.py | 39 +-- bitbake/lib/bb/cooker.py | 159 +++++++----- bitbake/lib/bb/runqueue.py | 17 +- .../bb/tests/runqueue-tests/conf/bitbake.conf | 3 +- .../runqueue-tests/conf/multiconfig/mc1.conf | 1 + .../runqueue-tests/conf/multiconfig/mc2.conf | 1 + .../recipes/fails-mc/fails-mc1.bb | 5 + .../recipes/fails-mc/fails-mc2.bb | 4 + bitbake/lib/bb/tests/runqueue.py | 15 ++ bitbake/lib/bb/tinfoil.py | 26 +- bitbake/lib/bblayers/query.py | 4 +- 13 files changed, 360 insertions(+), 172 deletions(-) create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb
Signature · 1 line
2.26.2
PATCH v3 1/8 bitbake: cooker: Split file collections per multiconfig
Splits the cooker to track a collection per multiconfig instead of a single collection for all multiconfigs. Practically speaking, this allows each multiconfigs to each have different BBMASKs that apply to it instead of each one using the mask specified in the base configuration.
--- bitbake/lib/bb/cache.py | 2 +- bitbake/lib/bb/command.py | 31 ++++++-- bitbake/lib/bb/cooker.py | 139 ++++++++++++++++++++-------------- bitbake/lib/bb/runqueue.py | 11 +-- bitbake/lib/bb/tinfoil.py | 26 ++++--- bitbake/lib/bblayers/query.py | 4 +- 6 files changed, 129 insertions(+), 84 deletions(-)
bitbake/lib/bb/cache.py +1 -1
@@ -623,7 +623,7 @@ class Cache(NoCache): self.remove(fn) return False - if appends != info_array[0].appends: + if tuple(appends) != tuple(info_array[0].appends): logger.debug(2, "Cache: appends for %s changed", fn) logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) self.remove(fn)
bitbake/lib/bb/command.py +24 -7
@@ -232,7 +232,11 @@ class CommandsSync: def matchFile(self, command, params): fMatch = params[0] - return command.cooker.matchFile(fMatch) + try: + mc = params[0] + except IndexError: + mc = '' + return command.cooker.matchFile(fMatch, mc) matchFile.needconfig = False def getUIHandlerNum(self, command, params):
@@ -396,21 +400,33 @@ class CommandsSync: # Return list sorted by reverse priority order import bb.cache skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), - key=lambda x: (-command.cooker.collection.calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0]))) + key=lambda x: (-command.cooker.collections[mc].calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0]))) return list(skipdict.items()) getSkippedRecipes.readonly = True def getOverlayedRecipes(self, command, params): - return list(command.cooker.collection.overlayed.items()) + try: + mc = params[0] + except IndexError: + mc = '' + return list(command.cooker.collections[mc].overlayed.items()) getOverlayedRecipes.readonly = True def getFileAppends(self, command, params): fn = params[0] - return command.cooker.collection.get_file_appends(fn) + try: + mc = params[1] + except IndexError: + mc = '' + return command.cooker.collections[mc].get_file_appends(fn) getFileAppends.readonly = True def getAllAppends(self, command, params): - return command.cooker.collection.bbappends + try: + mc = params[0] + except IndexError: + mc = '' + return command.cooker.collections[mc].bbappends getAllAppends.readonly = True def findProviders(self, command, params):
@@ -496,6 +512,7 @@ class CommandsSync: for the recipe. """ fn = params[0] + mc = bb.runqueue.mc_from_tid(fn) appends = params[1] appendlist = params[2] if len(params) > 3:
@@ -507,7 +524,7 @@ class CommandsSync: if appendlist is not None: appendfiles = appendlist else: - appendfiles = command.cooker.collection.get_file_appends(fn) + appendfiles = command.cooker.collections[mc].get_file_appends(fn) else: appendfiles = [] # We are calling bb.cache locally here rather than on the server,
@@ -517,7 +534,7 @@ class CommandsSync: if config_data: # We have to use a different function here if we're passing in a datastore # NOTE: we took a copy above, so we don't do it here again - envdata = bb.cache.parse_recipe(config_data, fn, appendfiles)[''] + envdata = bb.cache.parse_recipe(config_data, fn, appendfiles, mc)[''] else: # Use the standard path parser = bb.cache.NoCache(command.cooker.databuilder)
bitbake/lib/bb/cooker.py +82 -57
268 lines not on this page · raw message
bitbake/lib/bb/runqueue.py +6 -5
@@ -1557,7 +1557,8 @@ class RunQueue: def rq_dump_sigfn(self, fn, options): bb_cache = bb.cache.NoCache(self.cooker.databuilder) - the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn)) + mc = bb.runqueue.mc_from_tid(fn) + the_data = bb_cache.loadDataFull(fn, self.cooker.collections[mc].get_file_appends(fn)) siggen = bb.parse.siggen dataCaches = self.rqdata.dataCaches siggen.dump_sigfn(fn, dataCaches, options)
@@ -2042,10 +2043,10 @@ class RunQueueExecute: if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: if not mc in self.rq.fakeworker: self.rq.start_fakeworker(self, mc) - self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") + self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") self.rq.fakeworker[mc].process.stdin.flush() else: - self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") + self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") self.rq.worker[mc].process.stdin.flush() self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
@@ -2129,10 +2130,10 @@ class RunQueueExecute: self.rq.state = runQueueFailed self.stats.taskFailed() return True - self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") + self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") self.rq.fakeworker[mc].process.stdin.flush() else: - self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") + self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") self.rq.worker[mc].process.stdin.flush() self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
bitbake/lib/bb/tinfoil.py +14 -12
@@ -117,15 +117,16 @@ class TinfoilCookerAdapter: class TinfoilCookerCollectionAdapter: """ cooker.collection adapter """ - def __init__(self, tinfoil): + def __init__(self, tinfoil, mc=''): self.tinfoil = tinfoil + self.mc = mc def get_file_appends(self, fn): - return self.tinfoil.get_file_appends(fn) + return self.tinfoil.get_file_appends(fn, self.mc) def __getattr__(self, name): if name == 'overlayed': - return self.tinfoil.get_overlayed_recipes() + return self.tinfoil.get_overlayed_recipes(self.mc) elif name == 'bbappends': - return self.tinfoil.run_command('getAllAppends') + return self.tinfoil.run_command('getAllAppends', self.mc) else: raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
@@ -185,10 +186,11 @@ class TinfoilCookerAdapter: def __init__(self, tinfoil): self.tinfoil = tinfoil - self.collection = self.TinfoilCookerCollectionAdapter(tinfoil) + self.collections = {} self.recipecaches = {} self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil) for mc in (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split(): + self.collections[mc] = self.TinfoilCookerCollectionAdapter(tinfoil) self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc) self._cache = {} def __getattr__(self, name):
@@ -196,7 +198,7 @@ class TinfoilCookerAdapter: if name in self._cache: return self._cache[name] elif name == 'skiplist': - attrvalue = self.tinfoil.get_skipped_recipes() + attrvalue = self.tinfoil.get_skipped_recipes(self.mc) elif name == 'bbfile_config_priorities': ret = self.tinfoil.run_command('getLayerPriorities') bbfile_config_priorities = []
@@ -492,18 +494,18 @@ class Tinfoil: raise Exception('Not connected to server (did you call .prepare()?)') return self.server_connection.events.waitEvent(timeout) - def get_overlayed_recipes(self): + def get_overlayed_recipes(self, mc=''): """ Find recipes which are overlayed (i.e. where recipes exist in multiple layers) """ - return defaultdict(list, self.run_command('getOverlayedRecipes')) + return defaultdict(list, self.run_command('getOverlayedRecipes', mc)) - def get_skipped_recipes(self): + def get_skipped_recipes(self, mc=''): """ Find recipes which were skipped (i.e. SkipRecipe was raised during parsing). """ - return OrderedDict(self.run_command('getSkippedRecipes')) + return OrderedDict(self.run_command('getSkippedRecipes', mc)) def get_all_providers(self, mc=''): return defaultdict(list, self.run_command('allProviders', mc))
@@ -534,11 +536,11 @@ class Tinfoil: raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn) return best[3] - def get_file_appends(self, fn): + def get_file_appends(self, fn, mc=''): """ Find the bbappends for a recipe file """ - return self.run_command('getFileAppends', fn) + return self.run_command('getFileAppends', fn, mc) def all_recipes(self, mc='', sort=True): """
bitbake/lib/bblayers/query.py +2 -2
@@ -320,12 +320,12 @@ Lists recipes with the bbappends that apply to them as subitems. def get_appends_for_files(self, filenames): appended, notappended = [], [] for filename in filenames: - _, cls, _ = bb.cache.virtualfn2realfn(filename) + _, cls, mc = bb.cache.virtualfn2realfn(filename) if cls: continue basename = os.path.basename(filename) - appends = self.tinfoil.cooker.collection.get_file_appends(basename) + appends = self.tinfoil.cooker.collections[mc].get_file_appends(basename) if appends: appended.append((basename, list(appends))) else:
Signature · 1 line
2.26.2
PATCH v3 2/8 bitbake: cache: Use multiconfig aware caches
In reply to Joshua WattSplits the parsing cache to maintain one cache per multiconfig instead of one global cache. This is necessary now that the files and appends can vary for each multiconfig. A bb.cache.MulticonfigCache dictionary-like proxy object is created instead of a single bb.cache.Cache object. This object will create and properly initialize bb.cache.Cache object for each multiconfig, and each of these caches has a dedicated cache file with a name based on the multiconfig.
--- bitbake/lib/bb/cache.py | 143 +++++++++++++++++++++++++++++---------- bitbake/lib/bb/cooker.py | 42 +++++++----- 2 files changed, 133 insertions(+), 52 deletions(-)
bitbake/lib/bb/cache.py +108 -35
@@ -19,7 +19,7 @@ import os import logging import pickle -from collections import defaultdict +from collections import defaultdict, Mapping import bb.utils import re
@@ -27,8 +27,11 @@ logger = logging.getLogger("BitBake.Cache") __cache_version__ = "152" -def getCacheFile(path, filename, data_hash): - return os.path.join(path, filename + "." + data_hash) +def getCacheFile(path, filename, mc, data_hash): + mcspec = '' + if mc: + mcspec = ".%s" % mc + return os.path.join(path, filename + mcspec + "." + data_hash) # RecipeInfoCommon defines common data retrieving methods # from meta data for caches. CoreRecipeInfo as well as other
@@ -354,14 +357,14 @@ class Cache(NoCache): """ BitBake Cache implementation """ - - def __init__(self, databuilder, data_hash, caches_array): + def __init__(self, databuilder, mc, data_hash, caches_array): super().__init__(databuilder) data = databuilder.data # Pass caches_array information into Cache Constructor # It will be used later for deciding whether we # need extra cache file dump/load support + self.mc = mc self.caches_array = caches_array self.cachedir = data.getVar("CACHE") self.clean = set()
@@ -379,7 +382,17 @@ class Cache(NoCache): return self.has_cache = True - self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash) + + def getCacheFile(self, cachefile): + return getCacheFile(self.cachedir, cachefile, self.mc, self.data_hash) + + def prepare_cache(self, progress): + if not self.has_cache: + return 0 + + loaded = 0 + + self.cachefile = self.getCacheFile("bb_cache.dat") logger.debug(1, "Cache dir: %s", self.cachedir) bb.utils.mkdirhier(self.cachedir)
@@ -387,18 +400,22 @@ class Cache(NoCache): cache_ok = True if self.caches_array: for cache_class in self.caches_array: - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + cachefile = self.getCacheFile(cache_class.cachefile) cache_ok = cache_ok and os.path.exists(cachefile) cache_class.init_cacheData(self) if cache_ok: - self.load_cachefile() + loaded = self.load_cachefile(progress) elif os.path.isfile(self.cachefile): logger.info("Out of date cache found, rebuilding...") else: logger.debug(1, "Cache file %s not found, building..." % self.cachefile) # We don't use the symlink, its just for debugging convinience - symlink = os.path.join(self.cachedir, "bb_cache.dat") + if self.mc: + symlink = os.path.join(self.cachedir, "bb_cache.dat.%s" % self.mc) + else: + symlink = os.path.join(self.cachedir, "bb_cache.dat") + if os.path.exists(symlink): bb.utils.remove(symlink) try:
@@ -406,21 +423,30 @@ class Cache(NoCache): except OSError: pass - def load_cachefile(self): - cachesize = 0 - previous_progress = 0 - previous_percent = 0 + return loaded + + def cachesize(self): + if not self.has_cache: + return 0 - # Calculate the correct cachesize of all those cache files + cachesize = 0 for cache_class in self.caches_array: - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) - with open(cachefile, "rb") as cachefile: - cachesize += os.fstat(cachefile.fileno()).st_size + cachefile = self.getCacheFile(cache_class.cachefile) + try: + with open(cachefile, "rb") as cachefile: + cachesize += os.fstat(cachefile.fileno()).st_size + except FileNotFoundError: + pass - bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data) + return cachesize + + def load_cachefile(self, progress): + cachesize = self.cachesize() + previous_progress = 0 + previous_percent = 0 for cache_class in self.caches_array: - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + cachefile = self.getCacheFile(cache_class.cachefile) logger.debug(1, 'Loading cache file: %s' % cachefile) with open(cachefile, "rb") as cachefile: pickled = pickle.Unpickler(cachefile)
@@ -460,23 +486,11 @@ class Cache(NoCache): self.depends_cache[key] = [value] # only fire events on even percentage boundaries current_progress = cachefile.tell() + previous_progress - if current_progress > cachesize: - # we might have calculated incorrect total size because a file - # might've been written out just after we checked its size - cachesize = current_progress - current_percent = 100 * current_progress / cachesize - if current_percent > previous_percent: - previous_percent = current_percent - bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize), - self.data) + progress(cachefile.tell() + previous_progress) previous_progress += current_progress - # Note: depends cache number is corresponding to the parsing file numbers. - # The same file has several caches, still regarded as one item in the cache - bb.event.fire(bb.event.CacheLoadCompleted(cachesize, - len(self.depends_cache)), - self.data) + return len(self.depends_cache) def parse(self, filename, appends): """Parse the specified filename, returning the recipe information"""
@@ -682,7 +696,7 @@ class Cache(NoCache): for cache_class in self.caches_array: cache_class_name = cache_class.__name__ - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + cachefile = self.getCacheFile(cache_class.cachefile) with open(cachefile, "wb") as f: p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL) p.dump(__cache_version__)
@@ -701,7 +715,7 @@ class Cache(NoCache): return bb.parse.cached_mtime_noerror(cachefile) def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None): - if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): + if cacheData is not None and isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): cacheData.add_from_recipeinfo(filename, info_array) if watcher:
@@ -727,6 +741,65 @@ class Cache(NoCache): info_array.append(cache_class(realfn, data)) self.add_info(file_name, info_array, cacheData, parsed) +class MulticonfigCache(Mapping): + def __init__(self, databuilder, data_hash, caches_array): + def progress(p): + nonlocal current_progress + nonlocal previous_progress + nonlocal previous_percent + nonlocal cachesize + + current_progress = previous_progress + p + + if current_progress > cachesize: + # we might have calculated incorrect total size because a file + # might've been written out just after we checked its size + cachesize = current_progress + current_percent = 100 * current_progress / cachesize + if current_percent > previous_percent: + previous_percent = current_percent + bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize), + databuilder.data) + + + cachesize = 0 + current_progress = 0 + previous_progress = 0 + previous_percent = 0 + self.__caches = {} + + for mc, mcdata in databuilder.mcdata.items(): + self.__caches[mc] = Cache(databuilder, mc, data_hash, caches_array) + + cachesize += self.__caches[mc].cachesize() + + bb.event.fire(bb.event.CacheLoadStarted(cachesize), databuilder.data) + loaded = 0 + + for c in self.__caches.values(): + loaded += c.prepare_cache(progress) + previous_progress = current_progress + + # Note: depends cache number is corresponding to the parsing file numbers. + # The same file has several caches, still regarded as one item in the cache + bb.event.fire(bb.event.CacheLoadCompleted(cachesize, loaded), databuilder.data) + + def __len__(self): + return len(self.__caches) + + def __getitem__(self, key): + return self.__caches[key] + + def __contains__(self, key): + return key in self.__caches + + def __iter__(self): + for k in self.__caches: + yield k + + def keys(self): + return self.__caches[key] + def init(cooker): """
bitbake/lib/bb/cooker.py +25 -17
108 lines not on this page · raw message
Signature · 1 line
2.26.2
PATCH v3 3/8 bitbake: lib: Add support for Logging Adapters
In reply to Joshua WattCreates a BBLoggingAdapter class that is monkey patched in place of the logginer.LoggingAdapter. The new adapter is compatible with the BBLogger class API, allowing adapters to be created for bitbake loggers. A new BBLoggerMixin class is used to reduce code duplication between the BBLogger and BBLoggerAdapter classes.
--- bitbake/lib/bb/__init__.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-)
bitbake/lib/bb/__init__.py +18 -4
@@ -35,12 +35,14 @@ class NullHandler(logging.Handler): def emit(self, record): pass -Logger = logging.getLoggerClass() -class BBLogger(Logger): - def __init__(self, name): +class BBLoggerMixin(object): + def __init__(self, *args, **kwargs): + # Does nothing to allow calling super() from derived classes + pass + + def setup_bblogger(self, name): if name.split(".")[0] == "BitBake": self.debug = self.bbdebug - Logger.__init__(self, name) def bbdebug(self, level, msg, *args, **kwargs): loglevel = logging.DEBUG - level + 1
@@ -60,10 +62,22 @@ class BBLogger(Logger): def verbnote(self, msg, *args, **kwargs): return self.log(logging.INFO + 2, msg, *args, **kwargs) +Logger = logging.getLoggerClass() +class BBLogger(Logger, BBLoggerMixin): + def __init__(self, name, *args, **kwargs): + self.setup_bblogger(name) + super().__init__(name, *args, **kwargs) logging.raiseExceptions = False logging.setLoggerClass(BBLogger) +class BBLoggerAdapter(logging.LoggerAdapter, BBLoggerMixin): + def __init__(self, logger, *args, **kwargs): + self.setup_bblogger(logger.name) + super().__init__(logger, *args, **kwargs) + +logging.LoggerAdapter = BBLoggerAdapter + logger = logging.getLogger("BitBake") logger.addHandler(NullHandler()) logger.setLevel(logging.DEBUG - 2)
Signature · 1 line
2.26.2
PATCH v3 4/8 bitbake: lib: Add PrefixLoggerAdapter helper
In reply to Joshua WattAdds a helper logger adapter to add a prefix to all log messages. This is useful to distinguish log messages between multiple instances of a object.
--- bitbake/lib/bb/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+)
bitbake/lib/bb/__init__.py +8 -0
@@ -84,6 +84,14 @@ logger.setLevel(logging.DEBUG - 2) mainlogger = logging.getLogger("BitBake.Main") +class PrefixLoggerAdapter(logging.LoggerAdapter): + def __init__(self, prefix, logger): + super().__init__(logger, {}) + self.__msg_prefix = prefix + + def process(self, msg, kwargs): + return "%s%s" %(self.__msg_prefix, msg), kwargs + # This has to be imported after the setLoggerClass, as the import of bb.msg # can result in construction of the various loggers. import bb.msg
Signature · 1 line
2.26.2
PATCH v3 5/8 bitbake: cache: Improve logging
In reply to Joshua WattImproves the logging of Cache objects by prefixing the log messages with the multiconfig name of the cache, so as to distinguish between multiple instances of the class. Also adds a more log messages.
--- bitbake/lib/bb/cache.py | 65 ++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 30 deletions(-)
bitbake/lib/bb/cache.py +35 -30
@@ -21,6 +21,7 @@ import logging import pickle from collections import defaultdict, Mapping import bb.utils +from bb import PrefixLoggerAdapter import re logger = logging.getLogger("BitBake.Cache")
@@ -365,6 +366,7 @@ class Cache(NoCache): # It will be used later for deciding whether we # need extra cache file dump/load support self.mc = mc + self.logger = PrefixLoggerAdapter("Cache: %s: " % (mc if mc else "default"), logger) self.caches_array = caches_array self.cachedir = data.getVar("CACHE") self.clean = set()
@@ -377,8 +379,8 @@ class Cache(NoCache): if self.cachedir in [None, '']: self.has_cache = False - logger.info("Not using a cache. " - "Set CACHE = <directory> to enable.") + self.logger.info("Not using a cache. " + "Set CACHE = <directory> to enable.") return self.has_cache = True
@@ -394,21 +396,23 @@ class Cache(NoCache): self.cachefile = self.getCacheFile("bb_cache.dat") - logger.debug(1, "Cache dir: %s", self.cachedir) + self.logger.debug(1, "Cache dir: %s", self.cachedir) bb.utils.mkdirhier(self.cachedir) cache_ok = True if self.caches_array: for cache_class in self.caches_array: cachefile = self.getCacheFile(cache_class.cachefile) - cache_ok = cache_ok and os.path.exists(cachefile) + cache_exists = os.path.exists(cachefile) + self.logger.debug(2, "Checking if %s exists: %r", cachefile, cache_exists) + cache_ok = cache_ok and cache_exists cache_class.init_cacheData(self) if cache_ok: loaded = self.load_cachefile(progress) elif os.path.isfile(self.cachefile): - logger.info("Out of date cache found, rebuilding...") + self.logger.info("Out of date cache found, rebuilding...") else: - logger.debug(1, "Cache file %s not found, building..." % self.cachefile) + self.logger.debug(1, "Cache file %s not found, building..." % self.cachefile) # We don't use the symlink, its just for debugging convinience if self.mc:
@@ -447,7 +451,7 @@ class Cache(NoCache): for cache_class in self.caches_array: cachefile = self.getCacheFile(cache_class.cachefile) - logger.debug(1, 'Loading cache file: %s' % cachefile) + self.logger.debug(1, 'Loading cache file: %s' % cachefile) with open(cachefile, "rb") as cachefile: pickled = pickle.Unpickler(cachefile) # Check cache version information
@@ -455,14 +459,14 @@ class Cache(NoCache): cache_ver = pickled.load() bitbake_ver = pickled.load() except Exception: - logger.info('Invalid cache, rebuilding...') + self.logger.info('Invalid cache, rebuilding...') return if cache_ver != __cache_version__: - logger.info('Cache version mismatch, rebuilding...') + self.logger.info('Cache version mismatch, rebuilding...') return elif bitbake_ver != bb.__version__: - logger.info('Bitbake version mismatch, rebuilding...') + self.logger.info('Bitbake version mismatch, rebuilding...') return # Load the rest of the cache file
@@ -494,7 +498,7 @@ class Cache(NoCache): def parse(self, filename, appends): """Parse the specified filename, returning the recipe information""" - logger.debug(1, "Parsing %s", filename) + self.logger.debug(1, "Parsing %s", filename) infos = [] datastores = self.load_bbfile(filename, appends) depends = []
@@ -548,7 +552,7 @@ class Cache(NoCache): cached, infos = self.load(fn, appends) for virtualfn, info_array in infos: if info_array[0].skipped: - logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason) + self.logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason) skipped += 1 else: self.add_info(virtualfn, info_array, cacheData, not cached)
@@ -584,21 +588,21 @@ class Cache(NoCache): # File isn't in depends_cache if not fn in self.depends_cache: - logger.debug(2, "Cache: %s is not cached", fn) + self.logger.debug(2, "%s is not cached", fn) return False mtime = bb.parse.cached_mtime_noerror(fn) # Check file still exists if mtime == 0: - logger.debug(2, "Cache: %s no longer exists", fn) + self.logger.debug(2, "%s no longer exists", fn) self.remove(fn) return False info_array = self.depends_cache[fn] # Check the file's timestamp if mtime != info_array[0].timestamp: - logger.debug(2, "Cache: %s changed", fn) + self.logger.debug(2, "%s changed", fn) self.remove(fn) return False
@@ -609,14 +613,14 @@ class Cache(NoCache): fmtime = bb.parse.cached_mtime_noerror(f) # Check if file still exists if old_mtime != 0 and fmtime == 0: - logger.debug(2, "Cache: %s's dependency %s was removed", - fn, f) + self.logger.debug(2, "%s's dependency %s was removed", + fn, f) self.remove(fn) return False if (fmtime != old_mtime): - logger.debug(2, "Cache: %s's dependency %s changed", - fn, f) + self.logger.debug(2, "%s's dependency %s changed", + fn, f) self.remove(fn) return False
@@ -632,14 +636,14 @@ class Cache(NoCache): continue f, exist = f.split(":") if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)): - logger.debug(2, "Cache: %s's file checksum list file %s changed", - fn, f) + self.logger.debug(2, "%s's file checksum list file %s changed", + fn, f) self.remove(fn) return False if tuple(appends) != tuple(info_array[0].appends): - logger.debug(2, "Cache: appends for %s changed", fn) - logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) + self.logger.debug(2, "appends for %s changed", fn) + self.logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) self.remove(fn) return False
@@ -648,10 +652,10 @@ class Cache(NoCache): virtualfn = variant2virtual(fn, cls) self.clean.add(virtualfn) if virtualfn not in self.depends_cache: - logger.debug(2, "Cache: %s is not cached", virtualfn) + self.logger.debug(2, "%s is not cached", virtualfn) invalid = True elif len(self.depends_cache[virtualfn]) != len(self.caches_array): - logger.debug(2, "Cache: Extra caches missing for %s?" % virtualfn) + self.logger.debug(2, "Extra caches missing for %s?" % virtualfn) invalid = True # If any one of the variants is not present, mark as invalid for all
@@ -659,10 +663,10 @@ class Cache(NoCache): for cls in info_array[0].variants: virtualfn = variant2virtual(fn, cls) if virtualfn in self.clean: - logger.debug(2, "Cache: Removing %s from cache", virtualfn) + self.logger.debug(2, "Removing %s from cache", virtualfn) self.clean.remove(virtualfn) if fn in self.clean: - logger.debug(2, "Cache: Marking %s as not clean", fn) + self.logger.debug(2, "Marking %s as not clean", fn) self.clean.remove(fn) return False
@@ -675,10 +679,10 @@ class Cache(NoCache): Called from the parser in error cases """ if fn in self.depends_cache: - logger.debug(1, "Removing %s from cache", fn) + self.logger.debug(1, "Removing %s from cache", fn) del self.depends_cache[fn] if fn in self.clean: - logger.debug(1, "Marking %s as unclean", fn) + self.logger.debug(1, "Marking %s as unclean", fn) self.clean.remove(fn) def sync(self):
@@ -691,12 +695,13 @@ class Cache(NoCache): return if self.cacheclean: - logger.debug(2, "Cache is clean, not saving.") + self.logger.debug(2, "Cache is clean, not saving.") return for cache_class in self.caches_array: cache_class_name = cache_class.__name__ cachefile = self.getCacheFile(cache_class.cachefile) + self.logger.debug(2, "Writing %s", cachefile) with open(cachefile, "wb") as f: p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL) p.dump(__cache_version__)
Signature · 1 line
2.26.2
PATCH v3 6/8 bitbake: cache: Cache size optimization
In reply to Joshua WattNow that there is a cache object per multiconfig, it is not necessary for each cache object to parse all other multiconfigs. Instead, each cache now only parses the files for it's multiconfig.
--- bitbake/lib/bb/cache.py | 22 ++++++++++++++++++---- bitbake/lib/bb/cooker.py | 8 +------- 2 files changed, 19 insertions(+), 11 deletions(-)
bitbake/lib/bb/cache.py +18 -4
@@ -328,7 +328,7 @@ class NoCache(object): bb_data = self.load_bbfile(virtualfn, appends, virtonly=True) return bb_data[virtual] - def load_bbfile(self, bbfile, appends, virtonly = False): + def load_bbfile(self, bbfile, appends, virtonly = False, mc=None): """ Load and parse one .bb build file Return the data and whether parsing resulted in the file being skipped
@@ -341,6 +341,10 @@ class NoCache(object): datastores = parse_recipe(bb_data, bbfile, appends, mc) return datastores + if mc is not None: + bb_data = self.databuilder.mcdata[mc].createCopy() + return parse_recipe(bb_data, bbfile, appends, mc) + bb_data = self.data.createCopy() datastores = parse_recipe(bb_data, bbfile, appends)
@@ -500,7 +504,7 @@ class Cache(NoCache): """Parse the specified filename, returning the recipe information""" self.logger.debug(1, "Parsing %s", filename) infos = [] - datastores = self.load_bbfile(filename, appends) + datastores = self.load_bbfile(filename, appends, mc=self.mc) depends = [] variants = [] # Process the "real" fn last so we can store variants list
@@ -720,8 +724,18 @@ class Cache(NoCache): return bb.parse.cached_mtime_noerror(cachefile) def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None): - if cacheData is not None and isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): - cacheData.add_from_recipeinfo(filename, info_array) + if self.mc is not None: + (fn, cls, mc) = virtualfn2realfn(filename) + if mc: + self.logger.error("Unexpected multiconfig %s", virtualfn) + return + + vfn = realfn2virtual(fn, cls, self.mc) + else: + vfn = filename + + if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): + cacheData.add_from_recipeinfo(vfn, info_array) if watcher: watcher(info_array[0].file_depends)
bitbake/lib/bb/cooker.py +1 -7
@@ -2199,13 +2199,7 @@ class CookerParser(object): if info_array[0].skipped: self.skipped += 1 self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0]) - (fn, cls, fnmc) = bb.cache.virtualfn2realfn(virtualfn) - - if fnmc == mc: - cache = self.cooker.recipecaches[mc] - else: - cache = None - self.bb_caches[mc].add_info(virtualfn, info_array, cache, + self.bb_caches[mc].add_info(virtualfn, info_array, self.cooker.recipecaches[mc], parsed=parsed, watcher = self.cooker.add_filewatch) return True
Signature · 1 line
2.26.2
PATCH v3 7/8 bitbake: tests: Add tests for BBMASK in multiconfig
In reply to Joshua WattAdds a test to validate that multiconfigs can independently mask off recipes by setting BBMASK. See the test description for further information about how the test works.
--- .../lib/bb/tests/runqueue-tests/conf/bitbake.conf | 3 ++- .../runqueue-tests/conf/multiconfig/mc1.conf | 1 + .../runqueue-tests/conf/multiconfig/mc2.conf | 1 + .../runqueue-tests/recipes/fails-mc/fails-mc1.bb | 5 +++++ .../runqueue-tests/recipes/fails-mc/fails-mc2.bb | 4 ++++ bitbake/lib/bb/tests/runqueue.py | 15 +++++++++++++++ 6 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb
bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf +2 -1
@@ -1,7 +1,8 @@ CACHE = "${TOPDIR}/cache" THISDIR = "${@os.path.dirname(d.getVar('FILE'))}" COREBASE := "${@os.path.normpath(os.path.dirname(d.getVar('FILE')+'/../../'))}" -BBFILES = "${COREBASE}/recipes/*.bb" +EXTRA_BBFILES ?= "" +BBFILES = "${COREBASE}/recipes/*.bb ${EXTRA_BBFILES}" PROVIDES = "${PN}" PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}" PF = "${BB_CURRENT_MC}:${PN}"
bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf +1 -0
@@ -1 +1,2 @@ TMPDIR = "${TOPDIR}/mc1/" +BBMASK += "recipes/fails-mc/fails-mc1.bb"
bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf +1 -0
@@ -1 +1,2 @@ TMPDIR = "${TOPDIR}/mc2/" +BBMASK += "recipes/fails-mc/fails-mc2.bb"
bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb new file +5 -0
@@ -0,0 +1,5 @@ +python () { + if d.getVar("BB_CURRENT_MC") == "mc1": + bb.fatal("Multiconfig is mc1") +} +
bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb new file +4 -0
@@ -0,0 +1,4 @@ +python () { + if d.getVar("BB_CURRENT_MC") == "mc2": + bb.fatal("Multiconfig is mc2") +}
bitbake/lib/bb/tests/runqueue.py +15 -0
@@ -232,6 +232,21 @@ class RunQueueTests(unittest.TestCase): expected.remove(x) self.assertEqual(set(tasks), set(expected)) + def test_multiconfig_bbmask(self): + # This test validates that multiconfigs can independently mask off + # recipes they do not want with BBMASK. It works by having recipes + # that will fail to parse for mc1 and mc2, then making each multiconfig + # build the one that does parse. This ensures that the recipes are in + # each multiconfigs BBFILES, but each is masking only the one that + # doesn't parse + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + extraenv = { + "BBMULTICONFIG" : "mc1 mc2", + "BB_SIGNATURE_HANDLER" : "basic", + "EXTRA_BBFILES": "${COREBASE}/recipes/fails-mc/*.bb", + } + cmd = ["bitbake", "mc:mc1:fails-mc2", "mc:mc2:fails-mc1"] + self.run_bitbakecmd(cmd, tempdir, "", extraenv=extraenv) @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required') def test_hashserv_single(self):
Signature · 1 line
2.26.2
PATCH v3 8/8 bitbake: command: Move split_mc_pn to runqueue
In reply to Joshua WattAll of the other multiconfig splitting functions are located in runqueue so move the function to split a pn/fn there also so that its easier to see them all together. Fixes a case where the findBestProvider() command wasn't working for multiconfig because it was looking for a prefix of "multiconfig:" instead of the newer "mc:"
--- bitbake/lib/bb/command.py | 8 +------- bitbake/lib/bb/runqueue.py | 6 ++++++ 2 files changed, 7 insertions(+), 7 deletions(-)
bitbake/lib/bb/command.py +1 -7
@@ -138,12 +138,6 @@ class Command: def reset(self): self.remotedatastores = bb.remotedata.RemoteDatastores(self.cooker) -def split_mc_pn(pn): - if pn.startswith("multiconfig:"): - _, mc, pn = pn.split(":", 2) - return (mc, pn) - return ('', pn) - class CommandsSync: """ A class of synchronous commands
@@ -438,7 +432,7 @@ class CommandsSync: findProviders.readonly = True def findBestProvider(self, command, params): - (mc, pn) = split_mc_pn(params[0]) + (mc, pn) = bb.runqueue.split_pn(params[0]) return command.cooker.findBestProvider(pn, mc) findBestProvider.readonly = True
bitbake/lib/bb/runqueue.py +6 -0
@@ -46,6 +46,12 @@ def split_tid(tid): (mc, fn, taskname, _) = split_tid_mcfn(tid) return (mc, fn, taskname) +def split_pn(pn): + if pn.startswith("mc:"): + _, mc, pn = pn.split(":", 2) + return (mc, pn) + return ('', pn) + def split_tid_mcfn(tid): if tid.startswith('mc:'): elems = tid.split(':')
Signature · 1 line
2.26.2
✗ patchtest: failure for Add support for per-multiconfig BBMASK
In reply to Joshua Watt== Series Details ==
Series: Add support for per-multiconfig BBMASK
Revision: 1
URL : https://patchwork.openembedded.org/series/24450/
State : failure
== Summary ==
Thank you for submitting this patch series to OpenEmbedded Core. This is an automated response. Several tests have been executed on the proposed series by patchtest resulting in the following failures:
* Issue Series sent to the wrong mailing list or some patches from the series correspond to different mailing lists [test_target_mailing_list]
Suggested fix Send the series again to the correct mailing list (ML) Suggested ML [email protected] [http://git.openembedded.org/bitbake/] Patch's path: bitbake/lib/bb/cache.py
* Issue Series does not apply on top of target branch [test_series_merge_on_head]
Suggested fix Rebase your series on top of targeted branch
Targeted branch master (currently at 982b7f98b8)
If you believe any of these test results are incorrect, please reply to the mailing list ([email protected]) raising your concerns. Otherwise we would appreciate you correcting the issues and submitting a new version of the patchset if applicable. Please ensure you add/increment the version number when sending the new version (i.e. [PATCH] -> [PATCH v2] -> [PATCH v3] -> ...).
---
Guidelines: https://www.openembedded.org/wiki/Commit_Patch_Message_Guidelines Test framework: http://git.yoctoproject.org/cgit/cgit.cgi/patchtest Test suite: http://git.yoctoproject.org/cgit/cgit.cgi/patchtest-oe
Re: [OE-core][PATCH v3 4/8] bitbake: lib: Add PrefixLoggerAdapter helper
In reply to Joshua WattQuoted 16 lines · Joshua Watt
Adds a helper logger adapter to add a prefix to all log messages. This
is useful to distinguish log messages between multiple instances of a object.Signed-off-by Joshua Watt <[email protected]>--- bitbake/lib/bb/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+)bitbake/lib/bb/__init__.py +0
-0@@ -84,6 +84,14 @@ logger.setLevel(logging.DEBUG - 2)
bitbake/lib/bb/__init__.py logger.setLevel(logging.DEBUG - 2) · line 94
Quoted 10 lines
bitbake/lib/bb/__init__.py +8
-0mainlogger = logging.getLogger("BitBake.Main") +class PrefixLoggerAdapter(logging.LoggerAdapter): + def __init__(self, prefix, logger): + super().__init__(logger, {}) + self.__msg_prefix = prefix + + def process(self, msg, kwargs): + return "%s%s" %(self.__msg_prefix, msg), kwargs +
Ignoring the mailing list issue, I think there is something odd going on with different python versions somewhere in here.
https://autobuilder.yoctoproject.org/typhoon/#/builders/81/builds/1004
https://autobuilder.yoctoproject.org/typhoon/#/builders/107/builds/585
https://autobuilder.yoctoproject.org/typhoon/#/builders/15/builds/2255
(all debian9)
and then selftest issues:
https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/1020
Cheers,
Richard
v4 cover Add support for per-multiconfig BBMASK
In reply to Joshua WattSplits apart the bitbake configuration data and caches so that each defined multiconfig can specify as separate BBMASK independently of the other multiconfigs. This can take multiple forms, such as a BBMASK in a multiconfig .conf file, or BBMASK assignment based on an override (see linked bugzilla for an example).
The most noticeable change is that bitbake now keeps a parsing cache file for each multiconfig instead of a single parsing cache file that caches all multiconfigs.
[YOCTO #13721]
V2: Fixes some syntax errors in tinfoil
V3: Optimizes caches so that each one only caches the files for it's own multiconfig
V4: Fixes compatability with Python < 3.6
Renames split_pn -> split_mc
Joshua Watt (8):
bitbake: cooker: Split file collections per multiconfig
bitbake: cache: Use multiconfig aware caches
bitbake: lib: Add support for Logging Adapters
bitbake: lib: Add PrefixLoggerAdapter helper
bitbake: cache: Improve logging
bitbake: cache: Cache size optimization
bitbake: tests: Add tests for BBMASK in multiconfig
bitbake: command: Move split_mc_pn to runqueue
bitbake/lib/bb/__init__.py | 50 +++- bitbake/lib/bb/cache.py | 228 ++++++++++++------ bitbake/lib/bb/command.py | 39 +-- bitbake/lib/bb/cooker.py | 159 +++++++----- bitbake/lib/bb/runqueue.py | 17 +- .../bb/tests/runqueue-tests/conf/bitbake.conf | 3 +- .../runqueue-tests/conf/multiconfig/mc1.conf | 1 + .../runqueue-tests/conf/multiconfig/mc2.conf | 1 + .../recipes/fails-mc/fails-mc1.bb | 5 + .../recipes/fails-mc/fails-mc2.bb | 4 + bitbake/lib/bb/tests/runqueue.py | 15 ++ bitbake/lib/bb/tinfoil.py | 26 +- bitbake/lib/bblayers/query.py | 4 +- 13 files changed, 380 insertions(+), 172 deletions(-) create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb
Signature · 1 line
2.26.2
PATCH v4 1/8 bitbake: cooker: Split file collections per multiconfig
Splits the cooker to track a collection per multiconfig instead of a single collection for all multiconfigs. Practically speaking, this allows each multiconfigs to each have different BBMASKs that apply to it instead of each one using the mask specified in the base configuration.
--- bitbake/lib/bb/cache.py | 2 +- bitbake/lib/bb/command.py | 31 ++++++-- bitbake/lib/bb/cooker.py | 139 ++++++++++++++++++++-------------- bitbake/lib/bb/runqueue.py | 11 +-- bitbake/lib/bb/tinfoil.py | 26 ++++--- bitbake/lib/bblayers/query.py | 4 +- 6 files changed, 129 insertions(+), 84 deletions(-)
bitbake/lib/bb/cache.py +1 -1
@@ -623,7 +623,7 @@ class Cache(NoCache): self.remove(fn) return False - if appends != info_array[0].appends: + if tuple(appends) != tuple(info_array[0].appends): logger.debug(2, "Cache: appends for %s changed", fn) logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) self.remove(fn)
bitbake/lib/bb/command.py +24 -7
@@ -232,7 +232,11 @@ class CommandsSync: def matchFile(self, command, params): fMatch = params[0] - return command.cooker.matchFile(fMatch) + try: + mc = params[0] + except IndexError: + mc = '' + return command.cooker.matchFile(fMatch, mc) matchFile.needconfig = False def getUIHandlerNum(self, command, params):
@@ -396,21 +400,33 @@ class CommandsSync: # Return list sorted by reverse priority order import bb.cache skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), - key=lambda x: (-command.cooker.collection.calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0]))) + key=lambda x: (-command.cooker.collections[mc].calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0]))) return list(skipdict.items()) getSkippedRecipes.readonly = True def getOverlayedRecipes(self, command, params): - return list(command.cooker.collection.overlayed.items()) + try: + mc = params[0] + except IndexError: + mc = '' + return list(command.cooker.collections[mc].overlayed.items()) getOverlayedRecipes.readonly = True def getFileAppends(self, command, params): fn = params[0] - return command.cooker.collection.get_file_appends(fn) + try: + mc = params[1] + except IndexError: + mc = '' + return command.cooker.collections[mc].get_file_appends(fn) getFileAppends.readonly = True def getAllAppends(self, command, params): - return command.cooker.collection.bbappends + try: + mc = params[0] + except IndexError: + mc = '' + return command.cooker.collections[mc].bbappends getAllAppends.readonly = True def findProviders(self, command, params):
@@ -496,6 +512,7 @@ class CommandsSync: for the recipe. """ fn = params[0] + mc = bb.runqueue.mc_from_tid(fn) appends = params[1] appendlist = params[2] if len(params) > 3:
@@ -507,7 +524,7 @@ class CommandsSync: if appendlist is not None: appendfiles = appendlist else: - appendfiles = command.cooker.collection.get_file_appends(fn) + appendfiles = command.cooker.collections[mc].get_file_appends(fn) else: appendfiles = [] # We are calling bb.cache locally here rather than on the server,
@@ -517,7 +534,7 @@ class CommandsSync: if config_data: # We have to use a different function here if we're passing in a datastore # NOTE: we took a copy above, so we don't do it here again - envdata = bb.cache.parse_recipe(config_data, fn, appendfiles)[''] + envdata = bb.cache.parse_recipe(config_data, fn, appendfiles, mc)[''] else: # Use the standard path parser = bb.cache.NoCache(command.cooker.databuilder)
bitbake/lib/bb/cooker.py +82 -57
268 lines not on this page · raw message
bitbake/lib/bb/runqueue.py +6 -5
@@ -1557,7 +1557,8 @@ class RunQueue: def rq_dump_sigfn(self, fn, options): bb_cache = bb.cache.NoCache(self.cooker.databuilder) - the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn)) + mc = bb.runqueue.mc_from_tid(fn) + the_data = bb_cache.loadDataFull(fn, self.cooker.collections[mc].get_file_appends(fn)) siggen = bb.parse.siggen dataCaches = self.rqdata.dataCaches siggen.dump_sigfn(fn, dataCaches, options)
@@ -2042,10 +2043,10 @@ class RunQueueExecute: if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: if not mc in self.rq.fakeworker: self.rq.start_fakeworker(self, mc) - self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") + self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") self.rq.fakeworker[mc].process.stdin.flush() else: - self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") + self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") self.rq.worker[mc].process.stdin.flush() self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
@@ -2129,10 +2130,10 @@ class RunQueueExecute: self.rq.state = runQueueFailed self.stats.taskFailed() return True - self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") + self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") self.rq.fakeworker[mc].process.stdin.flush() else: - self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") + self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") self.rq.worker[mc].process.stdin.flush() self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
bitbake/lib/bb/tinfoil.py +14 -12
@@ -117,15 +117,16 @@ class TinfoilCookerAdapter: class TinfoilCookerCollectionAdapter: """ cooker.collection adapter """ - def __init__(self, tinfoil): + def __init__(self, tinfoil, mc=''): self.tinfoil = tinfoil + self.mc = mc def get_file_appends(self, fn): - return self.tinfoil.get_file_appends(fn) + return self.tinfoil.get_file_appends(fn, self.mc) def __getattr__(self, name): if name == 'overlayed': - return self.tinfoil.get_overlayed_recipes() + return self.tinfoil.get_overlayed_recipes(self.mc) elif name == 'bbappends': - return self.tinfoil.run_command('getAllAppends') + return self.tinfoil.run_command('getAllAppends', self.mc) else: raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
@@ -185,10 +186,11 @@ class TinfoilCookerAdapter: def __init__(self, tinfoil): self.tinfoil = tinfoil - self.collection = self.TinfoilCookerCollectionAdapter(tinfoil) + self.collections = {} self.recipecaches = {} self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil) for mc in (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split(): + self.collections[mc] = self.TinfoilCookerCollectionAdapter(tinfoil) self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc) self._cache = {} def __getattr__(self, name):
@@ -196,7 +198,7 @@ class TinfoilCookerAdapter: if name in self._cache: return self._cache[name] elif name == 'skiplist': - attrvalue = self.tinfoil.get_skipped_recipes() + attrvalue = self.tinfoil.get_skipped_recipes(self.mc) elif name == 'bbfile_config_priorities': ret = self.tinfoil.run_command('getLayerPriorities') bbfile_config_priorities = []
@@ -492,18 +494,18 @@ class Tinfoil: raise Exception('Not connected to server (did you call .prepare()?)') return self.server_connection.events.waitEvent(timeout) - def get_overlayed_recipes(self): + def get_overlayed_recipes(self, mc=''): """ Find recipes which are overlayed (i.e. where recipes exist in multiple layers) """ - return defaultdict(list, self.run_command('getOverlayedRecipes')) + return defaultdict(list, self.run_command('getOverlayedRecipes', mc)) - def get_skipped_recipes(self): + def get_skipped_recipes(self, mc=''): """ Find recipes which were skipped (i.e. SkipRecipe was raised during parsing). """ - return OrderedDict(self.run_command('getSkippedRecipes')) + return OrderedDict(self.run_command('getSkippedRecipes', mc)) def get_all_providers(self, mc=''): return defaultdict(list, self.run_command('allProviders', mc))
@@ -534,11 +536,11 @@ class Tinfoil: raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn) return best[3] - def get_file_appends(self, fn): + def get_file_appends(self, fn, mc=''): """ Find the bbappends for a recipe file """ - return self.run_command('getFileAppends', fn) + return self.run_command('getFileAppends', fn, mc) def all_recipes(self, mc='', sort=True): """
bitbake/lib/bblayers/query.py +2 -2
@@ -320,12 +320,12 @@ Lists recipes with the bbappends that apply to them as subitems. def get_appends_for_files(self, filenames): appended, notappended = [], [] for filename in filenames: - _, cls, _ = bb.cache.virtualfn2realfn(filename) + _, cls, mc = bb.cache.virtualfn2realfn(filename) if cls: continue basename = os.path.basename(filename) - appends = self.tinfoil.cooker.collection.get_file_appends(basename) + appends = self.tinfoil.cooker.collections[mc].get_file_appends(basename) if appends: appended.append((basename, list(appends))) else:
Signature · 1 line
2.26.2
PATCH v4 2/8 bitbake: cache: Use multiconfig aware caches
In reply to Joshua WattSplits the parsing cache to maintain one cache per multiconfig instead of one global cache. This is necessary now that the files and appends can vary for each multiconfig. A bb.cache.MulticonfigCache dictionary-like proxy object is created instead of a single bb.cache.Cache object. This object will create and properly initialize bb.cache.Cache object for each multiconfig, and each of these caches has a dedicated cache file with a name based on the multiconfig.
--- bitbake/lib/bb/cache.py | 143 +++++++++++++++++++++++++++++---------- bitbake/lib/bb/cooker.py | 42 +++++++----- 2 files changed, 133 insertions(+), 52 deletions(-)
bitbake/lib/bb/cache.py +108 -35
@@ -19,7 +19,7 @@ import os import logging import pickle -from collections import defaultdict +from collections import defaultdict, Mapping import bb.utils import re
@@ -27,8 +27,11 @@ logger = logging.getLogger("BitBake.Cache") __cache_version__ = "152" -def getCacheFile(path, filename, data_hash): - return os.path.join(path, filename + "." + data_hash) +def getCacheFile(path, filename, mc, data_hash): + mcspec = '' + if mc: + mcspec = ".%s" % mc + return os.path.join(path, filename + mcspec + "." + data_hash) # RecipeInfoCommon defines common data retrieving methods # from meta data for caches. CoreRecipeInfo as well as other
@@ -354,14 +357,14 @@ class Cache(NoCache): """ BitBake Cache implementation """ - - def __init__(self, databuilder, data_hash, caches_array): + def __init__(self, databuilder, mc, data_hash, caches_array): super().__init__(databuilder) data = databuilder.data # Pass caches_array information into Cache Constructor # It will be used later for deciding whether we # need extra cache file dump/load support + self.mc = mc self.caches_array = caches_array self.cachedir = data.getVar("CACHE") self.clean = set()
@@ -379,7 +382,17 @@ class Cache(NoCache): return self.has_cache = True - self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash) + + def getCacheFile(self, cachefile): + return getCacheFile(self.cachedir, cachefile, self.mc, self.data_hash) + + def prepare_cache(self, progress): + if not self.has_cache: + return 0 + + loaded = 0 + + self.cachefile = self.getCacheFile("bb_cache.dat") logger.debug(1, "Cache dir: %s", self.cachedir) bb.utils.mkdirhier(self.cachedir)
@@ -387,18 +400,22 @@ class Cache(NoCache): cache_ok = True if self.caches_array: for cache_class in self.caches_array: - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + cachefile = self.getCacheFile(cache_class.cachefile) cache_ok = cache_ok and os.path.exists(cachefile) cache_class.init_cacheData(self) if cache_ok: - self.load_cachefile() + loaded = self.load_cachefile(progress) elif os.path.isfile(self.cachefile): logger.info("Out of date cache found, rebuilding...") else: logger.debug(1, "Cache file %s not found, building..." % self.cachefile) # We don't use the symlink, its just for debugging convinience - symlink = os.path.join(self.cachedir, "bb_cache.dat") + if self.mc: + symlink = os.path.join(self.cachedir, "bb_cache.dat.%s" % self.mc) + else: + symlink = os.path.join(self.cachedir, "bb_cache.dat") + if os.path.exists(symlink): bb.utils.remove(symlink) try:
@@ -406,21 +423,30 @@ class Cache(NoCache): except OSError: pass - def load_cachefile(self): - cachesize = 0 - previous_progress = 0 - previous_percent = 0 + return loaded + + def cachesize(self): + if not self.has_cache: + return 0 - # Calculate the correct cachesize of all those cache files + cachesize = 0 for cache_class in self.caches_array: - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) - with open(cachefile, "rb") as cachefile: - cachesize += os.fstat(cachefile.fileno()).st_size + cachefile = self.getCacheFile(cache_class.cachefile) + try: + with open(cachefile, "rb") as cachefile: + cachesize += os.fstat(cachefile.fileno()).st_size + except FileNotFoundError: + pass - bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data) + return cachesize + + def load_cachefile(self, progress): + cachesize = self.cachesize() + previous_progress = 0 + previous_percent = 0 for cache_class in self.caches_array: - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + cachefile = self.getCacheFile(cache_class.cachefile) logger.debug(1, 'Loading cache file: %s' % cachefile) with open(cachefile, "rb") as cachefile: pickled = pickle.Unpickler(cachefile)
@@ -460,23 +486,11 @@ class Cache(NoCache): self.depends_cache[key] = [value] # only fire events on even percentage boundaries current_progress = cachefile.tell() + previous_progress - if current_progress > cachesize: - # we might have calculated incorrect total size because a file - # might've been written out just after we checked its size - cachesize = current_progress - current_percent = 100 * current_progress / cachesize - if current_percent > previous_percent: - previous_percent = current_percent - bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize), - self.data) + progress(cachefile.tell() + previous_progress) previous_progress += current_progress - # Note: depends cache number is corresponding to the parsing file numbers. - # The same file has several caches, still regarded as one item in the cache - bb.event.fire(bb.event.CacheLoadCompleted(cachesize, - len(self.depends_cache)), - self.data) + return len(self.depends_cache) def parse(self, filename, appends): """Parse the specified filename, returning the recipe information"""
@@ -682,7 +696,7 @@ class Cache(NoCache): for cache_class in self.caches_array: cache_class_name = cache_class.__name__ - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + cachefile = self.getCacheFile(cache_class.cachefile) with open(cachefile, "wb") as f: p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL) p.dump(__cache_version__)
@@ -701,7 +715,7 @@ class Cache(NoCache): return bb.parse.cached_mtime_noerror(cachefile) def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None): - if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): + if cacheData is not None and isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): cacheData.add_from_recipeinfo(filename, info_array) if watcher:
@@ -727,6 +741,65 @@ class Cache(NoCache): info_array.append(cache_class(realfn, data)) self.add_info(file_name, info_array, cacheData, parsed) +class MulticonfigCache(Mapping): + def __init__(self, databuilder, data_hash, caches_array): + def progress(p): + nonlocal current_progress + nonlocal previous_progress + nonlocal previous_percent + nonlocal cachesize + + current_progress = previous_progress + p + + if current_progress > cachesize: + # we might have calculated incorrect total size because a file + # might've been written out just after we checked its size + cachesize = current_progress + current_percent = 100 * current_progress / cachesize + if current_percent > previous_percent: + previous_percent = current_percent + bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize), + databuilder.data) + + + cachesize = 0 + current_progress = 0 + previous_progress = 0 + previous_percent = 0 + self.__caches = {} + + for mc, mcdata in databuilder.mcdata.items(): + self.__caches[mc] = Cache(databuilder, mc, data_hash, caches_array) + + cachesize += self.__caches[mc].cachesize() + + bb.event.fire(bb.event.CacheLoadStarted(cachesize), databuilder.data) + loaded = 0 + + for c in self.__caches.values(): + loaded += c.prepare_cache(progress) + previous_progress = current_progress + + # Note: depends cache number is corresponding to the parsing file numbers. + # The same file has several caches, still regarded as one item in the cache + bb.event.fire(bb.event.CacheLoadCompleted(cachesize, loaded), databuilder.data) + + def __len__(self): + return len(self.__caches) + + def __getitem__(self, key): + return self.__caches[key] + + def __contains__(self, key): + return key in self.__caches + + def __iter__(self): + for k in self.__caches: + yield k + + def keys(self): + return self.__caches[key] + def init(cooker): """
bitbake/lib/bb/cooker.py +25 -17
108 lines not on this page · raw message
Signature · 1 line
2.26.2
PATCH v4 3/8 bitbake: lib: Add support for Logging Adapters
In reply to Joshua WattCreates a BBLoggingAdapter class that is monkey patched in place of the logginer.LoggingAdapter. The new adapter is compatible with the BBLogger class API, allowing adapters to be created for bitbake loggers. A new BBLoggerMixin class is used to reduce code duplication between the BBLogger and BBLoggerAdapter classes.
--- bitbake/lib/bb/__init__.py | 42 ++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-)
bitbake/lib/bb/__init__.py +38 -4
@@ -35,12 +35,14 @@ class NullHandler(logging.Handler): def emit(self, record): pass -Logger = logging.getLoggerClass() -class BBLogger(Logger): - def __init__(self, name): +class BBLoggerMixin(object): + def __init__(self, *args, **kwargs): + # Does nothing to allow calling super() from derived classes + pass + + def setup_bblogger(self, name): if name.split(".")[0] == "BitBake": self.debug = self.bbdebug - Logger.__init__(self, name) def bbdebug(self, level, msg, *args, **kwargs): loglevel = logging.DEBUG - level + 1
@@ -60,10 +62,42 @@ class BBLogger(Logger): def verbnote(self, msg, *args, **kwargs): return self.log(logging.INFO + 2, msg, *args, **kwargs) +Logger = logging.getLoggerClass() +class BBLogger(Logger, BBLoggerMixin): + def __init__(self, name, *args, **kwargs): + self.setup_bblogger(name) + super().__init__(name, *args, **kwargs) logging.raiseExceptions = False logging.setLoggerClass(BBLogger) +class BBLoggerAdapter(logging.LoggerAdapter, BBLoggerMixin): + def __init__(self, logger, *args, **kwargs): + self.setup_bblogger(logger.name) + super().__init__(logger, *args, **kwargs) + + if sys.version_info < (3, 6): + # These properties were added in Python 3.6. Add them in older versions + # for compatibility + @property + def manager(self): + return self.logger.manager + + @manager.setter + def manager(self, value): + self.logger.manager = value + + @property + def name(self): + return self.logger.name + + def __repr__(self): + logger = self.logger + level = getLevelName(logger.getEffectiveLevel()) + return '<%s %s (%s)>' % (self.__class__.__name__, logger.name, level) + +logging.LoggerAdapter = BBLoggerAdapter + logger = logging.getLogger("BitBake") logger.addHandler(NullHandler()) logger.setLevel(logging.DEBUG - 2)
Signature · 1 line
2.26.2
PATCH v4 4/8 bitbake: lib: Add PrefixLoggerAdapter helper
In reply to Joshua WattAdds a helper logger adapter to add a prefix to all log messages. This is useful to distinguish log messages between multiple instances of a object.
--- bitbake/lib/bb/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+)
bitbake/lib/bb/__init__.py +8 -0
@@ -104,6 +104,14 @@ logger.setLevel(logging.DEBUG - 2) mainlogger = logging.getLogger("BitBake.Main") +class PrefixLoggerAdapter(logging.LoggerAdapter): + def __init__(self, prefix, logger): + super().__init__(logger, {}) + self.__msg_prefix = prefix + + def process(self, msg, kwargs): + return "%s%s" %(self.__msg_prefix, msg), kwargs + # This has to be imported after the setLoggerClass, as the import of bb.msg # can result in construction of the various loggers. import bb.msg
Signature · 1 line
2.26.2
PATCH v4 5/8 bitbake: cache: Improve logging
In reply to Joshua WattImproves the logging of Cache objects by prefixing the log messages with the multiconfig name of the cache, so as to distinguish between multiple instances of the class. Also adds a more log messages.
--- bitbake/lib/bb/cache.py | 65 ++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 30 deletions(-)
bitbake/lib/bb/cache.py +35 -30
@@ -21,6 +21,7 @@ import logging import pickle from collections import defaultdict, Mapping import bb.utils +from bb import PrefixLoggerAdapter import re logger = logging.getLogger("BitBake.Cache")
@@ -365,6 +366,7 @@ class Cache(NoCache): # It will be used later for deciding whether we # need extra cache file dump/load support self.mc = mc + self.logger = PrefixLoggerAdapter("Cache: %s: " % (mc if mc else "default"), logger) self.caches_array = caches_array self.cachedir = data.getVar("CACHE") self.clean = set()
@@ -377,8 +379,8 @@ class Cache(NoCache): if self.cachedir in [None, '']: self.has_cache = False - logger.info("Not using a cache. " - "Set CACHE = <directory> to enable.") + self.logger.info("Not using a cache. " + "Set CACHE = <directory> to enable.") return self.has_cache = True
@@ -394,21 +396,23 @@ class Cache(NoCache): self.cachefile = self.getCacheFile("bb_cache.dat") - logger.debug(1, "Cache dir: %s", self.cachedir) + self.logger.debug(1, "Cache dir: %s", self.cachedir) bb.utils.mkdirhier(self.cachedir) cache_ok = True if self.caches_array: for cache_class in self.caches_array: cachefile = self.getCacheFile(cache_class.cachefile) - cache_ok = cache_ok and os.path.exists(cachefile) + cache_exists = os.path.exists(cachefile) + self.logger.debug(2, "Checking if %s exists: %r", cachefile, cache_exists) + cache_ok = cache_ok and cache_exists cache_class.init_cacheData(self) if cache_ok: loaded = self.load_cachefile(progress) elif os.path.isfile(self.cachefile): - logger.info("Out of date cache found, rebuilding...") + self.logger.info("Out of date cache found, rebuilding...") else: - logger.debug(1, "Cache file %s not found, building..." % self.cachefile) + self.logger.debug(1, "Cache file %s not found, building..." % self.cachefile) # We don't use the symlink, its just for debugging convinience if self.mc:
@@ -447,7 +451,7 @@ class Cache(NoCache): for cache_class in self.caches_array: cachefile = self.getCacheFile(cache_class.cachefile) - logger.debug(1, 'Loading cache file: %s' % cachefile) + self.logger.debug(1, 'Loading cache file: %s' % cachefile) with open(cachefile, "rb") as cachefile: pickled = pickle.Unpickler(cachefile) # Check cache version information
@@ -455,14 +459,14 @@ class Cache(NoCache): cache_ver = pickled.load() bitbake_ver = pickled.load() except Exception: - logger.info('Invalid cache, rebuilding...') + self.logger.info('Invalid cache, rebuilding...') return if cache_ver != __cache_version__: - logger.info('Cache version mismatch, rebuilding...') + self.logger.info('Cache version mismatch, rebuilding...') return elif bitbake_ver != bb.__version__: - logger.info('Bitbake version mismatch, rebuilding...') + self.logger.info('Bitbake version mismatch, rebuilding...') return # Load the rest of the cache file
@@ -494,7 +498,7 @@ class Cache(NoCache): def parse(self, filename, appends): """Parse the specified filename, returning the recipe information""" - logger.debug(1, "Parsing %s", filename) + self.logger.debug(1, "Parsing %s", filename) infos = [] datastores = self.load_bbfile(filename, appends) depends = []
@@ -548,7 +552,7 @@ class Cache(NoCache): cached, infos = self.load(fn, appends) for virtualfn, info_array in infos: if info_array[0].skipped: - logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason) + self.logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason) skipped += 1 else: self.add_info(virtualfn, info_array, cacheData, not cached)
@@ -584,21 +588,21 @@ class Cache(NoCache): # File isn't in depends_cache if not fn in self.depends_cache: - logger.debug(2, "Cache: %s is not cached", fn) + self.logger.debug(2, "%s is not cached", fn) return False mtime = bb.parse.cached_mtime_noerror(fn) # Check file still exists if mtime == 0: - logger.debug(2, "Cache: %s no longer exists", fn) + self.logger.debug(2, "%s no longer exists", fn) self.remove(fn) return False info_array = self.depends_cache[fn] # Check the file's timestamp if mtime != info_array[0].timestamp: - logger.debug(2, "Cache: %s changed", fn) + self.logger.debug(2, "%s changed", fn) self.remove(fn) return False
@@ -609,14 +613,14 @@ class Cache(NoCache): fmtime = bb.parse.cached_mtime_noerror(f) # Check if file still exists if old_mtime != 0 and fmtime == 0: - logger.debug(2, "Cache: %s's dependency %s was removed", - fn, f) + self.logger.debug(2, "%s's dependency %s was removed", + fn, f) self.remove(fn) return False if (fmtime != old_mtime): - logger.debug(2, "Cache: %s's dependency %s changed", - fn, f) + self.logger.debug(2, "%s's dependency %s changed", + fn, f) self.remove(fn) return False
@@ -632,14 +636,14 @@ class Cache(NoCache): continue f, exist = f.split(":") if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)): - logger.debug(2, "Cache: %s's file checksum list file %s changed", - fn, f) + self.logger.debug(2, "%s's file checksum list file %s changed", + fn, f) self.remove(fn) return False if tuple(appends) != tuple(info_array[0].appends): - logger.debug(2, "Cache: appends for %s changed", fn) - logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) + self.logger.debug(2, "appends for %s changed", fn) + self.logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) self.remove(fn) return False
@@ -648,10 +652,10 @@ class Cache(NoCache): virtualfn = variant2virtual(fn, cls) self.clean.add(virtualfn) if virtualfn not in self.depends_cache: - logger.debug(2, "Cache: %s is not cached", virtualfn) + self.logger.debug(2, "%s is not cached", virtualfn) invalid = True elif len(self.depends_cache[virtualfn]) != len(self.caches_array): - logger.debug(2, "Cache: Extra caches missing for %s?" % virtualfn) + self.logger.debug(2, "Extra caches missing for %s?" % virtualfn) invalid = True # If any one of the variants is not present, mark as invalid for all
@@ -659,10 +663,10 @@ class Cache(NoCache): for cls in info_array[0].variants: virtualfn = variant2virtual(fn, cls) if virtualfn in self.clean: - logger.debug(2, "Cache: Removing %s from cache", virtualfn) + self.logger.debug(2, "Removing %s from cache", virtualfn) self.clean.remove(virtualfn) if fn in self.clean: - logger.debug(2, "Cache: Marking %s as not clean", fn) + self.logger.debug(2, "Marking %s as not clean", fn) self.clean.remove(fn) return False
@@ -675,10 +679,10 @@ class Cache(NoCache): Called from the parser in error cases """ if fn in self.depends_cache: - logger.debug(1, "Removing %s from cache", fn) + self.logger.debug(1, "Removing %s from cache", fn) del self.depends_cache[fn] if fn in self.clean: - logger.debug(1, "Marking %s as unclean", fn) + self.logger.debug(1, "Marking %s as unclean", fn) self.clean.remove(fn) def sync(self):
@@ -691,12 +695,13 @@ class Cache(NoCache): return if self.cacheclean: - logger.debug(2, "Cache is clean, not saving.") + self.logger.debug(2, "Cache is clean, not saving.") return for cache_class in self.caches_array: cache_class_name = cache_class.__name__ cachefile = self.getCacheFile(cache_class.cachefile) + self.logger.debug(2, "Writing %s", cachefile) with open(cachefile, "wb") as f: p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL) p.dump(__cache_version__)
Signature · 1 line
2.26.2
PATCH v4 6/8 bitbake: cache: Cache size optimization
In reply to Joshua WattNow that there is a cache object per multiconfig, it is not necessary for each cache object to parse all other multiconfigs. Instead, each cache now only parses the files for it's multiconfig.
--- bitbake/lib/bb/cache.py | 22 ++++++++++++++++++---- bitbake/lib/bb/cooker.py | 8 +------- 2 files changed, 19 insertions(+), 11 deletions(-)
bitbake/lib/bb/cache.py +18 -4
@@ -328,7 +328,7 @@ class NoCache(object): bb_data = self.load_bbfile(virtualfn, appends, virtonly=True) return bb_data[virtual] - def load_bbfile(self, bbfile, appends, virtonly = False): + def load_bbfile(self, bbfile, appends, virtonly = False, mc=None): """ Load and parse one .bb build file Return the data and whether parsing resulted in the file being skipped
@@ -341,6 +341,10 @@ class NoCache(object): datastores = parse_recipe(bb_data, bbfile, appends, mc) return datastores + if mc is not None: + bb_data = self.databuilder.mcdata[mc].createCopy() + return parse_recipe(bb_data, bbfile, appends, mc) + bb_data = self.data.createCopy() datastores = parse_recipe(bb_data, bbfile, appends)
@@ -500,7 +504,7 @@ class Cache(NoCache): """Parse the specified filename, returning the recipe information""" self.logger.debug(1, "Parsing %s", filename) infos = [] - datastores = self.load_bbfile(filename, appends) + datastores = self.load_bbfile(filename, appends, mc=self.mc) depends = [] variants = [] # Process the "real" fn last so we can store variants list
@@ -720,8 +724,18 @@ class Cache(NoCache): return bb.parse.cached_mtime_noerror(cachefile) def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None): - if cacheData is not None and isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): - cacheData.add_from_recipeinfo(filename, info_array) + if self.mc is not None: + (fn, cls, mc) = virtualfn2realfn(filename) + if mc: + self.logger.error("Unexpected multiconfig %s", virtualfn) + return + + vfn = realfn2virtual(fn, cls, self.mc) + else: + vfn = filename + + if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): + cacheData.add_from_recipeinfo(vfn, info_array) if watcher: watcher(info_array[0].file_depends)
bitbake/lib/bb/cooker.py +1 -7
@@ -2199,13 +2199,7 @@ class CookerParser(object): if info_array[0].skipped: self.skipped += 1 self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0]) - (fn, cls, fnmc) = bb.cache.virtualfn2realfn(virtualfn) - - if fnmc == mc: - cache = self.cooker.recipecaches[mc] - else: - cache = None - self.bb_caches[mc].add_info(virtualfn, info_array, cache, + self.bb_caches[mc].add_info(virtualfn, info_array, self.cooker.recipecaches[mc], parsed=parsed, watcher = self.cooker.add_filewatch) return True
Signature · 1 line
2.26.2
PATCH v4 7/8 bitbake: tests: Add tests for BBMASK in multiconfig
In reply to Joshua WattAdds a test to validate that multiconfigs can independently mask off recipes by setting BBMASK. See the test description for further information about how the test works.
--- .../lib/bb/tests/runqueue-tests/conf/bitbake.conf | 3 ++- .../runqueue-tests/conf/multiconfig/mc1.conf | 1 + .../runqueue-tests/conf/multiconfig/mc2.conf | 1 + .../runqueue-tests/recipes/fails-mc/fails-mc1.bb | 5 +++++ .../runqueue-tests/recipes/fails-mc/fails-mc2.bb | 4 ++++ bitbake/lib/bb/tests/runqueue.py | 15 +++++++++++++++ 6 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb
bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf +2 -1
@@ -1,7 +1,8 @@ CACHE = "${TOPDIR}/cache" THISDIR = "${@os.path.dirname(d.getVar('FILE'))}" COREBASE := "${@os.path.normpath(os.path.dirname(d.getVar('FILE')+'/../../'))}" -BBFILES = "${COREBASE}/recipes/*.bb" +EXTRA_BBFILES ?= "" +BBFILES = "${COREBASE}/recipes/*.bb ${EXTRA_BBFILES}" PROVIDES = "${PN}" PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}" PF = "${BB_CURRENT_MC}:${PN}"
bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf +1 -0
@@ -1 +1,2 @@ TMPDIR = "${TOPDIR}/mc1/" +BBMASK += "recipes/fails-mc/fails-mc1.bb"
bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf +1 -0
@@ -1 +1,2 @@ TMPDIR = "${TOPDIR}/mc2/" +BBMASK += "recipes/fails-mc/fails-mc2.bb"
bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb new file +5 -0
@@ -0,0 +1,5 @@ +python () { + if d.getVar("BB_CURRENT_MC") == "mc1": + bb.fatal("Multiconfig is mc1") +} +
bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb new file +4 -0
@@ -0,0 +1,4 @@ +python () { + if d.getVar("BB_CURRENT_MC") == "mc2": + bb.fatal("Multiconfig is mc2") +}
bitbake/lib/bb/tests/runqueue.py +15 -0
@@ -232,6 +232,21 @@ class RunQueueTests(unittest.TestCase): expected.remove(x) self.assertEqual(set(tasks), set(expected)) + def test_multiconfig_bbmask(self): + # This test validates that multiconfigs can independently mask off + # recipes they do not want with BBMASK. It works by having recipes + # that will fail to parse for mc1 and mc2, then making each multiconfig + # build the one that does parse. This ensures that the recipes are in + # each multiconfigs BBFILES, but each is masking only the one that + # doesn't parse + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + extraenv = { + "BBMULTICONFIG" : "mc1 mc2", + "BB_SIGNATURE_HANDLER" : "basic", + "EXTRA_BBFILES": "${COREBASE}/recipes/fails-mc/*.bb", + } + cmd = ["bitbake", "mc:mc1:fails-mc2", "mc:mc2:fails-mc1"] + self.run_bitbakecmd(cmd, tempdir, "", extraenv=extraenv) @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required') def test_hashserv_single(self):
Signature · 1 line
2.26.2
PATCH v4 8/8 bitbake: command: Move split_mc_pn to runqueue
In reply to Joshua WattAll of the other multiconfig splitting functions are located in runqueue so move the function to split a pn/fn there also so that its easier to see them all together. Fixes a case where the findBestProvider() command wasn't working for multiconfig because it was looking for a prefix of "multiconfig:" instead of the newer "mc:"
--- bitbake/lib/bb/command.py | 8 +------- bitbake/lib/bb/runqueue.py | 6 ++++++ 2 files changed, 7 insertions(+), 7 deletions(-)
bitbake/lib/bb/command.py +1 -7
@@ -138,12 +138,6 @@ class Command: def reset(self): self.remotedatastores = bb.remotedata.RemoteDatastores(self.cooker) -def split_mc_pn(pn): - if pn.startswith("multiconfig:"): - _, mc, pn = pn.split(":", 2) - return (mc, pn) - return ('', pn) - class CommandsSync: """ A class of synchronous commands
@@ -438,7 +432,7 @@ class CommandsSync: findProviders.readonly = True def findBestProvider(self, command, params): - (mc, pn) = split_mc_pn(params[0]) + (mc, pn) = bb.runqueue.split_mc(params[0]) return command.cooker.findBestProvider(pn, mc) findBestProvider.readonly = True
bitbake/lib/bb/runqueue.py +6 -0
@@ -46,6 +46,12 @@ def split_tid(tid): (mc, fn, taskname, _) = split_tid_mcfn(tid) return (mc, fn, taskname) +def split_mc(n): + if n.startswith("mc:"): + _, mc, n = n.split(":", 2) + return (mc, n) + return ('', n) + def split_tid_mcfn(tid): if tid.startswith('mc:'): elems = tid.split(':')
Signature · 1 line
2.26.2
Hey Joshua,
Interesting approach, it looks good, it does look like the next step that has to be taken, initially this would have been too invasive but I think this should be fine.
Do you have data on the performance increase/decrease by using multiple caches?, are there any implications?
Cheers,
Alejandro
Quoted 49 lines · Joshua Watt
Splits apart the bitbake configuration data and caches so that each defined multiconfig can specify as separate BBMASK independently of the other multiconfigs. This can take multiple forms, such as a
BBMASKin a multiconfig .conf file, or BBMASK assignment based on an override (see linked bugzilla for an example).The most noticeable change is that bitbake now keeps a parsing cache file for each multiconfig instead of a single parsing cache file that caches all multiconfigs.
[YOCTO #13721]
V2: Fixes some syntax errors in tinfoil
V3: Optimizes caches so that each one only caches the files for it's own multiconfig
V4: Fixes compatability with Python < 3.6
Renames split_pn -> split_mcJoshua Watt (8):
bitbake: cooker: Split file collections per multiconfig
bitbake: cache: Use multiconfig aware caches
bitbake: lib: Add support for Logging Adapters
bitbake: lib: Add PrefixLoggerAdapter helper
bitbake: cache: Improve logging
bitbake: cache: Cache size optimization
bitbake: tests: Add tests for BBMASK in multiconfig
bitbake: command: Movesplit_mc_pnto runqueuebitbake/lib/bb/__init__.py | 50 +++- bitbake/lib/bb/cache.py | 228 ++++++++++++------ bitbake/lib/bb/command.py | 39 +-- bitbake/lib/bb/cooker.py | 159 +++++++----- bitbake/lib/bb/runqueue.py | 17 +- .../bb/tests/runqueue-tests/conf/bitbake.conf | 3 +- .../runqueue-tests/conf/multiconfig/mc1.conf | 1 + .../runqueue-tests/conf/multiconfig/mc2.conf | 1 + .../recipes/fails-mc/fails-mc1.bb | 5 + .../recipes/fails-mc/fails-mc2.bb | 4 + bitbake/lib/bb/tests/runqueue.py | 15 ++ bitbake/lib/bb/tinfoil.py | 26 +- bitbake/lib/bblayers/query.py | 4 +- 13 files changed, 380 insertions(+), 172 deletions(-)create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb
Quoted 7 lines · Alejandro Hernandez Samaniego
Hey Joshua,
Interesting approach, it looks good, it does look like the next step that has to be taken, initially this would have been too invasive but I think this should be fine.
Do you have data on the performance increase/decrease by using multiple caches?, are there any implications?
I don't have numbers, but my feeling is that it should be roughly the same. It's the same amount of data being cached and same number of recipes being parsed, the only difference is that the same cache data is split between multiple files instead of included in a single one.
There might be a bit of performance degredation since the recipe file access pattern has changed (e.g. it now parses all files for a given multiconfig in series before moving to the next multiconfig instead of parsing each file for each multiconfig), but I don't expect that to be too significant of an issue.
Quoted 56 lines
Cheers,
Alejandro
Quoted 49 lines · Joshua Watt
Splits apart the bitbake configuration data and caches so that each defined multiconfig can specify as separate BBMASK independently of the other multiconfigs. This can take multiple forms, such as a
BBMASKin a multiconfig .conf file, or BBMASK assignment based on an override (see linked bugzilla for an example).The most noticeable change is that bitbake now keeps a parsing cache file for each multiconfig instead of a single parsing cache file that caches all multiconfigs.
[YOCTO #13721]
V2: Fixes some syntax errors in tinfoil
V3: Optimizes caches so that each one only caches the files for it's own multiconfig
V4: Fixes compatability with Python < 3.6
Renames split_pn -> split_mcJoshua Watt (8):
bitbake: cooker: Split file collections per multiconfig
bitbake: cache: Use multiconfig aware caches
bitbake: lib: Add support for Logging Adapters
bitbake: lib: Add PrefixLoggerAdapter helper
bitbake: cache: Improve logging
bitbake: cache: Cache size optimization
bitbake: tests: Add tests for BBMASK in multiconfig
bitbake: command: Movesplit_mc_pnto runqueuebitbake/lib/bb/__init__.py | 50 +++- bitbake/lib/bb/cache.py | 228 ++++++++++++------ bitbake/lib/bb/command.py | 39 +-- bitbake/lib/bb/cooker.py | 159 +++++++----- bitbake/lib/bb/runqueue.py | 17 +- .../bb/tests/runqueue-tests/conf/bitbake.conf | 3 +- .../runqueue-tests/conf/multiconfig/mc1.conf | 1 + .../runqueue-tests/conf/multiconfig/mc2.conf | 1 + .../recipes/fails-mc/fails-mc1.bb | 5 + .../recipes/fails-mc/fails-mc2.bb | 4 + bitbake/lib/bb/tests/runqueue.py | 15 ++ bitbake/lib/bb/tinfoil.py | 26 +- bitbake/lib/bblayers/query.py | 4 +- 13 files changed, 380 insertions(+), 172 deletions(-)create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb
Quoted 15 lines · Joshua Watt
Splits apart the bitbake configuration data and caches so that each defined multiconfig can specify as separate BBMASK independently of the
other multiconfigs. This can take multiple forms, such as aBBMASKin
a multiconfig .conf file, or BBMASK assignment based on an override (see
linked bugzilla for an example).The most noticeable change is that bitbake now keeps a parsing cache file for each multiconfig instead of a single parsing cache file that caches all multiconfigs.
[YOCTO #13721]
Quoted 8 lines
V2: Fixes some syntax errors in tinfoil
V3: Optimizes caches so that each one only caches the files for it's own
multiconfigV4: Fixes compatability with Python < 3.6
Renames split_pn -> split_mc
Its looking better, thanks. There are still the selftest failures:
https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/1023
Cheers,
Richard
v5 cover Add support for per-multiconfig BBMASK
In reply to Joshua WattSplits apart the bitbake configuration data and caches so that each defined multiconfig can specify as separate BBMASK independently of the other multiconfigs. This can take multiple forms, such as a BBMASK in a multiconfig .conf file, or BBMASK assignment based on an override (see linked bugzilla for an example).
The most noticeable change is that bitbake now keeps a parsing cache file for each multiconfig instead of a single parsing cache file that caches all multiconfigs.
[YOCTO #13721]
V2: Fixes some syntax errors in tinfoil
V3: Optimizes caches so that each one only caches the files for it's own multiconfig
V4: Fixes compatability with Python < 3.6
Renames split_pn -> split_mc
V5: Fixes oe-selftest errors
Joshua Watt (8):
bitbake: cooker: Split file collections per multiconfig
bitbake: cache: Use multiconfig aware caches
bitbake: lib: Add support for Logging Adapters
bitbake: lib: Add PrefixLoggerAdapter helper
bitbake: cache: Improve logging
bitbake: cache: Cache size optimization
bitbake: tests: Add tests for BBMASK in multiconfig
bitbake: command: Move split_mc_pn to runqueue
bitbake/lib/bb/__init__.py | 50 +++- bitbake/lib/bb/cache.py | 228 ++++++++++++------ bitbake/lib/bb/command.py | 45 ++-- bitbake/lib/bb/cooker.py | 159 +++++++----- bitbake/lib/bb/runqueue.py | 17 +- .../bb/tests/runqueue-tests/conf/bitbake.conf | 3 +- .../runqueue-tests/conf/multiconfig/mc1.conf | 1 + .../runqueue-tests/conf/multiconfig/mc2.conf | 1 + .../recipes/fails-mc/fails-mc1.bb | 5 + .../recipes/fails-mc/fails-mc2.bb | 4 + bitbake/lib/bb/tests/runqueue.py | 15 ++ bitbake/lib/bb/tinfoil.py | 24 +- bitbake/lib/bblayers/action.py | 22 +- bitbake/lib/bblayers/query.py | 4 +- 14 files changed, 399 insertions(+), 179 deletions(-) create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb
Signature · 1 line
2.26.2
PATCH v5 1/8 bitbake: cooker: Split file collections per multiconfig
Splits the cooker to track a collection per multiconfig instead of a single collection for all multiconfigs. Practically speaking, this allows each multiconfigs to each have different BBMASKs that apply to it instead of each one using the mask specified in the base configuration.
--- bitbake/lib/bb/cache.py | 2 +- bitbake/lib/bb/command.py | 37 +++++++-- bitbake/lib/bb/cooker.py | 139 +++++++++++++++++++-------------- bitbake/lib/bb/runqueue.py | 11 +-- bitbake/lib/bb/tinfoil.py | 24 +++--- bitbake/lib/bblayers/action.py | 22 ++++-- bitbake/lib/bblayers/query.py | 4 +- 7 files changed, 148 insertions(+), 91 deletions(-)
bitbake/lib/bb/cache.py +1 -1
@@ -623,7 +623,7 @@ class Cache(NoCache): self.remove(fn) return False - if appends != info_array[0].appends: + if tuple(appends) != tuple(info_array[0].appends): logger.debug(2, "Cache: appends for %s changed", fn) logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) self.remove(fn)
bitbake/lib/bb/command.py +29 -8
@@ -232,7 +232,11 @@ class CommandsSync: def matchFile(self, command, params): fMatch = params[0] - return command.cooker.matchFile(fMatch) + try: + mc = params[0] + except IndexError: + mc = '' + return command.cooker.matchFile(fMatch, mc) matchFile.needconfig = False def getUIHandlerNum(self, command, params):
@@ -395,22 +399,38 @@ class CommandsSync: def getSkippedRecipes(self, command, params): # Return list sorted by reverse priority order import bb.cache - skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), - key=lambda x: (-command.cooker.collection.calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0]))) + def sortkey(x): + vfn, _ = x + realfn, _, mc = bb.cache.virtualfn2realfn(vfn) + return (-command.cooker.collections[mc].calc_bbfile_priority(realfn), vfn) + + skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), key=sortkey)) return list(skipdict.items()) getSkippedRecipes.readonly = True def getOverlayedRecipes(self, command, params): - return list(command.cooker.collection.overlayed.items()) + try: + mc = params[0] + except IndexError: + mc = '' + return list(command.cooker.collections[mc].overlayed.items()) getOverlayedRecipes.readonly = True def getFileAppends(self, command, params): fn = params[0] - return command.cooker.collection.get_file_appends(fn) + try: + mc = params[1] + except IndexError: + mc = '' + return command.cooker.collections[mc].get_file_appends(fn) getFileAppends.readonly = True def getAllAppends(self, command, params): - return command.cooker.collection.bbappends + try: + mc = params[0] + except IndexError: + mc = '' + return command.cooker.collections[mc].bbappends getAllAppends.readonly = True def findProviders(self, command, params):
@@ -496,6 +516,7 @@ class CommandsSync: for the recipe. """ fn = params[0] + mc = bb.runqueue.mc_from_tid(fn) appends = params[1] appendlist = params[2] if len(params) > 3:
@@ -507,7 +528,7 @@ class CommandsSync: if appendlist is not None: appendfiles = appendlist else: - appendfiles = command.cooker.collection.get_file_appends(fn) + appendfiles = command.cooker.collections[mc].get_file_appends(fn) else: appendfiles = [] # We are calling bb.cache locally here rather than on the server,
@@ -517,7 +538,7 @@ class CommandsSync: if config_data: # We have to use a different function here if we're passing in a datastore # NOTE: we took a copy above, so we don't do it here again - envdata = bb.cache.parse_recipe(config_data, fn, appendfiles)[''] + envdata = bb.cache.parse_recipe(config_data, fn, appendfiles, mc)[''] else: # Use the standard path parser = bb.cache.NoCache(command.cooker.databuilder)
bitbake/lib/bb/cooker.py +82 -57
268 lines not on this page · raw message
bitbake/lib/bb/runqueue.py +6 -5
@@ -1557,7 +1557,8 @@ class RunQueue: def rq_dump_sigfn(self, fn, options): bb_cache = bb.cache.NoCache(self.cooker.databuilder) - the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn)) + mc = bb.runqueue.mc_from_tid(fn) + the_data = bb_cache.loadDataFull(fn, self.cooker.collections[mc].get_file_appends(fn)) siggen = bb.parse.siggen dataCaches = self.rqdata.dataCaches siggen.dump_sigfn(fn, dataCaches, options)
@@ -2042,10 +2043,10 @@ class RunQueueExecute: if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: if not mc in self.rq.fakeworker: self.rq.start_fakeworker(self, mc) - self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") + self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") self.rq.fakeworker[mc].process.stdin.flush() else: - self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") + self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") self.rq.worker[mc].process.stdin.flush() self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
@@ -2129,10 +2130,10 @@ class RunQueueExecute: self.rq.state = runQueueFailed self.stats.taskFailed() return True - self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") + self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") self.rq.fakeworker[mc].process.stdin.flush() else: - self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") + self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>") self.rq.worker[mc].process.stdin.flush() self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
bitbake/lib/bb/tinfoil.py +13 -11
@@ -117,15 +117,16 @@ class TinfoilCookerAdapter: class TinfoilCookerCollectionAdapter: """ cooker.collection adapter """ - def __init__(self, tinfoil): + def __init__(self, tinfoil, mc=''): self.tinfoil = tinfoil + self.mc = mc def get_file_appends(self, fn): - return self.tinfoil.get_file_appends(fn) + return self.tinfoil.get_file_appends(fn, self.mc) def __getattr__(self, name): if name == 'overlayed': - return self.tinfoil.get_overlayed_recipes() + return self.tinfoil.get_overlayed_recipes(self.mc) elif name == 'bbappends': - return self.tinfoil.run_command('getAllAppends') + return self.tinfoil.run_command('getAllAppends', self.mc) else: raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
@@ -185,10 +186,11 @@ class TinfoilCookerAdapter: def __init__(self, tinfoil): self.tinfoil = tinfoil - self.collection = self.TinfoilCookerCollectionAdapter(tinfoil) + self.multiconfigs = [''] + (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split() + self.collections = {} self.recipecaches = {} - self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil) - for mc in (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split(): + for mc in self.multiconfigs: + self.collections[mc] = self.TinfoilCookerCollectionAdapter(tinfoil, mc) self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc) self._cache = {} def __getattr__(self, name):
@@ -492,11 +494,11 @@ class Tinfoil: raise Exception('Not connected to server (did you call .prepare()?)') return self.server_connection.events.waitEvent(timeout) - def get_overlayed_recipes(self): + def get_overlayed_recipes(self, mc=''): """ Find recipes which are overlayed (i.e. where recipes exist in multiple layers) """ - return defaultdict(list, self.run_command('getOverlayedRecipes')) + return defaultdict(list, self.run_command('getOverlayedRecipes', mc)) def get_skipped_recipes(self): """
@@ -534,11 +536,11 @@ class Tinfoil: raise bb.providers.NoProvider('Unable to find any recipe file matching "%s"' % pn) return best[3] - def get_file_appends(self, fn): + def get_file_appends(self, fn, mc=''): """ Find the bbappends for a recipe file """ - return self.run_command('getFileAppends', fn) + return self.run_command('getFileAppends', fn, mc) def all_recipes(self, mc='', sort=True): """
bitbake/lib/bblayers/action.py +15 -7
@@ -143,11 +143,12 @@ build results (as the layer priority order has effectively changed). applied_appends = [] for layer in layers: - overlayed = [] - for f in self.tinfoil.cooker.collection.overlayed.keys(): - for of in self.tinfoil.cooker.collection.overlayed[f]: - if of.startswith(layer): - overlayed.append(of) + overlayed = set() + for mc in self.tinfoil.cooker.multiconfigs: + for f in self.tinfoil.cooker.collections[mc].overlayed.keys(): + for of in self.tinfoil.cooker.collections[mc].overlayed[f]: + if of.startswith(layer): + overlayed.add(of) logger.plain('Copying files from %s...' % layer ) for root, dirs, files in os.walk(layer):
@@ -174,14 +175,21 @@ build results (as the layer priority order has effectively changed). logger.warning('Overwriting file %s', fdest) bb.utils.copyfile(f1full, fdest) if ext == '.bb': - for append in self.tinfoil.cooker.collection.get_file_appends(f1full): + appends = set() + for mc in self.tinfoil.cooker.multiconfigs: + appends |= set(self.tinfoil.cooker.collections[mc].get_file_appends(f1full)) + for append in appends: if layer_path_match(append): logger.plain(' Applying append %s to %s' % (append, fdest)) self.apply_append(append, fdest) applied_appends.append(append) # Take care of when some layers are excluded and yet we have included bbappends for those recipes - for b in self.tinfoil.cooker.collection.bbappends: + bbappends = set() + for mc in self.tinfoil.cooker.multiconfigs: + bbappends |= set(self.tinfoil.cooker.collections[mc].bbappends) + + for b in bbappends: (recipename, appendname) = b if appendname not in applied_appends: first_append = None
bitbake/lib/bblayers/query.py +2 -2
@@ -320,12 +320,12 @@ Lists recipes with the bbappends that apply to them as subitems. def get_appends_for_files(self, filenames): appended, notappended = [], [] for filename in filenames: - _, cls, _ = bb.cache.virtualfn2realfn(filename) + _, cls, mc = bb.cache.virtualfn2realfn(filename) if cls: continue basename = os.path.basename(filename) - appends = self.tinfoil.cooker.collection.get_file_appends(basename) + appends = self.tinfoil.cooker.collections[mc].get_file_appends(basename) if appends: appended.append((basename, list(appends))) else:
Signature · 1 line
2.26.2
PATCH v5 2/8 bitbake: cache: Use multiconfig aware caches
In reply to Joshua WattSplits the parsing cache to maintain one cache per multiconfig instead of one global cache. This is necessary now that the files and appends can vary for each multiconfig. A bb.cache.MulticonfigCache dictionary-like proxy object is created instead of a single bb.cache.Cache object. This object will create and properly initialize bb.cache.Cache object for each multiconfig, and each of these caches has a dedicated cache file with a name based on the multiconfig.
--- bitbake/lib/bb/cache.py | 143 +++++++++++++++++++++++++++++---------- bitbake/lib/bb/cooker.py | 42 +++++++----- 2 files changed, 133 insertions(+), 52 deletions(-)
bitbake/lib/bb/cache.py +108 -35
@@ -19,7 +19,7 @@ import os import logging import pickle -from collections import defaultdict +from collections import defaultdict, Mapping import bb.utils import re
@@ -27,8 +27,11 @@ logger = logging.getLogger("BitBake.Cache") __cache_version__ = "152" -def getCacheFile(path, filename, data_hash): - return os.path.join(path, filename + "." + data_hash) +def getCacheFile(path, filename, mc, data_hash): + mcspec = '' + if mc: + mcspec = ".%s" % mc + return os.path.join(path, filename + mcspec + "." + data_hash) # RecipeInfoCommon defines common data retrieving methods # from meta data for caches. CoreRecipeInfo as well as other
@@ -354,14 +357,14 @@ class Cache(NoCache): """ BitBake Cache implementation """ - - def __init__(self, databuilder, data_hash, caches_array): + def __init__(self, databuilder, mc, data_hash, caches_array): super().__init__(databuilder) data = databuilder.data # Pass caches_array information into Cache Constructor # It will be used later for deciding whether we # need extra cache file dump/load support + self.mc = mc self.caches_array = caches_array self.cachedir = data.getVar("CACHE") self.clean = set()
@@ -379,7 +382,17 @@ class Cache(NoCache): return self.has_cache = True - self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash) + + def getCacheFile(self, cachefile): + return getCacheFile(self.cachedir, cachefile, self.mc, self.data_hash) + + def prepare_cache(self, progress): + if not self.has_cache: + return 0 + + loaded = 0 + + self.cachefile = self.getCacheFile("bb_cache.dat") logger.debug(1, "Cache dir: %s", self.cachedir) bb.utils.mkdirhier(self.cachedir)
@@ -387,18 +400,22 @@ class Cache(NoCache): cache_ok = True if self.caches_array: for cache_class in self.caches_array: - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + cachefile = self.getCacheFile(cache_class.cachefile) cache_ok = cache_ok and os.path.exists(cachefile) cache_class.init_cacheData(self) if cache_ok: - self.load_cachefile() + loaded = self.load_cachefile(progress) elif os.path.isfile(self.cachefile): logger.info("Out of date cache found, rebuilding...") else: logger.debug(1, "Cache file %s not found, building..." % self.cachefile) # We don't use the symlink, its just for debugging convinience - symlink = os.path.join(self.cachedir, "bb_cache.dat") + if self.mc: + symlink = os.path.join(self.cachedir, "bb_cache.dat.%s" % self.mc) + else: + symlink = os.path.join(self.cachedir, "bb_cache.dat") + if os.path.exists(symlink): bb.utils.remove(symlink) try:
@@ -406,21 +423,30 @@ class Cache(NoCache): except OSError: pass - def load_cachefile(self): - cachesize = 0 - previous_progress = 0 - previous_percent = 0 + return loaded + + def cachesize(self): + if not self.has_cache: + return 0 - # Calculate the correct cachesize of all those cache files + cachesize = 0 for cache_class in self.caches_array: - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) - with open(cachefile, "rb") as cachefile: - cachesize += os.fstat(cachefile.fileno()).st_size + cachefile = self.getCacheFile(cache_class.cachefile) + try: + with open(cachefile, "rb") as cachefile: + cachesize += os.fstat(cachefile.fileno()).st_size + except FileNotFoundError: + pass - bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data) + return cachesize + + def load_cachefile(self, progress): + cachesize = self.cachesize() + previous_progress = 0 + previous_percent = 0 for cache_class in self.caches_array: - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + cachefile = self.getCacheFile(cache_class.cachefile) logger.debug(1, 'Loading cache file: %s' % cachefile) with open(cachefile, "rb") as cachefile: pickled = pickle.Unpickler(cachefile)
@@ -460,23 +486,11 @@ class Cache(NoCache): self.depends_cache[key] = [value] # only fire events on even percentage boundaries current_progress = cachefile.tell() + previous_progress - if current_progress > cachesize: - # we might have calculated incorrect total size because a file - # might've been written out just after we checked its size - cachesize = current_progress - current_percent = 100 * current_progress / cachesize - if current_percent > previous_percent: - previous_percent = current_percent - bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize), - self.data) + progress(cachefile.tell() + previous_progress) previous_progress += current_progress - # Note: depends cache number is corresponding to the parsing file numbers. - # The same file has several caches, still regarded as one item in the cache - bb.event.fire(bb.event.CacheLoadCompleted(cachesize, - len(self.depends_cache)), - self.data) + return len(self.depends_cache) def parse(self, filename, appends): """Parse the specified filename, returning the recipe information"""
@@ -682,7 +696,7 @@ class Cache(NoCache): for cache_class in self.caches_array: cache_class_name = cache_class.__name__ - cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash) + cachefile = self.getCacheFile(cache_class.cachefile) with open(cachefile, "wb") as f: p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL) p.dump(__cache_version__)
@@ -701,7 +715,7 @@ class Cache(NoCache): return bb.parse.cached_mtime_noerror(cachefile) def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None): - if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): + if cacheData is not None and isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): cacheData.add_from_recipeinfo(filename, info_array) if watcher:
@@ -727,6 +741,65 @@ class Cache(NoCache): info_array.append(cache_class(realfn, data)) self.add_info(file_name, info_array, cacheData, parsed) +class MulticonfigCache(Mapping): + def __init__(self, databuilder, data_hash, caches_array): + def progress(p): + nonlocal current_progress + nonlocal previous_progress + nonlocal previous_percent + nonlocal cachesize + + current_progress = previous_progress + p + + if current_progress > cachesize: + # we might have calculated incorrect total size because a file + # might've been written out just after we checked its size + cachesize = current_progress + current_percent = 100 * current_progress / cachesize + if current_percent > previous_percent: + previous_percent = current_percent + bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize), + databuilder.data) + + + cachesize = 0 + current_progress = 0 + previous_progress = 0 + previous_percent = 0 + self.__caches = {} + + for mc, mcdata in databuilder.mcdata.items(): + self.__caches[mc] = Cache(databuilder, mc, data_hash, caches_array) + + cachesize += self.__caches[mc].cachesize() + + bb.event.fire(bb.event.CacheLoadStarted(cachesize), databuilder.data) + loaded = 0 + + for c in self.__caches.values(): + loaded += c.prepare_cache(progress) + previous_progress = current_progress + + # Note: depends cache number is corresponding to the parsing file numbers. + # The same file has several caches, still regarded as one item in the cache + bb.event.fire(bb.event.CacheLoadCompleted(cachesize, loaded), databuilder.data) + + def __len__(self): + return len(self.__caches) + + def __getitem__(self, key): + return self.__caches[key] + + def __contains__(self, key): + return key in self.__caches + + def __iter__(self): + for k in self.__caches: + yield k + + def keys(self): + return self.__caches[key] + def init(cooker): """
bitbake/lib/bb/cooker.py +25 -17
108 lines not on this page · raw message
Signature · 1 line
2.26.2
PATCH v5 3/8 bitbake: lib: Add support for Logging Adapters
In reply to Joshua WattCreates a BBLoggingAdapter class that is monkey patched in place of the logginer.LoggingAdapter. The new adapter is compatible with the BBLogger class API, allowing adapters to be created for bitbake loggers. A new BBLoggerMixin class is used to reduce code duplication between the BBLogger and BBLoggerAdapter classes.
--- bitbake/lib/bb/__init__.py | 42 ++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-)
bitbake/lib/bb/__init__.py +38 -4
@@ -35,12 +35,14 @@ class NullHandler(logging.Handler): def emit(self, record): pass -Logger = logging.getLoggerClass() -class BBLogger(Logger): - def __init__(self, name): +class BBLoggerMixin(object): + def __init__(self, *args, **kwargs): + # Does nothing to allow calling super() from derived classes + pass + + def setup_bblogger(self, name): if name.split(".")[0] == "BitBake": self.debug = self.bbdebug - Logger.__init__(self, name) def bbdebug(self, level, msg, *args, **kwargs): loglevel = logging.DEBUG - level + 1
@@ -60,10 +62,42 @@ class BBLogger(Logger): def verbnote(self, msg, *args, **kwargs): return self.log(logging.INFO + 2, msg, *args, **kwargs) +Logger = logging.getLoggerClass() +class BBLogger(Logger, BBLoggerMixin): + def __init__(self, name, *args, **kwargs): + self.setup_bblogger(name) + super().__init__(name, *args, **kwargs) logging.raiseExceptions = False logging.setLoggerClass(BBLogger) +class BBLoggerAdapter(logging.LoggerAdapter, BBLoggerMixin): + def __init__(self, logger, *args, **kwargs): + self.setup_bblogger(logger.name) + super().__init__(logger, *args, **kwargs) + + if sys.version_info < (3, 6): + # These properties were added in Python 3.6. Add them in older versions + # for compatibility + @property + def manager(self): + return self.logger.manager + + @manager.setter + def manager(self, value): + self.logger.manager = value + + @property + def name(self): + return self.logger.name + + def __repr__(self): + logger = self.logger + level = getLevelName(logger.getEffectiveLevel()) + return '<%s %s (%s)>' % (self.__class__.__name__, logger.name, level) + +logging.LoggerAdapter = BBLoggerAdapter + logger = logging.getLogger("BitBake") logger.addHandler(NullHandler()) logger.setLevel(logging.DEBUG - 2)
Signature · 1 line
2.26.2
PATCH v5 4/8 bitbake: lib: Add PrefixLoggerAdapter helper
In reply to Joshua WattAdds a helper logger adapter to add a prefix to all log messages. This is useful to distinguish log messages between multiple instances of a object.
--- bitbake/lib/bb/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+)
bitbake/lib/bb/__init__.py +8 -0
@@ -104,6 +104,14 @@ logger.setLevel(logging.DEBUG - 2) mainlogger = logging.getLogger("BitBake.Main") +class PrefixLoggerAdapter(logging.LoggerAdapter): + def __init__(self, prefix, logger): + super().__init__(logger, {}) + self.__msg_prefix = prefix + + def process(self, msg, kwargs): + return "%s%s" %(self.__msg_prefix, msg), kwargs + # This has to be imported after the setLoggerClass, as the import of bb.msg # can result in construction of the various loggers. import bb.msg
Signature · 1 line
2.26.2
PATCH v5 5/8 bitbake: cache: Improve logging
In reply to Joshua WattImproves the logging of Cache objects by prefixing the log messages with the multiconfig name of the cache, so as to distinguish between multiple instances of the class. Also adds a more log messages.
--- bitbake/lib/bb/cache.py | 65 ++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 30 deletions(-)
bitbake/lib/bb/cache.py +35 -30
@@ -21,6 +21,7 @@ import logging import pickle from collections import defaultdict, Mapping import bb.utils +from bb import PrefixLoggerAdapter import re logger = logging.getLogger("BitBake.Cache")
@@ -365,6 +366,7 @@ class Cache(NoCache): # It will be used later for deciding whether we # need extra cache file dump/load support self.mc = mc + self.logger = PrefixLoggerAdapter("Cache: %s: " % (mc if mc else "default"), logger) self.caches_array = caches_array self.cachedir = data.getVar("CACHE") self.clean = set()
@@ -377,8 +379,8 @@ class Cache(NoCache): if self.cachedir in [None, '']: self.has_cache = False - logger.info("Not using a cache. " - "Set CACHE = <directory> to enable.") + self.logger.info("Not using a cache. " + "Set CACHE = <directory> to enable.") return self.has_cache = True
@@ -394,21 +396,23 @@ class Cache(NoCache): self.cachefile = self.getCacheFile("bb_cache.dat") - logger.debug(1, "Cache dir: %s", self.cachedir) + self.logger.debug(1, "Cache dir: %s", self.cachedir) bb.utils.mkdirhier(self.cachedir) cache_ok = True if self.caches_array: for cache_class in self.caches_array: cachefile = self.getCacheFile(cache_class.cachefile) - cache_ok = cache_ok and os.path.exists(cachefile) + cache_exists = os.path.exists(cachefile) + self.logger.debug(2, "Checking if %s exists: %r", cachefile, cache_exists) + cache_ok = cache_ok and cache_exists cache_class.init_cacheData(self) if cache_ok: loaded = self.load_cachefile(progress) elif os.path.isfile(self.cachefile): - logger.info("Out of date cache found, rebuilding...") + self.logger.info("Out of date cache found, rebuilding...") else: - logger.debug(1, "Cache file %s not found, building..." % self.cachefile) + self.logger.debug(1, "Cache file %s not found, building..." % self.cachefile) # We don't use the symlink, its just for debugging convinience if self.mc:
@@ -447,7 +451,7 @@ class Cache(NoCache): for cache_class in self.caches_array: cachefile = self.getCacheFile(cache_class.cachefile) - logger.debug(1, 'Loading cache file: %s' % cachefile) + self.logger.debug(1, 'Loading cache file: %s' % cachefile) with open(cachefile, "rb") as cachefile: pickled = pickle.Unpickler(cachefile) # Check cache version information
@@ -455,14 +459,14 @@ class Cache(NoCache): cache_ver = pickled.load() bitbake_ver = pickled.load() except Exception: - logger.info('Invalid cache, rebuilding...') + self.logger.info('Invalid cache, rebuilding...') return if cache_ver != __cache_version__: - logger.info('Cache version mismatch, rebuilding...') + self.logger.info('Cache version mismatch, rebuilding...') return elif bitbake_ver != bb.__version__: - logger.info('Bitbake version mismatch, rebuilding...') + self.logger.info('Bitbake version mismatch, rebuilding...') return # Load the rest of the cache file
@@ -494,7 +498,7 @@ class Cache(NoCache): def parse(self, filename, appends): """Parse the specified filename, returning the recipe information""" - logger.debug(1, "Parsing %s", filename) + self.logger.debug(1, "Parsing %s", filename) infos = [] datastores = self.load_bbfile(filename, appends) depends = []
@@ -548,7 +552,7 @@ class Cache(NoCache): cached, infos = self.load(fn, appends) for virtualfn, info_array in infos: if info_array[0].skipped: - logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason) + self.logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason) skipped += 1 else: self.add_info(virtualfn, info_array, cacheData, not cached)
@@ -584,21 +588,21 @@ class Cache(NoCache): # File isn't in depends_cache if not fn in self.depends_cache: - logger.debug(2, "Cache: %s is not cached", fn) + self.logger.debug(2, "%s is not cached", fn) return False mtime = bb.parse.cached_mtime_noerror(fn) # Check file still exists if mtime == 0: - logger.debug(2, "Cache: %s no longer exists", fn) + self.logger.debug(2, "%s no longer exists", fn) self.remove(fn) return False info_array = self.depends_cache[fn] # Check the file's timestamp if mtime != info_array[0].timestamp: - logger.debug(2, "Cache: %s changed", fn) + self.logger.debug(2, "%s changed", fn) self.remove(fn) return False
@@ -609,14 +613,14 @@ class Cache(NoCache): fmtime = bb.parse.cached_mtime_noerror(f) # Check if file still exists if old_mtime != 0 and fmtime == 0: - logger.debug(2, "Cache: %s's dependency %s was removed", - fn, f) + self.logger.debug(2, "%s's dependency %s was removed", + fn, f) self.remove(fn) return False if (fmtime != old_mtime): - logger.debug(2, "Cache: %s's dependency %s changed", - fn, f) + self.logger.debug(2, "%s's dependency %s changed", + fn, f) self.remove(fn) return False
@@ -632,14 +636,14 @@ class Cache(NoCache): continue f, exist = f.split(":") if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)): - logger.debug(2, "Cache: %s's file checksum list file %s changed", - fn, f) + self.logger.debug(2, "%s's file checksum list file %s changed", + fn, f) self.remove(fn) return False if tuple(appends) != tuple(info_array[0].appends): - logger.debug(2, "Cache: appends for %s changed", fn) - logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) + self.logger.debug(2, "appends for %s changed", fn) + self.logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends))) self.remove(fn) return False
@@ -648,10 +652,10 @@ class Cache(NoCache): virtualfn = variant2virtual(fn, cls) self.clean.add(virtualfn) if virtualfn not in self.depends_cache: - logger.debug(2, "Cache: %s is not cached", virtualfn) + self.logger.debug(2, "%s is not cached", virtualfn) invalid = True elif len(self.depends_cache[virtualfn]) != len(self.caches_array): - logger.debug(2, "Cache: Extra caches missing for %s?" % virtualfn) + self.logger.debug(2, "Extra caches missing for %s?" % virtualfn) invalid = True # If any one of the variants is not present, mark as invalid for all
@@ -659,10 +663,10 @@ class Cache(NoCache): for cls in info_array[0].variants: virtualfn = variant2virtual(fn, cls) if virtualfn in self.clean: - logger.debug(2, "Cache: Removing %s from cache", virtualfn) + self.logger.debug(2, "Removing %s from cache", virtualfn) self.clean.remove(virtualfn) if fn in self.clean: - logger.debug(2, "Cache: Marking %s as not clean", fn) + self.logger.debug(2, "Marking %s as not clean", fn) self.clean.remove(fn) return False
@@ -675,10 +679,10 @@ class Cache(NoCache): Called from the parser in error cases """ if fn in self.depends_cache: - logger.debug(1, "Removing %s from cache", fn) + self.logger.debug(1, "Removing %s from cache", fn) del self.depends_cache[fn] if fn in self.clean: - logger.debug(1, "Marking %s as unclean", fn) + self.logger.debug(1, "Marking %s as unclean", fn) self.clean.remove(fn) def sync(self):
@@ -691,12 +695,13 @@ class Cache(NoCache): return if self.cacheclean: - logger.debug(2, "Cache is clean, not saving.") + self.logger.debug(2, "Cache is clean, not saving.") return for cache_class in self.caches_array: cache_class_name = cache_class.__name__ cachefile = self.getCacheFile(cache_class.cachefile) + self.logger.debug(2, "Writing %s", cachefile) with open(cachefile, "wb") as f: p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL) p.dump(__cache_version__)
Signature · 1 line
2.26.2
PATCH v5 6/8 bitbake: cache: Cache size optimization
In reply to Joshua WattNow that there is a cache object per multiconfig, it is not necessary for each cache object to parse all other multiconfigs. Instead, each cache now only parses the files for it's multiconfig.
--- bitbake/lib/bb/cache.py | 22 ++++++++++++++++++---- bitbake/lib/bb/cooker.py | 8 +------- 2 files changed, 19 insertions(+), 11 deletions(-)
bitbake/lib/bb/cache.py +18 -4
@@ -328,7 +328,7 @@ class NoCache(object): bb_data = self.load_bbfile(virtualfn, appends, virtonly=True) return bb_data[virtual] - def load_bbfile(self, bbfile, appends, virtonly = False): + def load_bbfile(self, bbfile, appends, virtonly = False, mc=None): """ Load and parse one .bb build file Return the data and whether parsing resulted in the file being skipped
@@ -341,6 +341,10 @@ class NoCache(object): datastores = parse_recipe(bb_data, bbfile, appends, mc) return datastores + if mc is not None: + bb_data = self.databuilder.mcdata[mc].createCopy() + return parse_recipe(bb_data, bbfile, appends, mc) + bb_data = self.data.createCopy() datastores = parse_recipe(bb_data, bbfile, appends)
@@ -500,7 +504,7 @@ class Cache(NoCache): """Parse the specified filename, returning the recipe information""" self.logger.debug(1, "Parsing %s", filename) infos = [] - datastores = self.load_bbfile(filename, appends) + datastores = self.load_bbfile(filename, appends, mc=self.mc) depends = [] variants = [] # Process the "real" fn last so we can store variants list
@@ -720,8 +724,18 @@ class Cache(NoCache): return bb.parse.cached_mtime_noerror(cachefile) def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None): - if cacheData is not None and isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): - cacheData.add_from_recipeinfo(filename, info_array) + if self.mc is not None: + (fn, cls, mc) = virtualfn2realfn(filename) + if mc: + self.logger.error("Unexpected multiconfig %s", virtualfn) + return + + vfn = realfn2virtual(fn, cls, self.mc) + else: + vfn = filename + + if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): + cacheData.add_from_recipeinfo(vfn, info_array) if watcher: watcher(info_array[0].file_depends)
bitbake/lib/bb/cooker.py +1 -7
@@ -2199,13 +2199,7 @@ class CookerParser(object): if info_array[0].skipped: self.skipped += 1 self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0]) - (fn, cls, fnmc) = bb.cache.virtualfn2realfn(virtualfn) - - if fnmc == mc: - cache = self.cooker.recipecaches[mc] - else: - cache = None - self.bb_caches[mc].add_info(virtualfn, info_array, cache, + self.bb_caches[mc].add_info(virtualfn, info_array, self.cooker.recipecaches[mc], parsed=parsed, watcher = self.cooker.add_filewatch) return True
Signature · 1 line
2.26.2
PATCH v5 7/8 bitbake: tests: Add tests for BBMASK in multiconfig
In reply to Joshua WattAdds a test to validate that multiconfigs can independently mask off recipes by setting BBMASK. See the test description for further information about how the test works.
--- .../lib/bb/tests/runqueue-tests/conf/bitbake.conf | 3 ++- .../runqueue-tests/conf/multiconfig/mc1.conf | 1 + .../runqueue-tests/conf/multiconfig/mc2.conf | 1 + .../runqueue-tests/recipes/fails-mc/fails-mc1.bb | 5 +++++ .../runqueue-tests/recipes/fails-mc/fails-mc2.bb | 4 ++++ bitbake/lib/bb/tests/runqueue.py | 15 +++++++++++++++ 6 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb create mode 100644 bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb
bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf +2 -1
@@ -1,7 +1,8 @@ CACHE = "${TOPDIR}/cache" THISDIR = "${@os.path.dirname(d.getVar('FILE'))}" COREBASE := "${@os.path.normpath(os.path.dirname(d.getVar('FILE')+'/../../'))}" -BBFILES = "${COREBASE}/recipes/*.bb" +EXTRA_BBFILES ?= "" +BBFILES = "${COREBASE}/recipes/*.bb ${EXTRA_BBFILES}" PROVIDES = "${PN}" PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}" PF = "${BB_CURRENT_MC}:${PN}"
bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf +1 -0
@@ -1 +1,2 @@ TMPDIR = "${TOPDIR}/mc1/" +BBMASK += "recipes/fails-mc/fails-mc1.bb"
bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf +1 -0
@@ -1 +1,2 @@ TMPDIR = "${TOPDIR}/mc2/" +BBMASK += "recipes/fails-mc/fails-mc2.bb"
bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb new file +5 -0
@@ -0,0 +1,5 @@ +python () { + if d.getVar("BB_CURRENT_MC") == "mc1": + bb.fatal("Multiconfig is mc1") +} +
bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb new file +4 -0
@@ -0,0 +1,4 @@ +python () { + if d.getVar("BB_CURRENT_MC") == "mc2": + bb.fatal("Multiconfig is mc2") +}
bitbake/lib/bb/tests/runqueue.py +15 -0
@@ -232,6 +232,21 @@ class RunQueueTests(unittest.TestCase): expected.remove(x) self.assertEqual(set(tasks), set(expected)) + def test_multiconfig_bbmask(self): + # This test validates that multiconfigs can independently mask off + # recipes they do not want with BBMASK. It works by having recipes + # that will fail to parse for mc1 and mc2, then making each multiconfig + # build the one that does parse. This ensures that the recipes are in + # each multiconfigs BBFILES, but each is masking only the one that + # doesn't parse + with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: + extraenv = { + "BBMULTICONFIG" : "mc1 mc2", + "BB_SIGNATURE_HANDLER" : "basic", + "EXTRA_BBFILES": "${COREBASE}/recipes/fails-mc/*.bb", + } + cmd = ["bitbake", "mc:mc1:fails-mc2", "mc:mc2:fails-mc1"] + self.run_bitbakecmd(cmd, tempdir, "", extraenv=extraenv) @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required') def test_hashserv_single(self):
Signature · 1 line
2.26.2
PATCH v5 8/8 bitbake: command: Move split_mc_pn to runqueue
In reply to Joshua WattAll of the other multiconfig splitting functions are located in runqueue so move the function to split a pn/fn there also so that its easier to see them all together. Fixes a case where the findBestProvider() command wasn't working for multiconfig because it was looking for a prefix of "multiconfig:" instead of the newer "mc:"
--- bitbake/lib/bb/command.py | 8 +------- bitbake/lib/bb/runqueue.py | 6 ++++++ 2 files changed, 7 insertions(+), 7 deletions(-)
bitbake/lib/bb/command.py +1 -7
@@ -138,12 +138,6 @@ class Command: def reset(self): self.remotedatastores = bb.remotedata.RemoteDatastores(self.cooker) -def split_mc_pn(pn): - if pn.startswith("multiconfig:"): - _, mc, pn = pn.split(":", 2) - return (mc, pn) - return ('', pn) - class CommandsSync: """ A class of synchronous commands
@@ -442,7 +436,7 @@ class CommandsSync: findProviders.readonly = True def findBestProvider(self, command, params): - (mc, pn) = split_mc_pn(params[0]) + (mc, pn) = bb.runqueue.split_mc(params[0]) return command.cooker.findBestProvider(pn, mc) findBestProvider.readonly = True
bitbake/lib/bb/runqueue.py +6 -0
@@ -46,6 +46,12 @@ def split_tid(tid): (mc, fn, taskname, _) = split_tid_mcfn(tid) return (mc, fn, taskname) +def split_mc(n): + if n.startswith("mc:"): + _, mc, n = n.split(":", 2) + return (mc, n) + return ('', n) + def split_tid_mcfn(tid): if tid.startswith('mc:'): elems = tid.split(':')
Signature · 1 line
2.26.2
Quoted 17 lines · Joshua Watt
Splits apart the bitbake configuration data and caches so that each defined multiconfig can specify as separate BBMASK independently of the
other multiconfigs. This can take multiple forms, such as aBBMASKin
a multiconfig .conf file, or BBMASK assignment based on an override (see
linked bugzilla for an example).The most noticeable change is that bitbake now keeps a parsing cache file for each multiconfig instead of a single parsing cache file that caches all multiconfigs.
[YOCTO #13721]
V2: Fixes some syntax errors in tinfoil
Quoted 8 lines
V3: Optimizes caches so that each one only caches the files for it's own
multiconfigV4: Fixes compatability with Python < 3.6
Renames split_pn -> split_mcV5: Fixes oe-selftest errors
I think there is an oe-selftest issue still left on centos7:
https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/1024
(and I've messed up the bitbake selftest patch merging I think :/)
Cheers,
Richard