More Bitbake logging improvements
cover More Bitbake logging improvements
Supplemental patches to improve logging in bitbake by using python structured logging. Fixes up some ResourceWarnings and improves the flexibility of the console and consolelog.
Joshua Watt (4):
bitbake: knotty: Add logging cleanup
bitbake: msg: Add helper to merge logging configs
bitbake: knotty: Update hash equivalence logging
bitbake: runqueue: Lower setscene complete logging level
bitbake/contrib/autobuilderlog.json | 18 +---- bitbake/lib/bb/msg.py | 100 ++++++++++++++++------------ bitbake/lib/bb/runqueue.py | 2 +- bitbake/lib/bb/ui/knotty.py | 70 ++++++++++++++++--- 4 files changed, 124 insertions(+), 66 deletions(-)
Signature · 1 line
2.17.1
PATCH 1/4 bitbake: knotty: Add logging cleanup
Adds code to close all loggers when bitbake exits. This prevents unclosed file ResourceWarnings. A form of this closing existed previously, but was removed in the new logging code.
--- bitbake/lib/bb/msg.py | 11 +++++++++++ bitbake/lib/bb/ui/knotty.py | 2 ++ 2 files changed, 13 insertions(+)
bitbake/lib/bb/msg.py +11 -0
@@ -284,3 +284,14 @@ def setLoggingConfig(defaultconfig, userconfigfile=None): # TODO: I don't think that setting the global log level should be necessary #if newlevel < bb.msg.loggerDefaultLogLevel: # bb.msg.loggerDefaultLogLevel = newlevel + +def cleanupLogging(): + # Iterate through all the handlers and close them if possible. Fixes + # 'Unclosed resource' warnings when bitbake exits, see + # https://bugs.python.org/issue23010 + handlers = set() + for logger_iter in logging.Logger.manager.loggerDict.keys(): + handlers.update(logging.getLogger(logger_iter).handlers) + + for h in handlers: + h.close()
bitbake/lib/bb/ui/knotty.py +2 -0
@@ -823,4 +823,6 @@ def main(server, eventHandler, params, tf = TerminalFilter): if e.errno == errno.EPIPE: pass + bb.msg.cleanupLogging() + return return_value
Signature · 1 line
2.17.1
PATCH 2/4 bitbake: msg: Add helper to merge logging configs
In reply to Joshua WattAdds a function that can be used by UI front ends to merge logging configuration fragments.
--- bitbake/lib/bb/msg.py | 89 +++++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 41 deletions(-)
bitbake/lib/bb/msg.py +48 -41
@@ -193,6 +193,53 @@ def has_console_handler(logger): return True return False +def mergeLoggingConfig(logconfig, userconfig): + logconfig = copy.deepcopy(logconfig) + userconfig = copy.deepcopy(userconfig) + + # Merge config with the default config + if userconfig.get('version') != logconfig['version']: + raise BaseException("Bad user configuration version. Expected %r, got %r" % (logconfig['version'], userconfig.get('version'))) + + # Set some defaults to make merging easier + userconfig.setdefault("loggers", {}) + + # If a handler, formatter, or filter is defined in the user + # config, it will replace an existing one in the default config + for k in ("handlers", "formatters", "filters"): + logconfig.setdefault(k, {}).update(userconfig.get(k, {})) + + seen_loggers = set() + for name, l in logconfig["loggers"].items(): + # If the merge option is set, merge the handlers and + # filters. Otherwise, if it is False, this logger won't get + # add to the set of seen loggers and will replace the + # existing one + if l.get('bitbake_merge', True): + ulogger = userconfig["loggers"].setdefault(name, {}) + ulogger.setdefault("handlers", []) + ulogger.setdefault("filters", []) + + # Merge lists + l.setdefault("handlers", []).extend(ulogger["handlers"]) + l.setdefault("filters", []).extend(ulogger["filters"]) + + # Replace other properties if present + if "level" in ulogger: + l["level"] = ulogger["level"] + + if "propagate" in ulogger: + l["propagate"] = ulogger["propagate"] + + seen_loggers.add(name) + + # Add all loggers present in the user config, but not any that + # have already been processed + for name in set(userconfig["loggers"].keys()) - seen_loggers: + logconfig["loggers"][name] = userconfig["loggers"][name] + + return logconfig + def setLoggingConfig(defaultconfig, userconfigfile=None): logconfig = copy.deepcopy(defaultconfig)
@@ -208,47 +255,7 @@ def setLoggingConfig(defaultconfig, userconfigfile=None): raise BaseException("Unrecognized file format: %s" % userconfigfile) if userconfig.get('bitbake_merge', True): - # Merge config with the default config - if userconfig.get('version') != logconfig['version']: - raise BaseException("Bad user configuration version. Expected %r, got %r" % (logconfig['version'], userconfig.get('version'))) - - # Set some defaults to make merging easier - userconfig.setdefault("loggers", {}) - - # If a handler, formatter, or filter is defined in the user - # config, it will replace an existing one in the default config - for k in ("handlers", "formatters", "filters"): - logconfig.setdefault(k, {}).update(userconfig.get(k, {})) - - seen_loggers = set() - for name, l in logconfig["loggers"].items(): - # If the merge option is set, merge the handlers and - # filters. Otherwise, if it is False, this logger won't get - # add to the set of seen loggers and will replace the - # existing one - if l.get('bitbake_merge', True): - ulogger = userconfig["loggers"].setdefault(name, {}) - ulogger.setdefault("handlers", []) - ulogger.setdefault("filters", []) - - # Merge lists - l.setdefault("handlers", []).extend(ulogger["handlers"]) - l.setdefault("filters", []).extend(ulogger["filters"]) - - # Replace other properties if present - if "level" in ulogger: - l["level"] = ulogger["level"] - - if "propagate" in ulogger: - l["propagate"] = ulogger["propagate"] - - seen_loggers.add(name) - - # Add all loggers present in the user config, but not any that - # have already been processed - for name in set(userconfig["loggers"].keys()) - seen_loggers: - logconfig["loggers"][name] = userconfig["loggers"][name] - + logconfig = mergeLoggingConfig(logconfig, userconfig) else: # Replace the entire default config logconfig = userconfig
Signature · 1 line
2.17.1
PATCH 3/4 bitbake: knotty: Update hash equivalence logging
In reply to Joshua WattUpdates hash equivalence logging so that the interesting VERBOSE messages are always logged to the consolelog file so that issues in individual user builds can be diagnosed. The autobuilder logging config then updates this so that they also are shown on stdout, since the consolelog file is not capture there.
In order to facilitate this, 2 new logging handlers were added, "BitBake.verbconsole" and "BitBake.verbconsolelog". Neither of these handlers are attached to anything by default, but they will log any messages that wouldn't otherwise be logged by the normal console or consolelog handlers. Users can attach whatever loggers the desire to this handler to get them to appear on the console or in the consolelog, as demonstrated by the autobuilderlog.json file.
--- bitbake/contrib/autobuilderlog.json | 18 +------- bitbake/lib/bb/ui/knotty.py | 68 +++++++++++++++++++++++++---- 2 files changed, 62 insertions(+), 24 deletions(-)
bitbake/contrib/autobuilderlog.json +2 -16
@@ -1,27 +1,13 @@ { "version": 1, - "handlers": { - "autobuilderlog": { - "class": "logging.FileHandler", - "formatter": "logfileFormatter", - "level": "DEBUG", - "filename": "autobuilder.log", - "mode": "w" - } - }, - "formatters": { - "logfileFormatter": { - "format": "%(name)s: %(levelname)s: %(message)s" - } - }, "loggers": { "BitBake.SigGen.HashEquiv": { "level": "VERBOSE", - "handlers": ["autobuilderlog"] + "handlers": ["BitBake.verbconsole"] }, "BitBake.RunQueue.HashEquiv": { "level": "VERBOSE", - "handlers": ["autobuilderlog"] + "handlers": ["BitBake.verbconsole"] } } }
bitbake/lib/bb/ui/knotty.py +60 -8
86 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH 4/4 bitbake: runqueue: Lower setscene complete logging level
In reply to Joshua WattLowers the level of the log message when setscene tasks have completed. This message can occur multiple times when hash equivalence is enabled, since the runqueue switches between executing setscene tasks and normal tasks. Since this is primarily of use when debugging hash equivalence, use the hash equivalence logger at VERBOSE level.
[YOCTO #13813]
--- bitbake/lib/bb/runqueue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
bitbake/lib/bb/runqueue.py +1 -1
@@ -2059,7 +2059,7 @@ class RunQueueExecute: self.update_holdofftasks() if not self.sq_live and not self.sqdone and not self.sq_deferred and not self.updated_taskhash_queue and not self.holdoff_tasks: - logger.info("Setscene tasks completed") + hashequiv_logger.verbose("Setscene tasks completed") err = self.summarise_scenequeue_errors() if err:
Signature · 1 line
2.17.1