bitbake/logging: Overhaul internal logging process
At the moment it bugs me a lot that we only have one effective logging level for bitbake, despite the logging module having provision to do more advanced things. This patch:
* Changes the core log level to the lowest level we have messages of (DEBUG-2) so messages always flow through the core logger
* Allows build.py's task logging code to log all the output regardless of what output is on the console and sets this so log files now always contain debug level messages even if these don't appear on the console
* Moves the verbose/debug/debug-domains code to be a UI side setting
* Adds a filter to the UI to only print the user requested output.
The result is more complete logfiles on disk but the usual output to the console.
There are some behaviour changes intentionally made by this patch:
a) the -v option now controls whether output is tee'd to the console.
Ultimately, we likely want to output a message to the user about where the log file is and avoid placing output directly onto the console for every executing task.
b) The functions get_debug_levels, the debug_levels variable, the set_debug_levels, the set_verbosity and set_debug_domains functions are removed from bb.msg.
c) The "logging" init function changes format.
d) All messages get fired to all handlers all the time leading to an increase in inter-process traffic. This could likely be hacked around short term with a function for a UI to only request events greater than level X. Longer term, having masks for event handlers would be better.
e) logger.getEffectiveLevel() is no longer a reliable guide to what will/won't get logged so for now we look at the default log levels instead.
[YOCTO #304]
---
bitbake/bin/bitbake +2 -1
@@ -190,7 +190,7 @@ Default BBFILES are the .bb files in the current directory.""") # server is daemonized this logfile will be truncated. cooker_logfile = os.path.join(os.getcwd(), "cooker.log") - bb.utils.init_logger(bb.msg, configuration.verbose, configuration.debug, + bb.msg.init_msgconfig(configuration.verbose, configuration.debug, configuration.debug_domains) # Ensure logging messages get sent to the UI as events
@@ -228,6 +228,7 @@ Default BBFILES are the .bb files in the current directory.""") try: return server.launchUI(ui_main, server_connection.connection, server_connection.events) finally: + bb.event.ui_queue = [] server_connection.terminate() return 1
bitbake/bin/bitdoc +2 -3
@@ -430,9 +430,8 @@ Create a set of html pages (documentation) for a bitbake.conf.... action = "store_true", dest = "verbose", default = False ) options, args = parser.parse_args( sys.argv ) - - if options.debug: - bb.msg.set_debug_level(options.debug) + + bb.msg.init_msgconfig(options.verbose, options.debug) return options.config, options.output
bitbake/lib/bb/__init__.py +1 -1
@@ -56,7 +56,7 @@ logging.setLoggerClass(BBLogger) logger = logging.getLogger("BitBake") logger.addHandler(NullHandler()) -logger.setLevel(logging.INFO) +logger.setLevel(logging.DEBUG - 2) # This has to be imported after the setLoggerClass, as the import of bb.msg # can result in construction of the various loggers.
bitbake/lib/bb/build.py +4 -2
@@ -223,7 +223,7 @@ def exec_func_shell(function, d, runfile, cwd=None): with open(runfile, 'w') as script: script.write('#!/bin/sh -e\n') - if logger.isEnabledFor(logging.DEBUG): + if bb.msg.loggerDefaults[1]: script.write("set -x\n") data.emit_func(function, script, d) if cwd:
@@ -234,7 +234,7 @@ def exec_func_shell(function, d, runfile, cwd=None): cmd = runfile - if logger.isEnabledFor(logging.DEBUG): + if bb.msg.loggerDefaults[1]: logfile = LogTee(logger, sys.stdout) else: logfile = sys.stdout
@@ -308,6 +308,8 @@ def _exec_task(fn, task, d, quieterr): # Ensure python logging goes to the logfile handler = logging.StreamHandler(logfile) handler.setFormatter(logformatter) + # Always enable full debug output into task logfiles + handler.setLevel(logging.DEBUG - 2) bblogger.addHandler(handler) localdata.setVar('BB_LOGFILE', logfn)
bitbake/lib/bb/msg.py +41 -29
103 lines not on this page · raw message
bitbake/lib/bb/ui/knotty.py +3 -2
@@ -74,6 +74,7 @@ def main(server, eventHandler): console = logging.StreamHandler(sys.stdout) format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") + bb.msg.addDefaultlogFilter(console) console.setFormatter(format) logger.addHandler(console)
@@ -120,8 +121,8 @@ def main(server, eventHandler): # For "normal" logging conditions, don't show note logs from tasks # but do show them if the user has changed the default log level to # include verbose/debug messages - if logger.getEffectiveLevel() > format.VERBOSE: - if event.taskpid != 0 and event.levelno <= format.NOTE: + #if logger.getEffectiveLevel() > format.VERBOSE: + if event.taskpid != 0 and event.levelno <= format.NOTE: continue logger.handle(event) continue
bitbake/lib/bb/utils.py +0 -15
@@ -830,21 +830,6 @@ def which(path, item, direction = 0): return "" -def init_logger(logger, verbose, debug, debug_domains): - """ - Set verbosity and debug levels in the logger - """ - - if debug: - bb.msg.set_debug_level(debug) - elif verbose: - bb.msg.set_verbose(True) - else: - bb.msg.set_debug_level(0) - - if debug_domains: - bb.msg.set_debug_domains(debug_domains) - def to_boolean(string, default=None): if not string: return default
bitbake/lib/bb/msg.py domain = _NamedTuple("Domain", ( · line 130
Quoted 4 lines · Richard Purdie
bitbake/lib/bb/msg.py +1
-3-def set_debug_level(level): - for log in loggers.itervalues(): - log.setLevel(logging.NOTSET) +loggerDefaults = [0, False, []]
This line is meaningless at first glance. It's impossible to tell what these values are intended to mean without finding its usages. Have you considered using a different container than a list for this, or just using separate variables?
Signature · 5 lines
Christopher Larson clarson at kergoth dot com Founder - BitBake, OpenEmbedded, OpenZaurus Maintainer - Tslib Senior Software Engineer, Mentor Graphics
Quoted 11 lines · Chris Larson
bitbake/lib/bb/msg.py domain = _NamedTuple("Domain", ( · line 130
Quoted 4 lines · Richard Purdie
bitbake/lib/bb/msg.py +1
-3-def set_debug_level(level): - for log in loggers.itervalues(): - log.setLevel(logging.NOTSET) +loggerDefaults = [0, False, []]This line is meaningless at first glance. It's impossible to tell what these values are intended to mean without finding its usages. Have you considered using a different container than a list for this, or just using separate variables?
Yes, I'd meant to do something about that but didn't get back around to it.
Cheers,
Richard
Quoted 4 lines · Richard Purdie
At the moment it bugs me a lot that we only have one effective logging level for bitbake, despite the logging module having provision to do more advanced things. This patch:
Quoted 8 lines
* Changes the core log level to the lowest level we have messages of (DEBUG-2) so messages always flow through the core logger
* Allows build.py's task logging code to log all the output regardless of what output is on the console and sets this so log files now always contain debug level messages even if these don't appear on the console
* Moves the verbose/debug/debug-domains code to be a UI side setting
* Adds a filter to the UI to only print the user requested output.
Can we also get a way (or big enough log level) to get timestamps thrown in?
Signature · 1 line
Tom
Quoted 15 lines · Tom Rini
Quoted 4 lines · Richard Purdie
At the moment it bugs me a lot that we only have one effective logging level for bitbake, despite the logging module having provision to do more advanced things. This patch:
Quoted 8 lines
* Changes the core log level to the lowest level we have messages of (DEBUG-2) so messages always flow through the core logger
* Allows build.py's task logging code to log all the output regardless of what output is on the console and sets this so log files now always contain debug level messages even if these don't appear on the console
* Moves the verbose/debug/debug-domains code to be a UI side setting
* Adds a filter to the UI to only print the user requested output.
Quoted 1 line
Can we also get a way (or big enough log level) to get timestamps thrown in?
I think it would be nice to provide a way to influence the format string used by the formatter in the UI, either directly, or via a logging configuration file as supported by the logging module.
Signature · 5 lines
Christopher Larson clarson at kergoth dot com Founder - BitBake, OpenEmbedded, OpenZaurus Maintainer - Tslib Senior Software Engineer, Mentor Graphics
Quoted 16 lines · Chris Larson
Quoted 15 lines · Tom Rini
Quoted 12 lines · Richard Purdie
At the moment it bugs me a lot that we only have one effective logging level for bitbake, despite the logging module having provision to do more advanced things. This patch:
* Changes the core log level to the lowest level we have messages of (DEBUG-2) so messages always flow through the core logger
* Allows build.py's task logging code to log all the output regardless of what output is on the console and sets this so log files now always contain debug level messages even if these don't appear on the console
* Moves the verbose/debug/debug-domains code to be a UI side setting
* Adds a filter to the UI to only print the user requested output.
Quoted 5 lines
Quoted 1 line
Can we also get a way (or big enough log level) to get timestamps thrown in?
I think it would be nice to provide a way to influence the format string used by the formatter in the UI, either directly, or via a logging configuration file as supported by the logging module.
Agreed, but that one is for a different patch.
Cheers,
Richard
PATCH v2 1/1 bitbake/logging: Overhaul internal logging process
In reply to Richard PurdieAt the moment it bugs me a lot that we only have one effective logging level for bitbake, despite the logging module having provision to do more advanced things. This patch:
* Changes the core log level to the lowest level we have messages of (DEBUG-2) so messages always flow through the core logger
* Allows build.py's task logging code to log all the output regardless of what output is on the console and sets this so log files now always contain debug level messages even if these don't appear on the console
* Moves the verbose/debug/debug-domains code to be a UI side setting
* Adds a filter to the UI to only print the user requested output.
The result is more complete logfiles on disk but the usual output to the console.
There are some behaviour changes intentionally made by this patch:
a) the -v option now controls whether output is tee'd to the console.
Ultimately, we likely want to output a message to the user about where the log file is and avoid placing output directly onto the console for every executing task.
b) The functions get_debug_levels, the debug_levels variable, the set_debug_levels, the set_verbosity and set_debug_domains functions are removed from bb.msg.
c) The "logging" init function changes format.
d) All messages get fired to all handlers all the time leading to an increase in inter-process traffic. This could likely be hacked around short term with a function for a UI to only request events greater than level X. Longer term, having masks for event handlers would be better.
e) logger.getEffectiveLevel() is no longer a reliable guide to what will/won't get logged so for now we look at the default log levels instead.
[YOCTO #304]
---
bitbake/bin/bitbake +2 -1
@@ -190,7 +190,7 @@ Default BBFILES are the .bb files in the current directory.""") # server is daemonized this logfile will be truncated. cooker_logfile = os.path.join(os.getcwd(), "cooker.log") - bb.utils.init_logger(bb.msg, configuration.verbose, configuration.debug, + bb.msg.init_msgconfig(configuration.verbose, configuration.debug, configuration.debug_domains) # Ensure logging messages get sent to the UI as events
@@ -228,6 +228,7 @@ Default BBFILES are the .bb files in the current directory.""") try: return server.launchUI(ui_main, server_connection.connection, server_connection.events) finally: + bb.event.ui_queue = [] server_connection.terminate() return 1
bitbake/bin/bitdoc +2 -3
@@ -430,9 +430,8 @@ Create a set of html pages (documentation) for a bitbake.conf.... action = "store_true", dest = "verbose", default = False ) options, args = parser.parse_args( sys.argv ) - - if options.debug: - bb.msg.set_debug_level(options.debug) + + bb.msg.init_msgconfig(options.verbose, options.debug) return options.config, options.output
bitbake/lib/bb/__init__.py +1 -1
@@ -56,7 +56,7 @@ logging.setLoggerClass(BBLogger) logger = logging.getLogger("BitBake") logger.addHandler(NullHandler()) -logger.setLevel(logging.INFO) +logger.setLevel(logging.DEBUG - 2) # This has to be imported after the setLoggerClass, as the import of bb.msg # can result in construction of the various loggers.
bitbake/lib/bb/build.py +4 -2
@@ -223,7 +223,7 @@ def exec_func_shell(function, d, runfile, cwd=None): with open(runfile, 'w') as script: script.write('#!/bin/sh -e\n') - if logger.isEnabledFor(logging.DEBUG): + if bb.msg.loggerVerbose[1]: script.write("set -x\n") data.emit_func(function, script, d) if cwd:
@@ -234,7 +234,7 @@ def exec_func_shell(function, d, runfile, cwd=None): cmd = runfile - if logger.isEnabledFor(logging.DEBUG): + if bb.msg.loggerVerbose[1]: logfile = LogTee(logger, sys.stdout) else: logfile = sys.stdout
@@ -308,6 +308,8 @@ def _exec_task(fn, task, d, quieterr): # Ensure python logging goes to the logfile handler = logging.StreamHandler(logfile) handler.setFormatter(logformatter) + # Always enable full debug output into task logfiles + handler.setLevel(logging.DEBUG - 2) bblogger.addHandler(handler) localdata.setVar('BB_LOGFILE', logfn)
bitbake/lib/bb/msg.py +45 -29
107 lines not on this page · raw message
bitbake/lib/bb/ui/knotty.py +3 -2
@@ -74,6 +74,7 @@ def main(server, eventHandler): console = logging.StreamHandler(sys.stdout) format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") + bb.msg.addDefaultlogFilter(console) console.setFormatter(format) logger.addHandler(console)
@@ -120,8 +121,8 @@ def main(server, eventHandler): # For "normal" logging conditions, don't show note logs from tasks # but do show them if the user has changed the default log level to # include verbose/debug messages - if logger.getEffectiveLevel() > format.VERBOSE: - if event.taskpid != 0 and event.levelno <= format.NOTE: + #if logger.getEffectiveLevel() > format.VERBOSE: + if event.taskpid != 0 and event.levelno <= format.NOTE: continue logger.handle(event) continue
bitbake/lib/bb/utils.py +0 -15
@@ -830,21 +830,6 @@ def which(path, item, direction = 0): return "" -def init_logger(logger, verbose, debug, debug_domains): - """ - Set verbosity and debug levels in the logger - """ - - if debug: - bb.msg.set_debug_level(debug) - elif verbose: - bb.msg.set_verbose(True) - else: - bb.msg.set_debug_level(0) - - if debug_domains: - bb.msg.set_debug_domains(debug_domains) - def to_boolean(string, default=None): if not string: return default