Skip to contentSkip to conversation

More Bitbake logging improvements

PATCH New Joshua Watt 1 participant 5 messages Mar 11, 2020

lore
Share & tools
lore
#0 lore raw

cover More Bitbake logging improvements

Supplemental patches to improve logging in bitbake by using python structured logging. Fixes up some ResourceWarnings and improves the flexibility of the console and consolelog.

Joshua Watt (4):
bitbake: knotty: Add logging cleanup
bitbake: msg: Add helper to merge logging configs
bitbake: knotty: Update hash equivalence logging
bitbake: runqueue: Lower setscene complete logging level

 bitbake/contrib/autobuilderlog.json |  18 +----
 bitbake/lib/bb/msg.py               | 100 ++++++++++++++++------------
 bitbake/lib/bb/runqueue.py          |   2 +-
 bitbake/lib/bb/ui/knotty.py         |  70 ++++++++++++++++---
 4 files changed, 124 insertions(+), 66 deletions(-)
Signature · 1 line
2.17.1
#1 lore raw

PATCH 1/4 bitbake: knotty: Add logging cleanup

Adds code to close all loggers when bitbake exits. This prevents unclosed file ResourceWarnings. A form of this closing existed previously, but was removed in the new logging code.

---
 bitbake/lib/bb/msg.py       | 11 +++++++++++
 bitbake/lib/bb/ui/knotty.py |  2 ++
 2 files changed, 13 insertions(+)
bitbake/lib/bb/msg.py +11 -0
index ced605d96d..ee3c38b5a4 100644
@@ -284,3 +284,14 @@ def setLoggingConfig(defaultconfig, userconfigfile=None):
         # TODO: I don't think that setting the global log level should be necessary
         #if newlevel < bb.msg.loggerDefaultLogLevel:
         #    bb.msg.loggerDefaultLogLevel = newlevel
+
+def cleanupLogging():
+    # Iterate through all the handlers and close them if possible. Fixes
+    # 'Unclosed resource' warnings when bitbake exits, see
+    # https://bugs.python.org/issue23010
+    handlers = set()
+    for logger_iter in logging.Logger.manager.loggerDict.keys():
+        handlers.update(logging.getLogger(logger_iter).handlers)
+
+    for h in handlers:
+        h.close()
bitbake/lib/bb/ui/knotty.py +2 -0
index b229687ef8..2dfce90a4f 100644
@@ -823,4 +823,6 @@ def main(server, eventHandler, params, tf = TerminalFilter):
         if e.errno == errno.EPIPE:
             pass
 
+    bb.msg.cleanupLogging()
+
     return return_value
Signature · 1 line
2.17.1
#2 lore raw

PATCH 2/4 bitbake: msg: Add helper to merge logging configs

In reply to Joshua Watt

Adds a function that can be used by UI front ends to merge logging configuration fragments.

---
 bitbake/lib/bb/msg.py | 89 +++++++++++++++++++++++--------------------
 1 file changed, 48 insertions(+), 41 deletions(-)
bitbake/lib/bb/msg.py +48 -41
index ee3c38b5a4..c22d0d3e15 100644
@@ -193,6 +193,53 @@ def has_console_handler(logger):
                 return True
     return False
 
+def mergeLoggingConfig(logconfig, userconfig):
+    logconfig = copy.deepcopy(logconfig)
+    userconfig = copy.deepcopy(userconfig)
+
+    # Merge config with the default config
+    if userconfig.get('version') != logconfig['version']:
+        raise BaseException("Bad user configuration version. Expected %r, got %r" % (logconfig['version'], userconfig.get('version')))
+
+    # Set some defaults to make merging easier
+    userconfig.setdefault("loggers", {})
+
+    # If a handler, formatter, or filter is defined in the user
+    # config, it will replace an existing one in the default config
+    for k in ("handlers", "formatters", "filters"):
+        logconfig.setdefault(k, {}).update(userconfig.get(k, {}))
+
+    seen_loggers = set()
+    for name, l in logconfig["loggers"].items():
+        # If the merge option is set, merge the handlers and
+        # filters. Otherwise, if it is False, this logger won't get
+        # add to the set of seen loggers and will replace the
+        # existing one
+        if l.get('bitbake_merge', True):
+            ulogger = userconfig["loggers"].setdefault(name, {})
+            ulogger.setdefault("handlers", [])
+            ulogger.setdefault("filters", [])
+
+            # Merge lists
+            l.setdefault("handlers", []).extend(ulogger["handlers"])
+            l.setdefault("filters", []).extend(ulogger["filters"])
+
+            # Replace other properties if present
+            if "level" in ulogger:
+                l["level"] = ulogger["level"]
+
+            if "propagate" in ulogger:
+                l["propagate"] = ulogger["propagate"]
+
+            seen_loggers.add(name)
+
+    # Add all loggers present in the user config, but not any that
+    # have already been processed
+    for name in set(userconfig["loggers"].keys()) - seen_loggers:
+        logconfig["loggers"][name] = userconfig["loggers"][name]
+
+    return logconfig
+
 def setLoggingConfig(defaultconfig, userconfigfile=None):
     logconfig = copy.deepcopy(defaultconfig)
 
@@ -208,47 +255,7 @@ def setLoggingConfig(defaultconfig, userconfigfile=None):
                 raise BaseException("Unrecognized file format: %s" % userconfigfile)
 
             if userconfig.get('bitbake_merge', True):
-                # Merge config with the default config
-                if userconfig.get('version') != logconfig['version']:
-                    raise BaseException("Bad user configuration version. Expected %r, got %r" % (logconfig['version'], userconfig.get('version')))
-
-                # Set some defaults to make merging easier
-                userconfig.setdefault("loggers", {})
-
-                # If a handler, formatter, or filter is defined in the user
-                # config, it will replace an existing one in the default config
-                for k in ("handlers", "formatters", "filters"):
-                    logconfig.setdefault(k, {}).update(userconfig.get(k, {}))
-
-                seen_loggers = set()
-                for name, l in logconfig["loggers"].items():
-                    # If the merge option is set, merge the handlers and
-                    # filters. Otherwise, if it is False, this logger won't get
-                    # add to the set of seen loggers and will replace the
-                    # existing one
-                    if l.get('bitbake_merge', True):
-                        ulogger = userconfig["loggers"].setdefault(name, {})
-                        ulogger.setdefault("handlers", [])
-                        ulogger.setdefault("filters", [])
-
-                        # Merge lists
-                        l.setdefault("handlers", []).extend(ulogger["handlers"])
-                        l.setdefault("filters", []).extend(ulogger["filters"])
-
-                        # Replace other properties if present
-                        if "level" in ulogger:
-                            l["level"] = ulogger["level"]
-
-                        if "propagate" in ulogger:
-                            l["propagate"] = ulogger["propagate"]
-
-                        seen_loggers.add(name)
-
-                # Add all loggers present in the user config, but not any that
-                # have already been processed
-                for name in set(userconfig["loggers"].keys()) - seen_loggers:
-                    logconfig["loggers"][name] = userconfig["loggers"][name]
-
+                logconfig = mergeLoggingConfig(logconfig, userconfig)
             else:
                 # Replace the entire default config
                 logconfig = userconfig
Signature · 1 line
2.17.1
#3 lore raw

PATCH 3/4 bitbake: knotty: Update hash equivalence logging

In reply to Joshua Watt

Updates hash equivalence logging so that the interesting VERBOSE messages are always logged to the consolelog file so that issues in individual user builds can be diagnosed. The autobuilder logging config then updates this so that they also are shown on stdout, since the consolelog file is not capture there.

In order to facilitate this, 2 new logging handlers were added, "BitBake.verbconsole" and "BitBake.verbconsolelog". Neither of these handlers are attached to anything by default, but they will log any messages that wouldn't otherwise be logged by the normal console or consolelog handlers. Users can attach whatever loggers the desire to this handler to get them to appear on the console or in the consolelog, as demonstrated by the autobuilderlog.json file.

---
 bitbake/contrib/autobuilderlog.json | 18 +-------
 bitbake/lib/bb/ui/knotty.py         | 68 +++++++++++++++++++++++++----
 2 files changed, 62 insertions(+), 24 deletions(-)
bitbake/contrib/autobuilderlog.json +2 -16
index 103a1141f6..193a675a1f 100644
@@ -1,27 +1,13 @@
 {
     "version": 1,
-    "handlers": {
-        "autobuilderlog": {
-            "class": "logging.FileHandler",
-            "formatter": "logfileFormatter",
-            "level": "DEBUG",
-            "filename": "autobuilder.log",
-            "mode": "w"
-        }
-    },
-    "formatters": {
-            "logfileFormatter": {
-                "format": "%(name)s: %(levelname)s: %(message)s"
-            }
-    },
     "loggers": {
         "BitBake.SigGen.HashEquiv": {
             "level": "VERBOSE",
-            "handlers": ["autobuilderlog"]
+            "handlers": ["BitBake.verbconsole"]
         },
         "BitBake.RunQueue.HashEquiv": {
             "level": "VERBOSE",
-            "handlers": ["autobuilderlog"]
+            "handlers": ["BitBake.verbconsole"]
         }
     }
 }
bitbake/lib/bb/ui/knotty.py +60 -8

86 lines not on this page · raw message

Signature · 1 line
2.17.1
#4 lore raw

PATCH 4/4 bitbake: runqueue: Lower setscene complete logging level

In reply to Joshua Watt

Lowers the level of the log message when setscene tasks have completed. This message can occur multiple times when hash equivalence is enabled, since the runqueue switches between executing setscene tasks and normal tasks. Since this is primarily of use when debugging hash equivalence, use the hash equivalence logger at VERBOSE level.

[YOCTO #13813]

---
 bitbake/lib/bb/runqueue.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
bitbake/lib/bb/runqueue.py +1 -1
index cef9b0fbbf..16f076f3b1 100644
@@ -2059,7 +2059,7 @@ class RunQueueExecute:
         self.update_holdofftasks()
 
         if not self.sq_live and not self.sqdone and not self.sq_deferred and not self.updated_taskhash_queue and not self.holdoff_tasks:
-            logger.info("Setscene tasks completed")
+            hashequiv_logger.verbose("Setscene tasks completed")
 
             err = self.summarise_scenequeue_errors()
             if err:
Signature · 1 line
2.17.1