Skip to contentSkip to conversation

scriptutils: Add support for specify stream on logger_create

PATCH New Aníbal Limón 1 participant 7 messages Mar 20, 2017

lore
Share & tools
lore
#0 lore raw

PATCH 1/7 scriptutils: Add support for specify stream on logger_create

It is a good idea to let the script to choose what stream wants to dump the logging output.

[YOCTO #11160]

---
 scripts/lib/scriptutils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
scripts/lib/scriptutils.py +2 -2
index 4233783..4ccbe5c 100644
@@ -22,9 +22,9 @@ import glob
 import argparse
 import subprocess
 
-def logger_create(name):
+def logger_create(name, stream=None):
     logger = logging.getLogger(name)
-    loggerhandler = logging.StreamHandler()
+    loggerhandler = logging.StreamHandler(stream=stream)
     loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
     logger.addHandler(loggerhandler)
     logger.setLevel(logging.INFO)
Signature · 1 line
2.1.4
#1 lore raw

PATCH 2/7 scripts/yocto-compat-layer.py: Dump log to stdout instead of stderr

The common unix tools uses stdout as standard for log output, by default python logging uses stderr if not stream is specified.

[YOCTO #11160]

---
 scripts/yocto-compat-layer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
scripts/yocto-compat-layer.py +1 -1
index 09dc5bf..8996fff 100755
@@ -31,7 +31,7 @@ DEFAULT_OUTPUT_LOG = '%s-%s.log' % (PROGNAME,
 OUTPUT_LOG_LINK = "%s.log" % PROGNAME
 CASES_PATHS = [os.path.join(os.path.abspath(os.path.dirname(__file__)),
                 'lib', 'compatlayer', 'cases')]
-logger = scriptutils.logger_create(PROGNAME)
+logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout)
 
 def test_layer_compatibility(td, layer):
     from compatlayer.context import CompatLayerTestContext
Signature · 1 line
2.1.4
#2 lore raw

PATCH 3/7 scripts/yocto-compat-layer.py: Make output log argument optional

In reply to Aníbal Limón

Only create a log file when --output-log option is specified, since logger is dumping to stdout by default is better to let the user decide if a log needs to be created.

[YOCTO #11160]

---
 scripts/yocto-compat-layer.py | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)
scripts/yocto-compat-layer.py +5 -12
index 8996fff..b96f3ca 100755
@@ -26,9 +26,6 @@ from compatlayer import LayerType, detect_layers, add_layer, get_signatures
 from oeqa.utils.commands import get_bb_vars
 
 PROGNAME = 'yocto-compat-layer'
-DEFAULT_OUTPUT_LOG = '%s-%s.log' % (PROGNAME,
-        time.strftime("%Y%m%d%H%M%S"))
-OUTPUT_LOG_LINK = "%s.log" % PROGNAME
 CASES_PATHS = [os.path.join(os.path.abspath(os.path.dirname(__file__)),
                 'lib', 'compatlayer', 'cases')]
 logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout)
@@ -49,9 +46,7 @@ def main():
     parser.add_argument('layers', metavar='LAYER_DIR', nargs='+',
             help='Layer to test compatibility with Yocto Project')
     parser.add_argument('-o', '--output-log',
-            help='Output log default: %s' % DEFAULT_OUTPUT_LOG,
-            action='store', default=DEFAULT_OUTPUT_LOG)
-
+            help='File to output log (optional)', action='store')
     parser.add_argument('-d', '--debug', help='Enable debug output',
             action='store_true')
     parser.add_argument('-q', '--quiet', help='Print only errors',
@@ -63,16 +58,14 @@ def main():
 
     args = parser.parse_args()
 
-    fh = logging.FileHandler(args.output_log)
-    fh.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
-    logger.addHandler(fh)
+    if args.output_log:
+        fh = logging.FileHandler(args.output_log)
+        fh.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
+        logger.addHandler(fh)
     if args.debug:
         logger.setLevel(logging.DEBUG)
     elif args.quiet:
         logger.setLevel(logging.ERROR)
-    if os.path.exists(OUTPUT_LOG_LINK):
-        os.unlink(OUTPUT_LOG_LINK)
-    os.symlink(args.output_log, OUTPUT_LOG_LINK)
 
     if not 'BUILDDIR' in os.environ:
         logger.error("You must source the environment before run this script.")
Signature · 1 line
2.1.4
#3 lore raw

PATCH 4/7 scripts/compatlayer: Add exclude of meta-world-pkgdata on get_signatures

In reply to Aníbal Limón

The meta-world-pkgdata recipe can be modified when a layer is added may be can add recipes to world target, so exclude by default.

[YOCTO #11162]

---
 scripts/lib/compatlayer/__init__.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
scripts/lib/compatlayer/__init__.py +14 -0
index a7eb862..888d303 100644
@@ -133,6 +133,11 @@ def get_signatures(builddir, failsafe=False):
     import subprocess
     import re
 
+    # some recipes needs to be excluded like meta-world-pkgdata
+    # because a layer can add recipes to a world build so signature
+    # will be change
+    exclude_recipes = ('meta-world-pkgdata',)
+
     sigs = {}
 
     cmd = 'bitbake '
@@ -153,6 +158,15 @@ def get_signatures(builddir, failsafe=False):
             line = line.strip()
             s = sig_regex.match(line)
             if s:
+                exclude = False
+                for er in exclude_recipes:
+                    (recipe, task) = s.group('task').split(':')
+                    if er == recipe:
+                        exclude = True
+                        break
+                if exclude:
+                    continue
+
                 sigs[s.group('task')] = s.group('hash')
 
     if not sigs:
Signature · 1 line
2.1.4
#4 lore raw

PATCH 5/7 scripts/lib/compatlayer: Remove require of meta- in layer dir name

In reply to Aníbal Limón

The layers isn't required to have a dirctory name start with meta- so remove the validation.

---
 scripts/lib/compatlayer/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
scripts/lib/compatlayer/__init__.py +1 -1
index 888d303..15dc95d 100644
@@ -118,7 +118,7 @@ def detect_layers(layer_directories):
         for root, dirs, files in os.walk(directory):
             dir_name = os.path.basename(root)
             conf_dir = os.path.join(root, 'conf')
-            if dir_name.startswith('meta-') and os.path.isdir(conf_dir):
+            if os.path.isdir(conf_dir):
                 layer = _detect_layer(root)
                 if layer:
                     layers.append(layer)
Signature · 1 line
2.1.4
#5 lore raw

PATCH 6/7 scripts/yocto-compat-layer.py: Add option to disable layer autodiscovery

In reply to Aníbal Limón

Sometimes there is a need to only analyze the layer specified by the command line, the new option -n will disable autodiscovery of layers and only will try to test specified layers.

---
 scripts/lib/compatlayer/__init__.py | 17 ++++++++++++-----
 scripts/yocto-compat-layer.py       |  4 +++-
 2 files changed, 15 insertions(+), 6 deletions(-)
scripts/lib/compatlayer/__init__.py +12 -5
index 15dc95d..b8ce771 100644
@@ -108,20 +108,27 @@ def _detect_layer(layer_path):
 
     return layer
 
-def detect_layers(layer_directories):
+def detect_layers(layer_directories, no_auto):
     layers = []
 
     for directory in layer_directories:
         if directory[-1] == '/':
             directory = directory[0:-1]
 
-        for root, dirs, files in os.walk(directory):
-            dir_name = os.path.basename(root)
-            conf_dir = os.path.join(root, 'conf')
+        if no_auto:
+            conf_dir = os.path.join(directory, 'conf')
             if os.path.isdir(conf_dir):
-                layer = _detect_layer(root)
+                layer = _detect_layer(directory)
                 if layer:
                     layers.append(layer)
+        else:
+            for root, dirs, files in os.walk(directory):
+                dir_name = os.path.basename(root)
+                conf_dir = os.path.join(root, 'conf')
+                if os.path.isdir(conf_dir):
+                    layer = _detect_layer(root)
+                    if layer:
+                        layers.append(layer)
 
     return layers
 
scripts/yocto-compat-layer.py +3 -1
index b96f3ca..b4de84a 100755
@@ -47,6 +47,8 @@ def main():
             help='Layer to test compatibility with Yocto Project')
     parser.add_argument('-o', '--output-log',
             help='File to output log (optional)', action='store')
+    parser.add_argument('-n', '--no-auto', help='Disable auto layer discovery',
+            action='store_true')
     parser.add_argument('-d', '--debug', help='Enable debug output',
             action='store_true')
     parser.add_argument('-q', '--quiet', help='Print only errors',
@@ -74,7 +76,7 @@ def main():
     builddir = os.environ['BUILDDIR']
     bblayersconf = os.path.join(builddir, 'conf', 'bblayers.conf')
 
-    layers = detect_layers(args.layers)
+    layers = detect_layers(args.layers, args.no_auto)
     if not layers:
         logger.error("Fail to detect layers")
         return 1
Signature · 1 line
2.1.4
#6 lore raw

PATCH 7/7 scripts/yocto-compat-layer.py: Handle layer dependencies when test

In reply to Aníbal Limón

If some layer depends on other tries to find layer dependency, if the layer dependency isn't found avoid to test the layer and notice the user.

---
 scripts/lib/compatlayer/__init__.py | 29 ++++++++++++++++++++++++++++-
 scripts/yocto-compat-layer.py       | 21 +++++++++++++--------
 2 files changed, 41 insertions(+), 9 deletions(-)
scripts/lib/compatlayer/__init__.py +28 -1
index b8ce771..435679e 100644
@@ -132,10 +132,37 @@ def detect_layers(layer_directories, no_auto):
 
     return layers
 
-def add_layer(bblayersconf, layer):
+def _find_layer_depends(depend, layers):
+    for layer in layers:
+        for collection in layer['collections']:
+            if depend == collection:
+                return layer
+    return None
+
+def add_layer(bblayersconf, layer, layers, logger):
+    logger.info('Adding layer %s' % layer['name'])
+
+    for collection in layer['collections']:
+        for depend in layer['collections'][collection]['depends'].split():
+            # core (oe-core) is suppose to be provided
+            if depend == 'core':
+                continue
+
+            layer_depend = _find_layer_depends(depend, layers)
+            if not layer_depend:
+                logger.error('Layer %s depends on %s and isn\'t found.' % \
+                        (layer['name'], depend))
+                return False
+
+            logger.info('Adding layer dependency %s' % layer_depend['name'])
+            with open(bblayersconf, 'a+') as f:
+                f.write("\nBBLAYERS += \"%s\"\n" % layer_depend['path'])
+
     with open(bblayersconf, 'a+') as f:
         f.write("\nBBLAYERS += \"%s\"\n" % layer['path'])
 
+    return True
+
 def get_signatures(builddir, failsafe=False):
     import subprocess
     import re
scripts/yocto-compat-layer.py +13 -8
index b4de84a..9e74033 100755
@@ -116,6 +116,7 @@ def main():
     td['sigs'] = get_signatures(td['builddir'])
     logger.info('')
 
+    layers_tested = 0
     for layer in layers:
         if layer['type'] == LayerType.ERROR_NO_LAYER_CONF or \
                 layer['type'] == LayerType.ERROR_BSP_DISTRO:
@@ -123,16 +124,20 @@ def main():
 
         shutil.copyfile(bblayersconf + '.backup', bblayersconf)
 
-        add_layer(bblayersconf, layer)
+        if not add_layer(bblayersconf, layer, layers, logger):
+            continue
+
         result = test_layer_compatibility(td, layer)
         results[layer['name']] = result
-
-    logger.info('')
-    logger.info('Summary of results:')
-    logger.info('')
-    for layer_name in results:
-        logger.info('%s ... %s' % (layer_name, 'PASS' if \
-                results[layer_name].wasSuccessful() else 'FAIL'))
+        layers_tested = layers_tested + 1
+
+    if layers_tested:
+        logger.info('')
+        logger.info('Summary of results:')
+        logger.info('')
+        for layer_name in results:
+            logger.info('%s ... %s' % (layer_name, 'PASS' if \
+                    results[layer_name].wasSuccessful() else 'FAIL'))
 
     cleanup_bblayers(None, None)
 
Signature · 1 line
2.1.4