Add communication mechanism for sending test results

From: Mariano Lopez <[email protected]>

Adds functions to get metadata from the host running the tests.

[YOCTO #9954]

Signed-off-by Mariano Lopez <[email protected]>

---
 meta/lib/oeqa/utils/metadata.py | 77
+++++++++++++++++++++++++++++++++++++++++

 1 file changed, 77 insertions(+)

 create mode 100644 meta/lib/oeqa/utils/metadata.py

meta/lib/oeqa/utils/metadata.py new file +23 -0

new file mode 100644
index 0000000..3be805c
@@ -0,0 +1,77 @@
+# Copyright (C) 2016 Intel Corporation
+#
+# Released under the MIT license (see COPYING.MIT)
+#
+# Functions to get metadata from the testing host used
+# for analytics of test results.
+
+from git import Repo
+from collections import OrderedDict
+from collections.abc import MutableMapping
+from xml.dom.minidom import parseString
+from xml.etree.ElementTree import Element, tostring
+
+from oe.lsb import distro_identifier
+from oeqa.utils.commands import runCmd, get_bb_var
+
+def metadata_from_bb():
+    """ Returns test's metadata as OrderedDict.
+
+        Data will be gathered using bitbake -e thanks to get_bb_var.
+    """
+
+    info_dict = OrderedDict()

meta/lib/oeqa/utils/metadata.py new file +12 -0

+    return info_dict
+
+def metadata_from_data_store(d):
+    """ Returns test's metadata as OrderedDict.
+
+        Data will be collected from the provided data store.
+    """
+    # TODO: Getting metadata from the data store would
+    # be useful when running within bitbake.
+    pass
+
+def get_layers(layers):

meta/lib/oeqa/utils/metadata.py new file +22 -0

+    return layer_dict
+
+def write_metadata_file(file_path, metadata):
+    """ Writes metadata to a XML file in directory. """
+
+    xml = dict_to_XML('metadata', metadata)
+    xml_doc = parseString(tostring(xml).decode('UTF-8'))
+    with open(file_path, 'w') as f:
+        f.write(xml_doc.toprettyxml())
+
+def dict_to_XML(tag, dictionary):
+    """ Return XML element converting dicts recursively. """
+
+    elem = Element(tag)
+    for key, val in dictionary.items():
+        if isinstance(val, MutableMapping):
+            child = (dict_to_XML(key, val))
+        else:
+            child = Element(key)
+            child.text = str(val)
+        elem.append(child)
+    return elem

--
2.7.3

From: Mariano Lopez <[email protected]>

This new option allows to commit the result to a git repository, along with the results it will add a metadata file for information of the current selftest run, such as: hostname, machine, distro, distro version, host version, and layers.

This implementation will have a branch per different hostname, testing branch, and machine.

To use this feature use:

oe-selftest <options> --repository <repository_link>

[YOCTO #9954]

Signed-off-by Mariano Lopez <[email protected]>

---
 scripts/oe-selftest | 98
+++++++++++++++++++++++++++++++++++++++++++++++++++++

 1 file changed, 98 insertions(+)

scripts/oe-selftest +0 -0

index deaa432..81dfa3d 100755
@@ -36,6 +36,7 @@ import re

 import fnmatch
 import collections
 import imp
+import git

 sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib')
 import scriptpath

scripts/oe-selftest +0 -0

@@ -46,6 +47,7 @@ import argparse_oe

 import oeqa.selftest
 import oeqa.utils.ftools as ftools
 from oeqa.utils.commands import runCmd, get_bb_var, get_test_layer +from oeqa.utils.metadata import metadata_from_bb, write_metadata_file
 from oeqa.selftest.base import oeSelfTest, get_available_machines
 try:

scripts/oe-selftest +0 -0

@@ -106,6 +108,8 @@ def get_args_parser():

                        help='List all tags that have been set to test cases.')
     parser.add_argument('--machine', required=False, dest='machine', choices=['random', 'all'], default=None,
                         help='Run tests on different machines (random/all).')
+    parser.add_argument('--repository', required=False, dest='repository', default='', action='store',
+                        help='Submit test results to a repository') return parser

scripts/oe-selftest +0 -0

@@ -572,6 +576,71 @@ def main():

         log.info("Finished")

+        if args.repository:

+            # Commit tests results to repository
+            metadata = metadata_from_bb()
+            git_dir = os.path.join(os.getcwd(), 'selftest') +            if not os.path.isdir(git_dir):
+                os.mkdir(git_dir)
+
+            log.debug('Checking for git repository in %s' % git_dir) +            try:
+                repo = git.Repo(git_dir)
+            except git.exc.InvalidGitRepositoryError:
+                log.debug("Couldn't find git repository %s; " +                         "cloning from %s" % (git_dir, args.repository))
+                repo = git.Repo.clone_from(args.repository, git_dir) +
+            r_branches = repo.git.branch(r=True)
+            r_branches = set(r_branches.replace('origin/', '').split())
+            l_branches = {str(branch) for branch in repo.branches} +            branch = '%s/%s/%s' % (metadata['hostname'], +                                   metadata['layers']['meta']['branc h'],
+                                   metadata['machine'])
+
+            if branch in r_branches:
+                log.debug('Found branch in remote repository,

checking'
+                          ' out and pulling')
+                repo.git.checkout(branch)
+                repo.git.pull()
+            elif branch in l_branches:
+                log.debug('Found branch in local repository, checking out')
+                repo.git.checkout(branch)

+            else:
+                log.debug('New branch %s' % branch) +                repo.git.checkout('master') +                repo.git.checkout(b=branch)

scripts/oe-selftest +13 -0

+
+            cleanResultsDir(repo)
+            xml_dir = os.path.join(os.getcwd(), log_prefix)
+            copyResultFiles(xml_dir, git_dir, repo)
+            metadata_file = os.path.join(git_dir, 'metadata.xml')
+            write_metadata_file(metadata_file, metadata)
+            repo.index.add([metadata_file])
+            repo.index.write()
+
+            # Get information for commit message
+            layer_info = ''
+            for layer, values in metadata['layers'].items():
+                layer_info = '%s%-17s = %s:%s\n' % (layer_info,

layer,
+                              values['branch'], values['revision']) +            msg = 'Selftest for build %s of %s %s for machine %s on %s\n\n%s' % (
+                   log_prefix[12:], metadata['distro'], metadata['distro_version'],
+                   metadata['machine'], metadata['hostname'], layer_info)

+
+            log.debug('Commiting results to local repository') +            repo.index.commit(msg)
+            if not repo.is_dirty():
+                if branch in r_branches:
+                    log.debug('Pushing changes to remote repository')
+                    repo.git.push()

+                else:
+                    log.debug('Pushing changes to remote repository '
+                              'creating new branch')
+                    repo.git.push('-u', 'origin', branch)

+            else:
+                log.error('Local repository is dirty, not pushing commits')
+
         if result.wasSuccessful():
             return 0
         else:

scripts/oe-selftest +0 -0

@@ -655,6 +724,35 @@ def buildResultClass(args):

     return StampedResult

scripts/oe-selftest +13 -0

+def cleanResultsDir(repo):
+    """ Remove result files from directory """
+
+    xml_files = []
+    directory = repo.working_tree_dir
+    for f in os.listdir(directory):
+        path = os.path.join(directory, f)
+        if os.path.isfile(path) and path.endswith('.xml'):
+            xml_files.append(f)
+    repo.index.remove(xml_files, working_tree=True)
+
+def copyResultFiles(src, dst, repo):
+    """ Copy result files from src to dst removing the time stamp.

"""

scripts/oe-selftest +12 -0

+
+    import shutil
+
+    re_time = re.compile("-[0-9]+")
+    file_list = []
+
+    for root, subdirs, files in os.walk(src):
+        tmp_dir = root.replace(src, '').lstrip('/')
+        for s in subdirs:
+            os.mkdir(os.path.join(dst, tmp_dir, s))
+        for f in files:
+            file_name = os.path.join(dst, tmp_dir, re_time.sub("",

f))
+            shutil.copy2(os.path.join(root, f), file_name) +            file_list.append(file_name)
+    repo.index.add(file_list)
+
 class TestRunner(_TestRunner):
     """Test runner class aware of exporting tests.""" def __init__(self, *args, **kwargs):
--
2.7.3

Quoted 47 lines · On Tue, 2016-11-29 at 08:42 -0600, [email protected] wrote:

From: Mariano Lopez <[email protected]>

Adds functions to get metadata from the host running the tests.

[YOCTO #9954]

Signed-off-by Mariano Lopez <[email protected]>

---
 meta/lib/oeqa/utils/metadata.py | 77
+++++++++++++++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 meta/lib/oeqa/utils/metadata.py

meta/lib/oeqa/utils/metadata.py new file +27 -0

new file mode 100644
index 0000000..3be805c
@@ -0,0 +1,77 @@
+# Copyright (C) 2016 Intel Corporation
+#
+# Released under the MIT license (see COPYING.MIT)
+#
+# Functions to get metadata from the testing host used
+# for analytics of test results.
+
+from git import Repo
+from collections import OrderedDict
+from collections.abc import MutableMapping
+from xml.dom.minidom import parseString
+from xml.etree.ElementTree import Element, tostring
+
+from oe.lsb import distro_identifier
+from oeqa.utils.commands import runCmd, get_bb_var
+
+def metadata_from_bb():
+    """ Returns test's metadata as OrderedDict.
+
+        Data will be gathered using bitbake -e thanks to get_bb_var.
+    """
+
+    info_dict = OrderedDict()
+    hostname = runCmd('hostname')
+    info_dict['hostname'] = hostname.output
+    info_dict['machine'] = get_bb_var('MACHINE')
+    info_dict['distro'] = get_bb_var('DISTRO')

meta/lib/oeqa/utils/metadata.py line 33

Quoted 6 lines

meta/lib/oeqa/utils/metadata.py new file +6 -0

+    info_dict['distro_version'] = get_bb_var('DISTRO_VERSION')
+    host_distro= distro_identifier()
+    host_distro, _, host_distro_release = host_distro.partition('-')
+    info_dict['host_distro'] = host_distro
+    info_dict['host_distro_release'] = host_distro_release
+    info_dict['layers'] = get_layers(get_bb_var('BBLAYERS'))

is none of the upper statements going to throw exceptions? otherwise try/except as appropriate.

Quoted 16 lines

meta/lib/oeqa/utils/metadata.py new file +16 -0

+    return info_dict
+
+def metadata_from_data_store(d):
+    """ Returns test's metadata as OrderedDict.
+
+        Data will be collected from the provided data store.
+    """
+    # TODO: Getting metadata from the data store would
+    # be useful when running within bitbake.
+    pass
+
+def get_layers(layers):
+    """ Returns layer name, branch, and revision as OrderedDict. """
+
+    layer_dict = OrderedDict()
+    for layer in layers.split():

meta/lib/oeqa/utils/metadata.py line 55

Quoted 6 lines

meta/lib/oeqa/utils/metadata.py new file +6 -0

+        layer_name = os.path.basename(layer)
+        layer_dict[layer_name] = OrderedDict()
+        repo = Repo(layer, search_parent_directories=True)
+        revision, branch = repo.head.object.name_rev.split()
+        layer_dict[layer_name]['branch'] = branch
+        layer_dict[layer_name]['revision'] = revision

same here for the try/except, did you test with usual cases of zero input and unexisting path's, etc?

Quoted 25 lines

meta/lib/oeqa/utils/metadata.py new file +22 -0

+    return layer_dict
+
+def write_metadata_file(file_path, metadata):
+    """ Writes metadata to a XML file in directory. """
+
+    xml = dict_to_XML('metadata', metadata)
+    xml_doc = parseString(tostring(xml).decode('UTF-8'))
+    with open(file_path, 'w') as f:
+        f.write(xml_doc.toprettyxml())
+
+def dict_to_XML(tag, dictionary):
+    """ Return XML element converting dicts recursively. """
+
+    elem = Element(tag)
+    for key, val in dictionary.items():
+        if isinstance(val, MutableMapping):
+            child = (dict_to_XML(key, val))
+        else:
+            child = Element(key)
+            child.text = str(val)
+        elem.append(child)
+    return elem

Signature · 1 line

2.7.3

Quoted 59 lines · On Tue, 2016-11-29 at 08:42 -0600, [email protected] wrote:

From: Mariano Lopez <[email protected]>

This new option allows to commit the result to a git repository, along with the results it will add a metadata file for information of the current selftest run, such as: hostname, machine, distro, distro version, host version, and layers.

This implementation will have a branch per different hostname, testing branch, and machine.

To use this feature use:

oe-selftest <options> --repository <repository_link>

[YOCTO #9954]

Signed-off-by Mariano Lopez <[email protected]>

---
 scripts/oe-selftest | 98
+++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 98 insertions(+)

scripts/oe-selftest +1 -0

index deaa432..81dfa3d 100755
@@ -36,6 +36,7 @@ import re
 import fnmatch
 import collections
 import imp
+import git
 
 sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) +
 '/lib')

import scriptpath

scripts/oe-selftest +1 -0

@@ -46,6 +47,7 @@ import argparse_oe
 import oeqa.selftest
 import oeqa.utils.ftools as ftools
 from oeqa.utils.commands import runCmd, get_bb_var, get_test_layer
+from oeqa.utils.metadata import metadata_from_bb,

write_metadata_file
from oeqa.selftest.base import oeSelfTest, get_available_machines

try:

scripts/oe-selftest +1 -0

@@ -106,6 +108,8 @@ def get_args_parser():
                        help='List all tags that have been set to
 test cases.')
     parser.add_argument('--machine', required=False, dest='machine',
 choices=['random', 'all'], default=None,
                         help='Run tests on different machines
 (random/all).')
+    parser.add_argument('--repository', required=False,

dest='repository', default='', action='store',

scripts/oe-selftest +1 -0

+                        help='Submit test results to a repository')
     return parser

@@ -572,6 +576,71 @@ def main():
 

scripts/oe-selftest def main(): · line 579

Quoted 3 lines

scripts/oe-selftest +1 -0

         log.info("Finished")
 
+        if args.repository:

oe-selftest is already long and not so modular, is there a chance to send these actions to methods outside oe-selftest and import them? it would also make them accessible to other data flows that wish to do the same.

Quoted 28 lines

scripts/oe-selftest +23 -0

+            # Commit tests results to repository
+            metadata = metadata_from_bb()
+            git_dir = os.path.join(os.getcwd(), 'selftest')
+            if not os.path.isdir(git_dir):
+                os.mkdir(git_dir)
+
+            log.debug('Checking for git repository in %s' % git_dir)
+            try:
+                repo = git.Repo(git_dir)
+            except git.exc.InvalidGitRepositoryError:
+                log.debug("Couldn't find git repository %s; "
+                         "cloning from %s" % (git_dir,
 args.repository))
+                repo = git.Repo.clone_from(args.repository, git_dir)
+
+            r_branches = repo.git.branch(r=True)
+            r_branches = set(r_branches.replace('origin/',
 '').split())
+            l_branches = {str(branch) for branch in repo.branches}
+            branch = '%s/%s/%s' % (metadata['hostname'],
+                                   metadata['layers']['meta']['branc
 h'],
+                                   metadata['machine'])
+
+            if branch in r_branches:
+                log.debug('Found branch in remote repository,

checking'
+ ' out and pulling')

Quoted 6 lines

scripts/oe-selftest +4 -0

+                repo.git.checkout(branch)
+                repo.git.pull()
+            elif branch in l_branches:
+                log.debug('Found branch in local repository,

checking out')
+ repo.git.checkout(branch)

can you explain why the remote branches have precedence over the local branches here?

Quoted 27 lines

scripts/oe-selftest +17 -0

+            else:
+                log.debug('New branch %s' % branch)
+                repo.git.checkout('master')
+                repo.git.checkout(b=branch)
+
+            cleanResultsDir(repo)
+            xml_dir = os.path.join(os.getcwd(), log_prefix)
+            copyResultFiles(xml_dir, git_dir, repo)
+            metadata_file = os.path.join(git_dir, 'metadata.xml')
+            write_metadata_file(metadata_file, metadata)
+            repo.index.add([metadata_file])
+            repo.index.write()
+
+            # Get information for commit message
+            layer_info = ''
+            for layer, values in metadata['layers'].items():
+                layer_info = '%s%-17s = %s:%s\n' % (layer_info,

layer,

scripts/oe-selftest +2 -0

+                              values['branch'], values['revision'])
+            msg = 'Selftest for build %s of %s %s for machine %s on

%s\n\n%s' % (
+ log_prefix[12:], metadata['distro'],
metadata['distro_version'],
+ metadata['machine'], metadata['hostname'],
layer_info)

scripts/oe-selftest +2 -0

+
+            log.debug('Commiting results to local repository')

Quoted 6 lines

scripts/oe-selftest +4 -0

+            repo.index.commit(msg)
+            if not repo.is_dirty():
+                if branch in r_branches:
+                    log.debug('Pushing changes to remote

repository')
+ repo.git.push()

what happens here if you don't have permission to push? does it throw an exception or a return code?

scripts/oe-selftest def main(): · line 640

Quoted 5 lines

scripts/oe-selftest +2 -0

+                else:
+                    log.debug('Pushing changes to remote repository

'

scripts/oe-selftest +2 -0

+                              'creating new branch')
+                    repo.git.push('-u', 'origin', branch)

same comment about the permission above

Quoted 48 lines

scripts/oe-selftest +2 -0

+            else:
+                log.error('Local repository is dirty, not pushing

commits')

scripts/oe-selftest +14 -0

+
         if result.wasSuccessful():
             return 0
         else:

@@ -655,6 +724,35 @@ def buildResultClass(args):
 
     return StampedResult
 
+def cleanResultsDir(repo):
+    """ Remove result files from directory """
+
+    xml_files = []
+    directory = repo.working_tree_dir
+    for f in os.listdir(directory):
+        path = os.path.join(directory, f)
+        if os.path.isfile(path) and path.endswith('.xml'):
+            xml_files.append(f)
+    repo.index.remove(xml_files, working_tree=True)
+
+def copyResultFiles(src, dst, repo):
+    """ Copy result files from src to dst removing the time stamp.

"""

scripts/oe-selftest +12 -0

+
+    import shutil
+
+    re_time = re.compile("-[0-9]+")
+    file_list = []
+
+    for root, subdirs, files in os.walk(src):
+        tmp_dir = root.replace(src, '').lstrip('/')
+        for s in subdirs:
+            os.mkdir(os.path.join(dst, tmp_dir, s))
+        for f in files:
+            file_name = os.path.join(dst, tmp_dir, re_time.sub("",

f))

scripts/oe-selftest +4 -0

+            shutil.copy2(os.path.join(root, f), file_name)
+            file_list.append(file_name)
+    repo.index.add(file_list)
+
 class TestRunner(_TestRunner):
     """Test runner class aware of exporting tests."""
     def __init__(self, *args, **kwargs):

Signature · 1 line

2.7.3

Quoted 43 lines · On Tue, 2016-11-29 at 08:42 -0600, [email protected] wrote:

From: Mariano Lopez <[email protected]>

Adds functions to get metadata from the host running the tests.

[YOCTO #9954]

Signed-off-by Mariano Lopez <[email protected]>

---
 meta/lib/oeqa/utils/metadata.py | 77
+++++++++++++++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 meta/lib/oeqa/utils/metadata.py

meta/lib/oeqa/utils/metadata.py new file +23 -0

new file mode 100644
index 0000000..3be805c
@@ -0,0 +1,77 @@
+# Copyright (C) 2016 Intel Corporation
+#
+# Released under the MIT license (see COPYING.MIT)
+#
+# Functions to get metadata from the testing host used
+# for analytics of test results.
+
+from git import Repo
+from collections import OrderedDict
+from collections.abc import MutableMapping
+from xml.dom.minidom import parseString
+from xml.etree.ElementTree import Element, tostring
+
+from oe.lsb import distro_identifier
+from oeqa.utils.commands import runCmd, get_bb_var
+
+def metadata_from_bb():
+    """ Returns test's metadata as OrderedDict.
+
+        Data will be gathered using bitbake -e thanks to get_bb_var.
+    """
+
+    info_dict = OrderedDict()

meta/lib/oeqa/utils/metadata.py line 33

Quoted 10 lines

meta/lib/oeqa/utils/metadata.py new file +10 -0

+    hostname = runCmd('hostname')
+    info_dict['hostname'] = hostname.output
+    info_dict['machine'] = get_bb_var('MACHINE')
+    info_dict['distro'] = get_bb_var('DISTRO')
+    info_dict['distro_version'] = get_bb_var('DISTRO_VERSION')
+    host_distro= distro_identifier()
+    host_distro, _, host_distro_release = host_distro.partition('-')
+    info_dict['host_distro'] = host_distro
+    info_dict['host_distro_release'] = host_distro_release
+    info_dict['layers'] = get_layers(get_bb_var('BBLAYERS'))

is none of the upper statements going to throw exceptions? otherwise try/except as appropriate.

Quoted 12 lines

meta/lib/oeqa/utils/metadata.py new file +12 -0

+    return info_dict
+
+def metadata_from_data_store(d):
+    """ Returns test's metadata as OrderedDict.
+
+        Data will be collected from the provided data store.
+    """
+    # TODO: Getting metadata from the data store would
+    # be useful when running within bitbake.
+    pass
+
+def get_layers(layers):

meta/lib/oeqa/utils/metadata.py line 55

Quoted 10 lines

meta/lib/oeqa/utils/metadata.py new file +10 -0

+    """ Returns layer name, branch, and revision as OrderedDict. """
+
+    layer_dict = OrderedDict()
+    for layer in layers.split():
+        layer_name = os.path.basename(layer)
+        layer_dict[layer_name] = OrderedDict()
+        repo = Repo(layer, search_parent_directories=True)
+        revision, branch = repo.head.object.name_rev.split()
+        layer_dict[layer_name]['branch'] = branch
+        layer_dict[layer_name]['revision'] = revision

same here for the try/except, did you test with usual cases of zero input and unexisting path's, etc?

Quoted 17 lines

meta/lib/oeqa/utils/metadata.py new file +17 -0

+    return layer_dict
+
+def write_metadata_file(file_path, metadata):
+    """ Writes metadata to a XML file in directory. """
+
+    xml = dict_to_XML('metadata', metadata)
+    xml_doc = parseString(tostring(xml).decode('UTF-8'))
+    with open(file_path, 'w') as f:
+        f.write(xml_doc.toprettyxml())
+
+def dict_to_XML(tag, dictionary):
+    """ Return XML element converting dicts recursively. """
+
+    elem = Element(tag)
+    for key, val in dictionary.items():
+        if isinstance(val, MutableMapping):
+            child = (dict_to_XML(key, val))

Quoted 8 lines

meta/lib/oeqa/utils/metadata.py new file +5 -0

+        else:
+            child = Element(key)
+            child.text = str(val)
+        elem.append(child)
+    return elem

Signature · 1 line

2.7.3