Add communication mechanism for sending test results

PATCH Superseded [email protected] 2 participants 8 messages Nov 29, 2016 – Nov 30, 2016

lore

Share & tools

lore

[email protected]

author

2016-11-29 14:42 UTC #0 lore raw

cover Add communication mechanism for sending test results

From: Mariano Lopez <[email protected]>

These changes add a mechanism for sending test result to a remote repository for later consumption of the data from other clients.

These patches were tested in our local autobuilders.

The following changes since commit 12a0ee049e453b6d0d2ce2f3fa981d1b6e02bd78:

dev-manual: Added note about RPM not dealing with post-install (2016-11-23 11:10:35 +0000)

are available in the git repository at:

git://git.yoctoproject.org/poky-contrib mariano/bug9954
http://git.yoctoproject.org/cgit.cgi/poky-contrib/log/?h=mariano/bug9954

Mariano Lopez (2):
oeqa/utils/metadata.py: Add metadata library
oe-selftest: Add option to submit test result to a git repository.

 meta/lib/oeqa/utils/metadata.py | 77 ++++++++++++++++++++++++++++++++
 scripts/oe-selftest             | 98 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 175 insertions(+)
 create mode 100644 meta/lib/oeqa/utils/metadata.py

Signature · 1 line

2.7.3

[email protected]

author

2016-11-29 14:42 UTC #1 lore raw

PATCH 1/2 oeqa/utils/metadata.py: Add metadata library

From: Mariano Lopez <[email protected]>

Adds functions to get metadata from the host running the tests.

[YOCTO #9954]

Signed-off-by Mariano Lopez <[email protected]>

---
 meta/lib/oeqa/utils/metadata.py | 77 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 meta/lib/oeqa/utils/metadata.py

meta/lib/oeqa/utils/metadata.py new file +77 -0

new file mode 100644
index 0000000..3be805c
@@ -0,0 +1,77 @@
+# Copyright (C) 2016 Intel Corporation
+#
+# Released under the MIT license (see COPYING.MIT)
+#
+# Functions to get metadata from the testing host used
+# for analytics of test results.
+
+from git import Repo
+from collections import OrderedDict
+from collections.abc import MutableMapping
+from xml.dom.minidom import parseString
+from xml.etree.ElementTree import Element, tostring
+
+from oe.lsb import distro_identifier
+from oeqa.utils.commands import runCmd, get_bb_var
+
+def metadata_from_bb():
+    """ Returns test's metadata as OrderedDict.
+
+        Data will be gathered using bitbake -e thanks to get_bb_var.
+    """
+
+    info_dict = OrderedDict()
+    hostname = runCmd('hostname')
+    info_dict['hostname'] = hostname.output
+    info_dict['machine'] = get_bb_var('MACHINE')
+    info_dict['distro'] = get_bb_var('DISTRO')
+    info_dict['distro_version'] = get_bb_var('DISTRO_VERSION')
+    host_distro= distro_identifier()
+    host_distro, _, host_distro_release = host_distro.partition('-')
+    info_dict['host_distro'] = host_distro
+    info_dict['host_distro_release'] = host_distro_release
+    info_dict['layers'] = get_layers(get_bb_var('BBLAYERS'))
+    return info_dict
+
+def metadata_from_data_store(d):
+    """ Returns test's metadata as OrderedDict.
+
+        Data will be collected from the provided data store.
+    """
+    # TODO: Getting metadata from the data store would
+    # be useful when running within bitbake.
+    pass
+
+def get_layers(layers):
+    """ Returns layer name, branch, and revision as OrderedDict. """
+
+    layer_dict = OrderedDict()
+    for layer in layers.split():
+        layer_name = os.path.basename(layer)
+        layer_dict[layer_name] = OrderedDict()
+        repo = Repo(layer, search_parent_directories=True)
+        revision, branch = repo.head.object.name_rev.split()
+        layer_dict[layer_name]['branch'] = branch
+        layer_dict[layer_name]['revision'] = revision
+    return layer_dict
+
+def write_metadata_file(file_path, metadata):
+    """ Writes metadata to a XML file in directory. """
+
+    xml = dict_to_XML('metadata', metadata)
+    xml_doc = parseString(tostring(xml).decode('UTF-8'))
+    with open(file_path, 'w') as f:
+        f.write(xml_doc.toprettyxml())
+
+def dict_to_XML(tag, dictionary):
+    """ Return XML element converting dicts recursively. """
+
+    elem = Element(tag)
+    for key, val in dictionary.items():
+        if isinstance(val, MutableMapping):
+            child = (dict_to_XML(key, val))
+        else:
+            child = Element(key)
+            child.text = str(val)
+        elem.append(child)
+    return elem

Signature · 1 line

2.7.3

Mariano Lopez

author

2016-11-29 22:38 UTC #3 lore raw

Re: [PATCH 1/2] oeqa/utils/metadata.py: Add metadata library

Quoted 48 lines · Benjamin Esquivel

Quoted 47 lines · On Tue, 2016-11-29 at 08:42 -0600, [email protected] wrote:

From: Mariano Lopez <[email protected]>

Adds functions to get metadata from the host running the tests.

[YOCTO #9954]

Signed-off-by Mariano Lopez <[email protected]>

---
 meta/lib/oeqa/utils/metadata.py | 77
+++++++++++++++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 meta/lib/oeqa/utils/metadata.py

meta/lib/oeqa/utils/metadata.py new file +27 -0

new file mode 100644
index 0000000..3be805c
@@ -0,0 +1,77 @@
+# Copyright (C) 2016 Intel Corporation
+#
+# Released under the MIT license (see COPYING.MIT)
+#
+# Functions to get metadata from the testing host used
+# for analytics of test results.
+
+from git import Repo
+from collections import OrderedDict
+from collections.abc import MutableMapping
+from xml.dom.minidom import parseString
+from xml.etree.ElementTree import Element, tostring
+
+from oe.lsb import distro_identifier
+from oeqa.utils.commands import runCmd, get_bb_var
+
+def metadata_from_bb():
+    """ Returns test's metadata as OrderedDict.
+
+        Data will be gathered using bitbake -e thanks to get_bb_var.
+    """
+
+    info_dict = OrderedDict()
+    hostname = runCmd('hostname')
+    info_dict['hostname'] = hostname.output
+    info_dict['machine'] = get_bb_var('MACHINE')
+    info_dict['distro'] = get_bb_var('DISTRO')

Quoted 8 lines

meta/lib/oeqa/utils/metadata.py line 33
Quoted 6 lines
meta/lib/oeqa/utils/metadata.py new file +6 -0
+    info_dict['distro_version'] = get_bb_var('DISTRO_VERSION')
+    host_distro= distro_identifier()
+    host_distro, _, host_distro_release = host_distro.partition('-')
+    info_dict['host_distro'] = host_distro
+    info_dict['host_distro_release'] = host_distro_release
+    info_dict['layers'] = get_layers(get_bb_var('BBLAYERS'))
is none of the upper statements going to throw exceptions? otherwise try/except as appropriate.

I really don't expect this code to throw an exception, these are bitbake commands running in the host.

Quoted 16 lines

meta/lib/oeqa/utils/metadata.py new file +16 -0

+    return info_dict
+
+def metadata_from_data_store(d):
+    """ Returns test's metadata as OrderedDict.
+
+        Data will be collected from the provided data store.
+    """
+    # TODO: Getting metadata from the data store would
+    # be useful when running within bitbake.
+    pass
+
+def get_layers(layers):
+    """ Returns layer name, branch, and revision as OrderedDict. """
+
+    layer_dict = OrderedDict()
+    for layer in layers.split():

Quoted 8 lines

meta/lib/oeqa/utils/metadata.py line 55
Quoted 6 lines
meta/lib/oeqa/utils/metadata.py new file +6 -0
+        layer_name = os.path.basename(layer)
+        layer_dict[layer_name] = OrderedDict()
+        repo = Repo(layer, search_parent_directories=True)
+        revision, branch = repo.head.object.name_rev.split()
+        layer_dict[layer_name]['branch'] = branch
+        layer_dict[layer_name]['revision'] = revision
same here for the try/except, did you test with usual cases of zero input and unexisting path's, etc?

If the function doesn't receive input, it won't enter in the loop, in case the directory doesn't exists bitbake would complain long before reaching this code. If the user gets creative enough to not pass a valid BBLAYER value I think s/he deserve to be bitten by the exception.

Quoted 25 lines

meta/lib/oeqa/utils/metadata.py new file +22 -0

+    return layer_dict
+
+def write_metadata_file(file_path, metadata):
+    """ Writes metadata to a XML file in directory. """
+
+    xml = dict_to_XML('metadata', metadata)
+    xml_doc = parseString(tostring(xml).decode('UTF-8'))
+    with open(file_path, 'w') as f:
+        f.write(xml_doc.toprettyxml())
+
+def dict_to_XML(tag, dictionary):
+    """ Return XML element converting dicts recursively. """
+
+    elem = Element(tag)
+    for key, val in dictionary.items():
+        if isinstance(val, MutableMapping):
+            child = (dict_to_XML(key, val))
+        else:
+            child = Element(key)
+            child.text = str(val)
+        elem.append(child)
+    return elem

Signature · 1 line

2.7.3

Mariano Lopez

author

2016-11-30 17:32 UTC #4 lore raw

Re: [PATCH 1/2] oeqa/utils/metadata.py: Add metadata library

In reply to Benjamin Esquivel

Quoted 97 lines · Benjamin Esquivel

Quoted 43 lines · On Tue, 2016-11-29 at 08:42 -0600, [email protected] wrote:

From: Mariano Lopez <[email protected]>

Adds functions to get metadata from the host running the tests.

[YOCTO #9954]

Signed-off-by Mariano Lopez <[email protected]>

---
 meta/lib/oeqa/utils/metadata.py | 77
+++++++++++++++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 meta/lib/oeqa/utils/metadata.py

meta/lib/oeqa/utils/metadata.py new file +23 -0

new file mode 100644
index 0000000..3be805c
@@ -0,0 +1,77 @@
+# Copyright (C) 2016 Intel Corporation
+#
+# Released under the MIT license (see COPYING.MIT)
+#
+# Functions to get metadata from the testing host used
+# for analytics of test results.
+
+from git import Repo
+from collections import OrderedDict
+from collections.abc import MutableMapping
+from xml.dom.minidom import parseString
+from xml.etree.ElementTree import Element, tostring
+
+from oe.lsb import distro_identifier
+from oeqa.utils.commands import runCmd, get_bb_var
+
+def metadata_from_bb():
+    """ Returns test's metadata as OrderedDict.
+
+        Data will be gathered using bitbake -e thanks to get_bb_var.
+    """
+
+    info_dict = OrderedDict()

meta/lib/oeqa/utils/metadata.py line 33

Quoted 10 lines

meta/lib/oeqa/utils/metadata.py new file +10 -0

+    hostname = runCmd('hostname')
+    info_dict['hostname'] = hostname.output
+    info_dict['machine'] = get_bb_var('MACHINE')
+    info_dict['distro'] = get_bb_var('DISTRO')
+    info_dict['distro_version'] = get_bb_var('DISTRO_VERSION')
+    host_distro= distro_identifier()
+    host_distro, _, host_distro_release = host_distro.partition('-')
+    info_dict['host_distro'] = host_distro
+    info_dict['host_distro_release'] = host_distro_release
+    info_dict['layers'] = get_layers(get_bb_var('BBLAYERS'))

is none of the upper statements going to throw exceptions? otherwise try/except as appropriate.

Quoted 12 lines

meta/lib/oeqa/utils/metadata.py new file +12 -0

+    return info_dict
+
+def metadata_from_data_store(d):
+    """ Returns test's metadata as OrderedDict.
+
+        Data will be collected from the provided data store.
+    """
+    # TODO: Getting metadata from the data store would
+    # be useful when running within bitbake.
+    pass
+
+def get_layers(layers):

meta/lib/oeqa/utils/metadata.py line 55

Quoted 10 lines

meta/lib/oeqa/utils/metadata.py new file +10 -0

+    """ Returns layer name, branch, and revision as OrderedDict. """
+
+    layer_dict = OrderedDict()
+    for layer in layers.split():
+        layer_name = os.path.basename(layer)
+        layer_dict[layer_name] = OrderedDict()
+        repo = Repo(layer, search_parent_directories=True)
+        revision, branch = repo.head.object.name_rev.split()
+        layer_dict[layer_name]['branch'] = branch
+        layer_dict[layer_name]['revision'] = revision

same here for the try/except, did you test with usual cases of zero input and unexisting path's, etc?

Quoted 17 lines

meta/lib/oeqa/utils/metadata.py new file +17 -0

+    return layer_dict
+
+def write_metadata_file(file_path, metadata):
+    """ Writes metadata to a XML file in directory. """
+
+    xml = dict_to_XML('metadata', metadata)
+    xml_doc = parseString(tostring(xml).decode('UTF-8'))
+    with open(file_path, 'w') as f:
+        f.write(xml_doc.toprettyxml())
+
+def dict_to_XML(tag, dictionary):
+    """ Return XML element converting dicts recursively. """
+
+    elem = Element(tag)
+    for key, val in dictionary.items():
+        if isinstance(val, MutableMapping):
+            child = (dict_to_XML(key, val))

Quoted 8 lines

Quoted 8 lines
meta/lib/oeqa/utils/metadata.py new file +5 -0
+        else:
+            child = Element(key)
+            child.text = str(val)
+        elem.append(child)
+    return elem
Signature · 1 line
2.7.3

I'll implement the proposded changes and will send another version

Mariano

[email protected]

author

2016-11-29 14:42 UTC #5 lore raw

PATCH 2/2 oe-selftest: Add option to submit test result to a git repository.

In reply to [email protected]

From: Mariano Lopez <[email protected]>

This new option allows to commit the result to a git repository, along with the results it will add a metadata file for information of the current selftest run, such as: hostname, machine, distro, distro version, host version, and layers.

This implementation will have a branch per different hostname, testing branch, and machine.

To use this feature use:

oe-selftest <options> --repository <repository_link>

[YOCTO #9954]

Signed-off-by Mariano Lopez <[email protected]>

---
 scripts/oe-selftest | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 98 insertions(+)

scripts/oe-selftest +98 -0

index deaa432..81dfa3d 100755
@@ -36,6 +36,7 @@ import re
 import fnmatch
 import collections
 import imp
+import git
 
 sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib')
 import scriptpath

@@ -46,6 +47,7 @@ import argparse_oe
 import oeqa.selftest
 import oeqa.utils.ftools as ftools
 from oeqa.utils.commands import runCmd, get_bb_var, get_test_layer
+from oeqa.utils.metadata import metadata_from_bb, write_metadata_file
 from oeqa.selftest.base import oeSelfTest, get_available_machines
 
 try:

@@ -106,6 +108,8 @@ def get_args_parser():
                        help='List all tags that have been set to test cases.')
     parser.add_argument('--machine', required=False, dest='machine', choices=['random', 'all'], default=None,
                         help='Run tests on different machines (random/all).')
+    parser.add_argument('--repository', required=False, dest='repository', default='', action='store',
+                        help='Submit test results to a repository')
     return parser

@@ -572,6 +576,71 @@ def main():
 
         log.info("Finished")
 
+        if args.repository:
+            # Commit tests results to repository
+            metadata = metadata_from_bb()
+            git_dir = os.path.join(os.getcwd(), 'selftest')
+            if not os.path.isdir(git_dir):
+                os.mkdir(git_dir)
+
+            log.debug('Checking for git repository in %s' % git_dir)
+            try:
+                repo = git.Repo(git_dir)
+            except git.exc.InvalidGitRepositoryError:
+                log.debug("Couldn't find git repository %s; "
+                         "cloning from %s" % (git_dir, args.repository))
+                repo = git.Repo.clone_from(args.repository, git_dir)
+
+            r_branches = repo.git.branch(r=True)
+            r_branches = set(r_branches.replace('origin/', '').split())
+            l_branches = {str(branch) for branch in repo.branches}
+            branch = '%s/%s/%s' % (metadata['hostname'],
+                                   metadata['layers']['meta']['branch'],
+                                   metadata['machine'])
+
+            if branch in r_branches:
+                log.debug('Found branch in remote repository, checking'
+                          ' out and pulling')
+                repo.git.checkout(branch)
+                repo.git.pull()
+            elif branch in l_branches:
+                log.debug('Found branch in local repository, checking out')
+                repo.git.checkout(branch)
+            else:
+                log.debug('New branch %s' % branch)
+                repo.git.checkout('master')
+                repo.git.checkout(b=branch)
+
+            cleanResultsDir(repo)
+            xml_dir = os.path.join(os.getcwd(), log_prefix)
+            copyResultFiles(xml_dir, git_dir, repo)
+            metadata_file = os.path.join(git_dir, 'metadata.xml')
+            write_metadata_file(metadata_file, metadata)
+            repo.index.add([metadata_file])
+            repo.index.write()
+
+            # Get information for commit message
+            layer_info = ''
+            for layer, values in metadata['layers'].items():
+                layer_info = '%s%-17s = %s:%s\n' % (layer_info, layer,
+                              values['branch'], values['revision'])
+            msg = 'Selftest for build %s of %s %s for machine %s on %s\n\n%s' % (
+                   log_prefix[12:], metadata['distro'], metadata['distro_version'],
+                   metadata['machine'], metadata['hostname'], layer_info)
+
+            log.debug('Commiting results to local repository')
+            repo.index.commit(msg)
+            if not repo.is_dirty():
+                if branch in r_branches:
+                    log.debug('Pushing changes to remote repository')
+                    repo.git.push()
+                else:
+                    log.debug('Pushing changes to remote repository '
+                              'creating new branch')
+                    repo.git.push('-u', 'origin', branch)
+            else:
+                log.error('Local repository is dirty, not pushing commits')
+
         if result.wasSuccessful():
             return 0
         else:

@@ -655,6 +724,35 @@ def buildResultClass(args):
 
     return StampedResult
 
+def cleanResultsDir(repo):
+    """ Remove result files from directory """
+
+    xml_files = []
+    directory = repo.working_tree_dir
+    for f in os.listdir(directory):
+        path = os.path.join(directory, f)
+        if os.path.isfile(path) and path.endswith('.xml'):
+            xml_files.append(f)
+    repo.index.remove(xml_files, working_tree=True)
+
+def copyResultFiles(src, dst, repo):
+    """ Copy result files from src to dst removing the time stamp. """
+
+    import shutil
+
+    re_time = re.compile("-[0-9]+")
+    file_list = []
+
+    for root, subdirs, files in os.walk(src):
+        tmp_dir = root.replace(src, '').lstrip('/')
+        for s in subdirs:
+            os.mkdir(os.path.join(dst, tmp_dir, s))
+        for f in files:
+            file_name = os.path.join(dst, tmp_dir, re_time.sub("", f))
+            shutil.copy2(os.path.join(root, f), file_name)
+            file_list.append(file_name)
+    repo.index.add(file_list)
+
 class TestRunner(_TestRunner):
     """Test runner class aware of exporting tests."""
     def __init__(self, *args, **kwargs):

Signature · 1 line

2.7.3

Benjamin Esquivel

2016-11-29 22:26 UTC #6 lore raw

Re: [PATCH 2/2] oe-selftest: Add option to submit test result to a git repository.

Quoted 58 lines · On Tue, 2016-11-29 at 08:42 -0600, [email protected] wrote:

From: Mariano Lopez <[email protected]>
This new option allows to commit the result to a git repository, along with the results it will add a metadata file for information of the current selftest run, such as: hostname, machine, distro, distro version, host version, and layers.
This implementation will have a branch per different hostname, testing branch, and machine.
To use this feature use:
oe-selftest <options> --repository <repository_link>
[YOCTO #9954]
Signed-off-by Mariano Lopez <[email protected]>
---
scripts/oe-selftest | 98
+++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 98 insertions(+)
scripts/oe-selftest +0 -0
index deaa432..81dfa3d 100755
@@ -36,6 +36,7 @@ import re
import fnmatch
import collections
import imp
+import git

sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib')
import scriptpath
scripts/oe-selftest +0 -0
@@ -46,6 +47,7 @@ import argparse_oe
import oeqa.selftest
import oeqa.utils.ftools as ftools
from oeqa.utils.commands import runCmd, get_bb_var, get_test_layer +from oeqa.utils.metadata import metadata_from_bb, write_metadata_file
from oeqa.selftest.base import oeSelfTest, get_available_machines
try:
scripts/oe-selftest +0 -0
@@ -106,6 +108,8 @@ def get_args_parser():
                        help='List all tags that have been set to test cases.')
     parser.add_argument('--machine', required=False, dest='machine', choices=['random', 'all'], default=None,
                         help='Run tests on different machines (random/all).')
+    parser.add_argument('--repository', required=False, dest='repository', default='', action='store',
+                        help='Submit test results to a repository') return parser
scripts/oe-selftest +0 -0
@@ -572,6 +576,71 @@ def main():

Quoted 4 lines

log.info("Finished")

+ if args.repository:

oe-selftest is already long and not so modular, is there a chance to send these actions to methods outside oe-selftest and import them? it would also make them accessible to other data flows that wish to do the same.

Quoted 26 lines

+            # Commit tests results to repository
+            metadata = metadata_from_bb()
+            git_dir = os.path.join(os.getcwd(), 'selftest') +            if not os.path.isdir(git_dir):
+                os.mkdir(git_dir)
+
+            log.debug('Checking for git repository in %s' % git_dir) +            try:
+                repo = git.Repo(git_dir)
+            except git.exc.InvalidGitRepositoryError:
+                log.debug("Couldn't find git repository %s; " +                         "cloning from %s" % (git_dir, args.repository))
+                repo = git.Repo.clone_from(args.repository, git_dir) +
+            r_branches = repo.git.branch(r=True)
+            r_branches = set(r_branches.replace('origin/', '').split())
+            l_branches = {str(branch) for branch in repo.branches} +            branch = '%s/%s/%s' % (metadata['hostname'], +                                   metadata['layers']['meta']['branc h'],
+                                   metadata['machine'])
+
+            if branch in r_branches:
+                log.debug('Found branch in remote repository,

Quoted 8 lines

checking'
+                          ' out and pulling')
+                repo.git.checkout(branch)
+                repo.git.pull()
+            elif branch in l_branches:
+                log.debug('Found branch in local repository, checking out')
+                repo.git.checkout(branch)

can you explain why the remote branches have precedence over the local branches here?

Quoted 25 lines

+ else:
+ log.debug('New branch %s' % branch) + repo.git.checkout('master') + repo.git.checkout(b=branch)
scripts/oe-selftest +13 -0
+
+            cleanResultsDir(repo)
+            xml_dir = os.path.join(os.getcwd(), log_prefix)
+            copyResultFiles(xml_dir, git_dir, repo)
+            metadata_file = os.path.join(git_dir, 'metadata.xml')
+            write_metadata_file(metadata_file, metadata)
+            repo.index.add([metadata_file])
+            repo.index.write()
+
+            # Get information for commit message
+            layer_info = ''
+            for layer, values in metadata['layers'].items():
+                layer_info = '%s%-17s = %s:%s\n' % (layer_info,
layer,
+                              values['branch'], values['revision']) +            msg = 'Selftest for build %s of %s %s for machine %s on %s\n\n%s' % (
+                   log_prefix[12:], metadata['distro'], metadata['distro_version'],
+                   metadata['machine'], metadata['hostname'], layer_info)

Quoted 8 lines

+
+            log.debug('Commiting results to local repository') +            repo.index.commit(msg)
+            if not repo.is_dirty():
+                if branch in r_branches:
+                    log.debug('Pushing changes to remote repository')
+                    repo.git.push()

what happens here if you don't have permission to push? does it throw an exception or a return code?

Quoted 5 lines

+                else:
+                    log.debug('Pushing changes to remote repository '
+                              'creating new branch')
+                    repo.git.push('-u', 'origin', branch)

same comment about the permission above

Quoted 48 lines

+            else:
+                log.error('Local repository is dirty, not pushing commits')
+
         if result.wasSuccessful():
             return 0
         else:
scripts/oe-selftest +0 -0
@@ -655,6 +724,35 @@ def buildResultClass(args):
return StampedResult
scripts/oe-selftest +13 -0
+def cleanResultsDir(repo):
+    """ Remove result files from directory """
+
+    xml_files = []
+    directory = repo.working_tree_dir
+    for f in os.listdir(directory):
+        path = os.path.join(directory, f)
+        if os.path.isfile(path) and path.endswith('.xml'):
+            xml_files.append(f)
+    repo.index.remove(xml_files, working_tree=True)
+
+def copyResultFiles(src, dst, repo):
+    """ Copy result files from src to dst removing the time stamp.
"""
scripts/oe-selftest +12 -0
+
+    import shutil
+
+    re_time = re.compile("-[0-9]+")
+    file_list = []
+
+    for root, subdirs, files in os.walk(src):
+        tmp_dir = root.replace(src, '').lstrip('/')
+        for s in subdirs:
+            os.mkdir(os.path.join(dst, tmp_dir, s))
+        for f in files:
+            file_name = os.path.join(dst, tmp_dir, re_time.sub("",
f))
+            shutil.copy2(os.path.join(root, f), file_name) +            file_list.append(file_name)
+    repo.index.add(file_list)
+
class TestRunner(_TestRunner):
     """Test runner class aware of exporting tests.""" def __init__(self, *args, **kwargs):
--
2.7.3

Mariano Lopez

author

2016-11-29 22:49 UTC #7 lore raw

Re: [PATCH 2/2] oe-selftest: Add option to submit test result to a git repository.

Quoted 60 lines · Benjamin Esquivel

Quoted 59 lines · On Tue, 2016-11-29 at 08:42 -0600, [email protected] wrote:
From: Mariano Lopez <[email protected]>
This new option allows to commit the result to a git repository, along with the results it will add a metadata file for information of the current selftest run, such as: hostname, machine, distro, distro version, host version, and layers.
This implementation will have a branch per different hostname, testing branch, and machine.
To use this feature use:
oe-selftest <options> --repository <repository_link>
[YOCTO #9954]
Signed-off-by Mariano Lopez <[email protected]>
---
 scripts/oe-selftest | 98
+++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 98 insertions(+)
scripts/oe-selftest +1 -0
index deaa432..81dfa3d 100755
@@ -36,6 +36,7 @@ import re
 import fnmatch
 import collections
 import imp
+import git
 
 sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) +
 '/lib')
import scriptpath
scripts/oe-selftest +1 -0
@@ -46,6 +47,7 @@ import argparse_oe
 import oeqa.selftest
 import oeqa.utils.ftools as ftools
 from oeqa.utils.commands import runCmd, get_bb_var, get_test_layer
+from oeqa.utils.metadata import metadata_from_bb,
write_metadata_file
from oeqa.selftest.base import oeSelfTest, get_available_machines
try:
scripts/oe-selftest +1 -0
@@ -106,6 +108,8 @@ def get_args_parser():
                        help='List all tags that have been set to
 test cases.')
     parser.add_argument('--machine', required=False, dest='machine',
 choices=['random', 'all'], default=None,
                         help='Run tests on different machines
 (random/all).')
+    parser.add_argument('--repository', required=False,
dest='repository', default='', action='store',
scripts/oe-selftest +1 -0
+                        help='Submit test results to a repository')
     return parser
@@ -572,6 +576,71 @@ def main():
 

Quoted 7 lines

scripts/oe-selftest def main(): · line 579
Quoted 3 lines
scripts/oe-selftest +1 -0
         log.info("Finished")
 
+        if args.repository:
oe-selftest is already long and not so modular, is there a chance to send these actions to methods outside oe-selftest and import them? it would also make them accessible to other data flows that wish to do the same.

It would be possible to move this code to a module, although the specific needs of every test would be different, for example the clean and add functions must be different for everyone. I would leave it here and check the needs of other test to check if this can be reused. I would do that work, because I'm already familiar with this.

Quoted 28 lines

scripts/oe-selftest +23 -0

+            # Commit tests results to repository
+            metadata = metadata_from_bb()
+            git_dir = os.path.join(os.getcwd(), 'selftest')
+            if not os.path.isdir(git_dir):
+                os.mkdir(git_dir)
+
+            log.debug('Checking for git repository in %s' % git_dir)
+            try:
+                repo = git.Repo(git_dir)
+            except git.exc.InvalidGitRepositoryError:
+                log.debug("Couldn't find git repository %s; "
+                         "cloning from %s" % (git_dir,
 args.repository))
+                repo = git.Repo.clone_from(args.repository, git_dir)
+
+            r_branches = repo.git.branch(r=True)
+            r_branches = set(r_branches.replace('origin/',
 '').split())
+            l_branches = {str(branch) for branch in repo.branches}
+            branch = '%s/%s/%s' % (metadata['hostname'],
+                                   metadata['layers']['meta']['branc
 h'],
+                                   metadata['machine'])
+
+            if branch in r_branches:
+                log.debug('Found branch in remote repository,

checking'
+ ' out and pulling')

Quoted 8 lines

Quoted 6 lines
scripts/oe-selftest +4 -0
+                repo.git.checkout(branch)
+                repo.git.pull()
+            elif branch in l_branches:
+                log.debug('Found branch in local repository,
checking out')
+ repo.git.checkout(branch)
can you explain why the remote branches have precedence over the local branches here?

Because some other system could have pushed before and our branch is not up date.

Quoted 27 lines

scripts/oe-selftest +17 -0

+            else:
+                log.debug('New branch %s' % branch)
+                repo.git.checkout('master')
+                repo.git.checkout(b=branch)
+
+            cleanResultsDir(repo)
+            xml_dir = os.path.join(os.getcwd(), log_prefix)
+            copyResultFiles(xml_dir, git_dir, repo)
+            metadata_file = os.path.join(git_dir, 'metadata.xml')
+            write_metadata_file(metadata_file, metadata)
+            repo.index.add([metadata_file])
+            repo.index.write()
+
+            # Get information for commit message
+            layer_info = ''
+            for layer, values in metadata['layers'].items():
+                layer_info = '%s%-17s = %s:%s\n' % (layer_info,

layer,

scripts/oe-selftest +2 -0

+                              values['branch'], values['revision'])
+            msg = 'Selftest for build %s of %s %s for machine %s on

%s\n\n%s' % (
+ log_prefix[12:], metadata['distro'],
metadata['distro_version'],
+ metadata['machine'], metadata['hostname'],
layer_info)

scripts/oe-selftest +2 -0

+
+            log.debug('Commiting results to local repository')

Quoted 8 lines

Quoted 6 lines
scripts/oe-selftest +4 -0
+            repo.index.commit(msg)
+            if not repo.is_dirty():
+                if branch in r_branches:
+                    log.debug('Pushing changes to remote
repository')
+ repo.git.push()
what happens here if you don't have permission to push? does it throw an exception or a return code?

It will throw an exception, that I think is good here, because your host won't be publishing the results, and for the consumer is like this test never happened. Usually you will act more on an exception than in a warning.

Quoted 6 lines

scripts/oe-selftest def main(): · line 640
Quoted 5 lines
scripts/oe-selftest +2 -0
+                else:
+                    log.debug('Pushing changes to remote repository
'
scripts/oe-selftest +2 -0
+                              'creating new branch')
+                    repo.git.push('-u', 'origin', branch)
same comment about the permission above

Same as above.

Quoted 48 lines

scripts/oe-selftest +2 -0

+            else:
+                log.error('Local repository is dirty, not pushing

commits')

scripts/oe-selftest +14 -0

+
         if result.wasSuccessful():
             return 0
         else:

@@ -655,6 +724,35 @@ def buildResultClass(args):
 
     return StampedResult
 
+def cleanResultsDir(repo):
+    """ Remove result files from directory """
+
+    xml_files = []
+    directory = repo.working_tree_dir
+    for f in os.listdir(directory):
+        path = os.path.join(directory, f)
+        if os.path.isfile(path) and path.endswith('.xml'):
+            xml_files.append(f)
+    repo.index.remove(xml_files, working_tree=True)
+
+def copyResultFiles(src, dst, repo):
+    """ Copy result files from src to dst removing the time stamp.

"""

scripts/oe-selftest +12 -0

+
+    import shutil
+
+    re_time = re.compile("-[0-9]+")
+    file_list = []
+
+    for root, subdirs, files in os.walk(src):
+        tmp_dir = root.replace(src, '').lstrip('/')
+        for s in subdirs:
+            os.mkdir(os.path.join(dst, tmp_dir, s))
+        for f in files:
+            file_name = os.path.join(dst, tmp_dir, re_time.sub("",

f))

scripts/oe-selftest +4 -0

+            shutil.copy2(os.path.join(root, f), file_name)
+            file_list.append(file_name)
+    repo.index.add(file_list)
+
 class TestRunner(_TestRunner):
     """Test runner class aware of exporting tests."""
     def __init__(self, *args, **kwargs):

Signature · 1 line

2.7.3