Enhancements to build perf reporting

index e0cb3f0db2..757e3f0cbf 100644
@@ -64,7 +64,7 @@ class GitRepo(object):
     def rev_parse(self, revision):
         """Do git rev-parse"""
         try:
-            return self.run_cmd(['rev-parse', revision])
+            return self.run_cmd(['rev-parse', '--verify', revision])
         except GitError:
             # Revision does not exist
             return None

index ced5ff2e12..fc32fbe39a 100755
@@ -370,7 +370,7 @@ def print_html_report(data, id_comp):
 
     # Chart options
     chart_opts = {'haxis': {'min': get_data_item(data[0][0], 'layers.meta.commit_count'),
-                            'max': get_data_item(data[0][0], 'layers.meta.commit_count')}
+                            'max': get_data_item(data[-1][0], 'layers.meta.commit_count')}
                  }
 
     print(html.template.render(metadata=metadata, test_data=tests, chart_opts=chart_opts))

index fc32fbe39a..34591bec16 100755
@@ -79,29 +79,50 @@ def get_test_runs(repo, tag_name, **kwargs):
     # Return field names and a sorted list of revs
     return undef_fields, sorted(revs)
 
-def list_test_revs(repo, tag_name, **kwargs):
+def list_test_revs(repo, tag_name, verbosity, **kwargs):
     """Get list of all tested revisions"""
     fields, revs = get_test_runs(repo, tag_name, **kwargs)
     ignore_fields = ['tag_number']
+    if verbosity < 2:
+        extra_fields = ['COMMITS', 'TEST RUNS']
+        ignore_fields.extend(['commit_number', 'commit'])
+    else:
+        extra_fields = ['TEST RUNS']
+
     print_fields = [i for i, f in enumerate(fields) if f not in ignore_fields]
 
     # Sort revs
-    rows = [[fields[i].upper() for i in print_fields] + ['TEST RUNS']]
-    prev = [''] * len(revs)
+    rows = [[fields[i].upper() for i in print_fields] + extra_fields]
+
+    prev = [''] * len(print_fields)
+    prev_commit = None
+    commit_cnt = 0
+    commit_field = fields.index('commit')
     for rev in revs:
         # Only use fields that we want to print
-        rev = [rev[i] for i in print_fields]
+        cols = [rev[i] for i in print_fields]
+
+
+        if cols != prev:
+            commit_cnt = 1
+            test_run_cnt = 1
+            new_row = [''] * (len(print_fields) + len(extra_fields))
 
-        if rev != prev:
-            new_row = [''] * len(print_fields) + [1]
             for i in print_fields:
-                if rev[i] != prev[i]:
+                if cols[i] != prev[i]:
                     break
-            new_row[i:-1] = rev[i:]
+            new_row[i:-len(extra_fields)] = cols[i:]
             rows.append(new_row)
         else:
-            rows[-1][-1] += 1
-        prev = rev
+            if rev[commit_field] != prev_commit:
+                commit_cnt += 1
+            test_run_cnt += 1
+
+        if verbosity < 2:
+            new_row[-2] = commit_cnt
+        new_row[-1] = test_run_cnt
+        prev = cols
+        prev_commit = rev[commit_field]
 
     print_table(rows)
 

@@ -408,7 +429,7 @@ Examine build performance test results from a Git repository"""
                         help="Verbose logging")
     parser.add_argument('--repo', '-r', required=True,
                         help="Results repository (local git clone)")
-    parser.add_argument('--list', '-l', action='store_true',
+    parser.add_argument('--list', '-l', action='count',
                         help="List available test runs")
     parser.add_argument('--html', action='store_true',
                         help="Generate report in html format")

@@ -444,7 +465,7 @@ def main(argv=None):
     repo = GitRepo(args.repo)
 
     if args.list:
-        list_test_revs(repo, args.tag_name)
+        list_test_revs(repo, args.tag_name, args.list)
         return 0
 
     # Determine hostname which to use

index 34591bec16..1a30c70e2b 100755
@@ -34,7 +34,7 @@ from build_perf import html
 
 scriptpath.add_oe_lib_path()
 
-from oeqa.utils.git import GitRepo
+from oeqa.utils.git import GitRepo, GitError
 
 
 # Setup logging

@@ -397,6 +397,43 @@ def print_html_report(data, id_comp):
     print(html.template.render(metadata=metadata, test_data=tests, chart_opts=chart_opts))
 
 
+def dump_buildstats(repo, outdir, notes_ref, revs):
+    """Dump buildstats of test results"""
+    full_ref = 'refs/notes/' + notes_ref
+    if not repo.rev_parse(full_ref):
+        log.error("No buildstats found, please try running "
+                  "'git fetch origin %s:%s' to fetch them from the remote",
+                  full_ref, full_ref)
+        return
+
+    missing = False
+    log.info("Writing out buildstats from 'refs/notes/%s' into '%s'",
+              notes_ref, outdir)
+    for rev in revs:
+        log.debug('Dumping buildstats for %s (%s)', rev.commit_number,
+                  rev.commit)
+        for tag in rev.tags:
+            log.debug('    %s', tag)
+            try:
+                bs_all = json.loads(repo.run_cmd(['notes', '--ref', notes_ref,
+                                                  'show', tag + '^0']))
+            except GitError:
+                log.warning("Buildstats not found for %s", tag)
+                missing = True
+            for measurement, buildstats in bs_all.items():
+                tag_base, run_id = tag.rsplit('/', 1)
+                tag_base = tag_base.replace('/', '_')
+                bs_dir = os.path.join(outdir, measurement, tag_base)
+                if not os.path.exists(bs_dir):
+                    os.makedirs(bs_dir)
+                with open(os.path.join(bs_dir, run_id + '.json'), 'w') as f:
+                    json.dump(buildstats, f, indent=2)
+    if missing:
+        log.info("Buildstats were missing for some test runs, please "
+                 "run 'git fetch origin %s:%s' and try again",
+                 full_ref, full_ref)
+
+
 def auto_args(repo, args):
     """Guess arguments, if not defined by the user"""
     # Get the latest commit in the repo

@@ -452,6 +489,8 @@ Examine build performance test results from a Git repository"""
     group.add_argument('--commit-number2',
                        help="Revision number to compare with, redundant if "
                             "--commit2 is specified")
+    parser.add_argument('--dump-buildstats', nargs='?', const='.',
+                        help="Dump buildstats of the tests")
 
     return parser.parse_args(argv)
 

@@ -546,6 +585,14 @@ def main(argv=None):
     else:
         print_html_report(data, index_l)
 
+    # Dump buildstats
+    if args.dump_buildstats:
+        notes_ref = 'buildstats/{}/{}/{}'.format(args.hostname, args.branch,
+                                                 args.machine)
+        dump_buildstats(repo, 'oe-build-perf-buildstats', notes_ref,
+                        [rev_l, rev_r])
+                        #revs_l.tags + revs_r.tags)
+
     return 0
 
 if __name__ == "__main__":

index adeba44988..4276464714 100755
@@ -52,8 +52,12 @@ class BSTask(dict):
     @property
     def cputime(self):
         """Sum of user and system time taken by the task"""
-        return self['rusage']['ru_stime'] + self['rusage']['ru_utime'] + \
-               self['child_rusage']['ru_stime'] + self['child_rusage']['ru_utime']
+        rusage = self['rusage']['ru_stime'] + self['rusage']['ru_utime']
+        if self['child_rusage']:
+            # Child rusage may have been optimized out
+            return rusage + self['child_rusage']['ru_stime'] + self['child_rusage']['ru_utime']
+        else:
+            return rusage
 
     @property
     def walltime(self):

@@ -73,12 +77,20 @@ class BSTask(dict):
     @property
     def read_ops(self):
         """Number of read operations on the block layer"""
-        return self['rusage']['ru_inblock'] + self['child_rusage']['ru_inblock']
+        if self['child_rusage']:
+            # Child rusage may have been optimized out
+            return self['rusage']['ru_inblock'] + self['child_rusage']['ru_inblock']
+        else:
+            return self['rusage']['ru_inblock']
 
     @property
     def write_ops(self):
         """Number of write operations on the block layer"""
-        return self['rusage']['ru_oublock'] + self['child_rusage']['ru_oublock']
+        if self['child_rusage']:
+            # Child rusage may have been optimized out
+            return self['rusage']['ru_oublock'] + self['child_rusage']['ru_oublock']
+        else:
+            return self['rusage']['ru_oublock']
 
 
 def read_buildstats_file(buildstat_file):