Running benchmarks via OSSTest

index 37eade2..e509064 100755
@@ -78,7 +78,8 @@ d-i preseed/late_command string \\
         in-target mkdir -p /boot/efi/EFI/boot; \\
         in-target cp /boot/efi/EFI/debian/grubx64.efi /boot/efi/EFI/boot/bootx64.efi ;\\
         in-target mkdir -p /root/.ssh; \\
-        in-target sh -c "echo -e '$authkeys'> /root/.ssh/authorized_keys";
+        in-target sh -c "echo -e '$authkeys'> /root/.ssh/authorized_keys";\\
+        in-target sed -i "/^deb cdrom:/s/^/#/" /etc/apt/sources.list
 END
     return $preseed_file;
 }

index c8db601..70afaec 100644
@@ -293,8 +293,8 @@ sub setupboot_grub2 ($$$) {
                 my (@missing) =
                     grep { !defined $entry->{$_} } 
 		        (defined $xenhopt
-			 ? qw(Title Hv KernDom0 KernVer)
-			 : qw(Title Hv KernOnly KernVer));
+                        ? qw(Title Hv KernDom0 KernVer)
+                        : qw(Title KernOnly KernVer));
 		if (@missing) {
 		    logm("(skipping entry at $entry->{StartLine};".
 			 " no @missing)");

@@ -321,11 +321,14 @@ sub setupboot_grub2 ($$$) {
                 die unless $entry;
                 $entry->{Hv}= $1;
             }
-            if (m/^\s*multiboot\s*\/(vmlinu[xz]-(\S+))/) {
+            if (m/^\s*linux\s*\/(vmlinu[xz]-(\S+))/) {
                 die unless $entry;
                 $entry->{KernOnly}= $1;
                 $entry->{KernVer}= $2;
             }
+            if (m/^\s*initrd\s*\/(initrd\S+)/) {
+                $entry->{Initrd}= $1;
+            }
             if (m/^\s*module\s*\/(vmlinu[xz]-(\S+))/) {
                 die unless $entry;
                 $entry->{KernDom0}= $1;

index a3b6936..cdff8d5 100644
@@ -1460,11 +1460,12 @@ sub prepareguest_part_xencfg ($$$$$) {
     my ($ho, $gho, $ram_mb, $xopts, $cfgrest) = @_;
     my $onreboot= $xopts->{OnReboot} || 'restart';
     my $vcpus= guest_var($gho, 'vcpus', $xopts->{DefVcpus} || 2);
+    my $memory= guest_var($gho, 'memory', $xopts->{DefMem} || $ram_mb);
     my $xoptcfg= $xopts->{ExtraConfig};
     $xoptcfg='' unless defined $xoptcfg;
     my $xencfg= <<END;
 name        = '$gho->{Name}'
-memory = ${ram_mb}
+memory = ${memory}
 vif         = [ 'type=ioemu,mac=$gho->{Ether}' ]
 #
 on_poweroff = 'destroy'

index f001418..f85b06d 100755
@@ -113,10 +113,13 @@ sub setcfg ($$) {
 
 sub otherfixupcfg () {
     my $vcpus= guest_var($gho,'vcpus',1);
+    my $ram_mb= guest_var($gho,'memory',512);
     $cfg =~ s/^dhcp/#$&/mg;
     $cfg =~ s/^on_crash.*/on_crash='preserve'/mg;
     $cfg =~ s/^vcpus.*//mg;
     $cfg .= "\nvcpus = $vcpus\n";
+    $cfg =~ s/^memory.*//mg;
+    $cfg .= "\nmemory = $ram_mb\n";
 
     # PCI passthrough
     # Look for runvars   <gn>_pcipassthrough_<devtype>=<hostident>

index cdff8d5..67befd0 100644
@@ -53,7 +53,8 @@ BEGIN {
                       target_getfile target_getfile_root
                       target_putfile target_putfile_root
                       target_putfilecontents_stash
-		      target_putfilecontents_root_stash
+                      target_putfilecontents_root_stash
+                      target_getfile_stash target_getfile_root_stash
                       target_put_guest_image target_editfile
                       target_editfile_root target_file_exists
                       target_run_apt

@@ -473,6 +474,20 @@ sub target_putfilecontents_root_stash ($$$$;$) {
     tpfcs_core(\&target_putfile_root, @_);
 }
 
+sub tgfs_core {
+    my ($tgetfilef,$ho,$timeout,$rsrc,$lleaf) = @_;
+    target_somefile_getleaf(\$lleaf,$rsrc,$ho);
+    $tgetfilef->($ho, $timeout, $rsrc, "$stash/$lleaf");
+}
+sub target_getfile_stash ($$$;$) {
+    my ($ho,$timeout,$rsrc,$lleaf) = @_;
+    tgfs_core(\&target_getfile, @_);
+}
+sub target_getfile_root_stash ($$$;$) {
+    my ($ho,$timeout,$rsrc,$lleaf) = @_;
+    tgfs_core(\&target_getfile_root, @_);
+}
+
 sub target_file_exists ($$) {
     my ($ho,$rfile) = @_;
     my $out= target_cmd_output($ho, "if test -e $rfile; then echo y; fi");

new file mode 100755
index 0000000..85184c9
@@ -0,0 +1,40 @@
+#!/bin/bash
+# This is part of "osstest", an automated testing framework for Xen.
+# Copyright (C) 2009-2014 Citrix Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -e
+
+if [ -f standalone.config ] ; then
+    . standalone.config
+fi
+
+. cri-getconfig
+
+fail () { echo >&2 "$0: $1"; exit 1; }
+
+# By default we try to grab Unixbench 5.1.3
+site=${UNIXBENCH_REPO:-http://byte-unixbench.googlecode.com/files}
+rfile=${UNIXBENCH_FILE:-unixbench-5.1.3.tgz}
+lfile=unixbench.tgz
+
+images=`getconfig Images`;
+dstdir="${images}/benchs"
+mkdir -p $dstdir
+
+wget ${site}/${rfile} -O ${dstdir}/${lfile} || \
+    fail "failed downloading the benchmark"
+
+echo >&2 "downloaded $dstdir/$lfile"

new file mode 100755
index 0000000..4852609
@@ -0,0 +1,77 @@
+#!/usr/bin/perl -w
+# This is part of "osstest", an automated testing framework for Xen.
+# Copyright (C) 2009-2014 Citrix Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use strict qw(vars);
+use DBI;
+use Osstest;
+use Osstest::TestSupport;
+use File::Basename;
+
+tsreadconfig();
+
+# what we expect as argument list is:
+#  host=<somehost> [<someguest>]
+our ($whhost,$gn) = @ARGV;
+$whhost ||= 'host';
+our $ho= selecthost($whhost);
+
+our $gho= $ho;
+if (defined $gn and $gn ne "") {
+  $gho= selectguest($gn, $ho);
+  my $err= guest_check_ip($gho);
+  die "$err $gho->{Name}" if defined $err;
+}
+
+# By default, we expect to find UnixBench 5.1.3, stored in
+# $c{Images}/benchs/unixbench.tgz. To use something different, define
+# r{'unixbench_file'}.
+our $unixbench_file= (defined($r{'unixbench_file'})) ?
+      $r{'unixbench_file'} : "$c{Images}/benchs/unixbench.tgz";
+
+# Prepare the target, by installing dependencies, and building the benchmark
+sub install_deps() {
+  target_install_packages_norec($gho, qw(build-essential libx11-dev
+                                         libgl1-mesa-dev libxext-dev
+                                         x11-apps));
+}
+
+# Ship the benchmark to the target machine
+sub ship() {
+  logm("Shipping and extracting $unixbench_file");
+  target_putfile_root($gho, 60, "$unixbench_file", "/root");
+
+  our $unixbench_filename= basename $unixbench_file;
+  target_cmd_root($gho, <<END, 200);
+      set -ex
+      rm -rf /root/unixbench/
+      mkdir /root/unixbench
+      tar xaf /root/$unixbench_filename -C /root/unixbench --strip-components=1
+END
+}
+
+sub build() {
+  target_cmd_root($gho, <<END, 200);
+      set -ex
+      cd /root/unixbench
+      sed -e "s/^# GRAPHIC_TESTS =/GRAPHIC_TESTS =/" -i Makefile
+      make
+END
+}
+
+install_deps();
+ship();
+build();

index 67befd0..7cc5be6 100644
@@ -57,6 +57,7 @@ BEGIN {
                       target_getfile_stash target_getfile_root_stash
                       target_put_guest_image target_editfile
                       target_editfile_root target_file_exists
+                      target_catfile_stash target_catfile_root_stash
                       target_run_apt
                       target_install_packages target_install_packages_norec
                       target_jobdir target_extract_jobdistpath_subdir

@@ -488,6 +489,28 @@ sub target_getfile_root_stash ($$$;$) {
     tgfs_core(\&target_getfile_root, @_);
 }
 
+sub tcfs_core {
+  my ($tcmdoutf,$ho,$rfile)= @_;
+  my $lleaf;
+
+  target_somefile_getleaf(\$lleaf,$rfile,$ho);
+
+  my $h= new IO::File "$stash/$lleaf", 'w' or die $!;
+  my $fdata= $tcmdoutf->($ho, "cat $rfile");
+  print $h $fdata or die $!;
+  close $h or die $!;
+}
+
+sub target_catfile_stash ($$) {
+  my ($ho,$rfile)= @_;
+  tcfs_core(\&target_cmd_output,@_);
+}
+
+sub target_catfile_root_stash ($$) {
+  my ($ho,$rfile)= @_;
+  tcfs_core(\&target_cmd_output_root,@_);
+}
+
 sub target_file_exists ($$) {
     my ($ho,$rfile) = @_;
     my $out= target_cmd_output($ho, "if test -e $rfile; then echo y; fi");

67 lines not on this page · raw message

new file mode 100755
index 0000000..6e5a9a8
@@ -0,0 +1,67 @@
+#!/usr/bin/perl -w
+# This is part of "osstest", an automated testing framework for Xen.
+# Copyright (C) 2009-2014 Citrix Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use strict qw(vars);
+use Osstest;
+use DBI;
+use IO::File;
+use POSIX;
+use Osstest::TestSupport;
+
+tsreadconfig();
+
+# what we expect as argument list is:
+#  host=<somehost> [<someguest>]
+our ($whhost,$gn) = @ARGV;
+$whhost ||= 'host';
+our $ho= selecthost($whhost);
+
+our $gho= $ho;
+if (defined $gn and $gn ne "") {
+  $gho= selectguest($gn, $ho);
+  my $err= guest_check_ip($gho);
+  die "$err $gho->{Name}" if defined $err;
+}
+
+our $run_id= (defined($r{'unixbench_run_id'})) ?
+      $r{'unixbench_run_id'} : '01';
+
+our $lresfile= $r{'unixbench_params'};
+$lresfile =~ tr/ //ds;
+$lresfile .= (defined($r{'unixbench_run_suffix'})) ?
+      $r{'unixbench_run_suffix'} : '';
+$lresfile = "unixbench$lresfile";
+
+# Unixbench stores results in a subdirectory called 'results'. The file name
+# is made up out of the box's hostname, today's date and a progressive id
+# (e.g., results/benny-2014-07-02-01).
+sub fetch() {
+  my $rresults_dir= "/root/unixbench/results";
+  my $gho_hostname= target_cmd_output_root($gho, 'hostname');
+  my $resultspat= "$rresults_dir/$gho_hostname*-$run_id";
+
+  my $rresfile= target_cmd_output_root($gho, <<END);
+    chmod a+r $resultspat 2>&1 ||:
+    echo $resultspat
+END
+
+  logm("Fetching $rresfile from $gho_hostname");
+  target_getfile_root_stash($gho, 60, "$rresfile",
+      "$lresfile");
+}
+
+fetch();

new file mode 100644
index 0000000..0c5c538
@@ -0,0 +1,115 @@
+# This is part of "osstest", an automated testing framework for Xen.
+# Copyright (C) 2009-2013 Citrix Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+package Osstest::Benchmarking;
+
+use strict;
+use warnings;
+
+use IO::File;
+use IPC::Cmd qw[can_run run];
+
+use Osstest;
+use Osstest::TestSupport;
+
+BEGIN {
+    use Exporter ();
+    our ($VERSION, @ISA, @EXPORT, @EXPORT_OK, %EXPORT_TAGS);
+    $VERSION     = 1.00;
+    @ISA         = qw(Exporter);
+    @EXPORT      = qw(unixbench_process_results
+                      unixbench_print_results
+                      unixbench_plot_results
+                      );
+    %EXPORT_TAGS = ( );
+
+    @EXPORT_OK   = qw();
+}
+
+#---------- manipulation of benchmarks results ----------
+
+sub unixbench_process_results ($$) {
+  my ($results_ref,$rfilen)= @_;
+  my $h= new IO::File "< $rfilen" or die "$!";
+
+  my $par;
+  while (<$h>) {
+    my ($bench,$val,$idx);
+    if (m/.*running ([0-9]*) parallel.*$/) {
+      $par= $1;
+    }
+    if (m/^(\S[a-zA-z0-9-\(\)\s]*)\s([0-9]+.[0-9]?)\s*([0-9]+.[0-9]?)\s*([0-9]+.[0-9]?)$/) {
+      $val= $3;
+      $idx= $4;
+      ($bench = $1) =~ s/\s+$//;
+      $$results_ref->{"$bench"}{Result}{"$par"}= $val;
+      $$results_ref->{"$bench"}{Index}{"$par"}= $idx;
+    }
+    next;
+  }
+  close($h);
+}
+
+sub unixbench_print_results ($$) {
+  my ($results,$rfilen)= @_;
+  open my $h, "|-", "tee $rfilen" or die "$!";
+
+  printf $h "%-50s","\"BENCHMARK NAME\"";
+  foreach my $i (sort keys $results->{'Double-Precision Whetstone'}{Index}) {
+    printf $h "%-15s","\"$i VCPUs\"";
+  }
+  print $h "\n";
+  foreach my $b (keys $results) {
+    printf $h "%-50s","\"$b\"";
+    foreach my $i (sort keys $results->{"$b"}{Index}) {
+      printf $h "%-15s",$results->{$b}{Index}{$i};
+    }
+    print $h "\n";
+  }
+  close($h);
+}
+
+sub unixbench_plot_results ($$$) {
+  my ($dataf,$num_cols,$pfile)= @_;
+  my $h= new IO::File "> $pfile.gp" or die "$!";
+
+  printf $h <<EOF;
+set terminal png enhanced font "/usr/share/fonts/truetype/ttf-dejavu/DejaVuSans.ttf" 8 size 800,600
+set output '$pfile.png'
+set title 'Unixbench INDEXes for $flight.$job'
+set key outside center top horizontal noreverse noenhanced autotitles nobox
+set xtics mirror rotate by -45 out
+set style data histogram
+set style histogram cluster gap 1
+set style fill solid border lt -1
+set boxwidth 1 absolute
+set bmargin 13
+set rmargin 14
+SKIP_COL=1
+NCOL=$num_cols
+HWIDTH=1.0/(NCOL+1.0)
+plot for [c=SKIP_COL+1:SKIP_COL+NCOL] '$dataf' using c:xtic(1) with histograms title columnhead, \\
+        for [c=SKIP_COL+1:SKIP_COL+NCOL]'' every ::1 using 0:c:c with labels notitle offset first -HWIDTH*(NCOL/2.0)+HWIDTH/2.0+(c-(SKIP_COL+1))*HWIDTH, character 2 rotate by 90
+EOF
+  close($h);
+
+  my $gp= can_run('gnuplot') or return;
+  my ($ok,$err)= run( command => "$gp $pfile.gp", verbose => 1 );
+  logm("WARNING: plotting file with \"$err\"") unless $ok;
+}
+
+1;

index 6e5a9a8..b480d15 100755
@@ -21,6 +21,7 @@ use DBI;
 use IO::File;
 use POSIX;
 use Osstest::TestSupport;
+use Osstest::Benchmarking;
 
 tsreadconfig();
 

@@ -46,6 +47,8 @@ $lresfile .= (defined($r{'unixbench_run_suffix'})) ?
       $r{'unixbench_run_suffix'} : '';
 $lresfile = "unixbench$lresfile";
 
+our $results;
+
 # Unixbench stores results in a subdirectory called 'results'. The file name
 # is made up out of the box's hostname, today's date and a progressive id
 # (e.g., results/benny-2014-07-02-01).

@@ -64,4 +67,18 @@ END
       "$lresfile");
 }
 
+sub process () {
+  my $resf= "$stash/$gho->{Name}--$lresfile";
+  my $dataf= "$resf-DATA";
+  my $plotf= "$resf-PLOT";
+
+  unixbench_process_results(\$results,$resf);
+  unixbench_print_results($results,$dataf);
+
+  # For plotting we need to know the number of data columns
+  my $ncols= keys $results->{'Double-Precision Whetstone'}{Index};
+  unixbench_plot_results($dataf,$ncols,$plotf);
+}
+
 fetch();
+process();

index 2cf810a..81c2040 100755
@@ -321,6 +321,37 @@ proc run-job/test-rumpuserxen {} {
     run-ts . =   ts-guest-destroy-hard          + host + $g
 }
 
+#-------- benchmarks --------
+
+proc bench-unixbench-run {args} {
+    run-ts . = ts-unixbench-build  + host $args
+    run-ts . = ts-unixbench-run    + host $args
+    run-ts . = ts-unixbench-reslts + host $args
+}
+
+proc bench-unixbench-host {} {
+    bench-unixbench-run
+}
+
+proc bench-unixbench-guest {g} {
+    bench-unixbench-run $g
+    run-ts . = ts-guest-stop + host $g
+}
+
+proc need-hosts/bench-unixbench-pv {} { return host }
+proc run-job/bench-unixbench-pv {} {
+    run-ts . = ts-debian-install + host
+    run-ts . = ts-debian-fixup   + host debian
+    run-ts . = ts-guest-start    + host debian
+    bench-unixbench-guest debian
+}
+
+proc need-hosts/bench-unixbench-hvm {} { return host }
+proc run-job/bench-unixbench-hvm {} {
+    run-ts . = ts-debian-hvm-install
+    bench-unixbench-guest debianhvm
+}
+
 #---------- builds ----------
 
 proc need-hosts/build {} { return BUILD }

index 3fe5ecc..45d1498 100644
@@ -190,6 +190,16 @@ test-$XENARCH-$DOM0ARCH-<CASE>
         Some tests also have a -$DOMUARCH suffix indicating the
         obvious thing.
 
+bench-$BENCHNAME-$ARCH-<XEN_OPTS>-<GUEST_OPTS>
+
+        A benchmarking job, running benchmark $BENCHNAME on a $ARCH
+        hypervisor and dom0 (and guest).
+
+        In the remainder of the job's name, <XEN_OPTS> tells something
+        about Xen's configuration (e.g., what scheduler will be used);
+        <GUEST_OPTS> tells about the guest's configuration (e.g., whether
+        it's HVM or PV, number of vCPUs, RAM, etc.).
+
 NB: $ARCH (and $XENARCH etc) are Debian arch names, i386, amd64, armhf.
 
 Standalone Mode

100 lines not on this page · raw message

index 17bb2c9..521682a 100755
@@ -22,6 +22,7 @@ set -ex
 . cri-args-hostlists
 . ap-common
 branch=$1; shift
+ftype=$1; shift
 select_branch
 info_linux_tree $branch ||:
 

@@ -231,7 +232,7 @@ if [ "x$NEW_REVISION" = "x$OLD_REVISION" ]; then
 fi
 
 $DAILY_BRANCH_PREMAKE_HOOK
-flight=`./make-flight $branch $xenbranch $OSSTEST_BLESSING "$@"`
+flight=`./make-$ftype-flight $branch $xenbranch $OSSTEST_BLESSING "$@"`
 $DAILY_BRANCH_POSTMAKE_HOOK
 
 heading=tmp/$flight.heading-info

index caf3fd5..22540c1 100755
@@ -39,6 +39,7 @@ Options:
 
 -c FILE, --config=FILE        Use FILE as configuration file
 -f FLIGHT, --flight=FLIGHT    Operate on FLIGHT
+-t FL_TYPE, --type=FL_TYPE    Flight type (for invoking make-<FL_TYPE>-flight)
 -h HOST, --host=HOST          Test host
 -r, --reuse                   Do not wipe test host (default)
 -R, --noreuse, --noreinstall  Wipe the test host (if job or test does so)

@@ -63,12 +64,13 @@ if [ x$op = x--help ] ; then
     exit 0
 fi
 
-TEMP=$(getopt -o c:f:h:rR --long config:,flight:,host:,reuse,noreuse,reinstall,lvextendmax:,baseline,help -- "$@")
+TEMP=$(getopt -o c:f:t:h:rR --long config:,flight:,ftype:,host:,reuse,noreuse,reinstall,lvextendmax:,baseline,help -- "$@")
 
 eval set -- "$TEMP"
 
 config=${OSSTEST_CONFIG-$HOME/.xen-osstest/config}
 flight="standalone"
+ftype=
 host=
 reuse=1 # Don't blow away machines by default
 lvextendmax=50 # Leave some LVM space free for running tests

@@ -78,6 +80,7 @@ while true ; do
     case "$1" in
 	-c|--config) config=$2; shift 2;;
 	-f|--flight) flight=$2; shift 2;;
+	-t|--type)   ftype=$2;  shift 2;;
 	-h|--host)   host=$2;   shift 2;;
 	-r|--reuse)  reuse=1;   shift 1;;
 	-R|--noreuse|--reinstall)reuse=0;shift 1;;

@@ -184,7 +187,7 @@ case $op in
         OSSTEST_FLIGHT=$flight \
         OSSTEST_CONFIG=$config \
         OSSTEST_NO_BASELINE=$nobaseline \
-            with_logging logs/$flight/make-flight.log ./cr-daily-branch $@ $branch
+            with_logging logs/$flight/make-flight.log ./cr-daily-branch $@ $branch $ftype
         ;;
 
     set-paths)

index 8555039..f041e6d 100755
@@ -23,14 +23,17 @@ usage(){
 usage: ./standalone-reset [<options>] [<branch> [<xenbranch> [<buildflight>]]]
  branch and xenbranch default, separately, to xen-unstable
 options:
- -f<flight>     generate flight "flight", default is "standalone"
+ -f <flight>      generate flight "flight", default is "standalone"
+ -t <flight_type> generate a different type of flight (it calls
+                  make-<flight_type>-flight)
 END
 }
 
 flight="standalone"
-while getopts "f:" opt; do
+while getopts "f:t:" opt; do
     case "$opt" in
         f) flight=${OPTARG};;
+        t) flight_type="-${OPTARG}";;
         *) usage; exit 1;;
     esac
 done

@@ -140,7 +143,7 @@ fi
 export BUILD_LVEXTEND_MAX
 
 OSSTEST_FLIGHT=$flight \
-./make-flight "$branch" "$xenbranch" play $buildflight >/dev/null
+./make${flight_type}-flight "$branch" "$xenbranch" play $buildflight >/dev/null
 
 #---------- done ----------

new file mode 100755
index 0000000..7ed252b
@@ -0,0 +1,49 @@
+#!/bin/bash
+# This is part of "osstest", an automated testing framework for Xen.
+# Copyright (C) 2009-2014 Citrix Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -e
+
+if [ -f standalone.config ] ; then
+    . standalone.config
+fi
+
+. cri-getconfig
+
+fail () { echo >&2 "$0: $1"; exit 1; }
+
+# By default we try to grab version 0.50, and we build linux 3.15.3
+site=${KERNBENCH_REMOTE:-http://ck.kolivas.org/apps/kernbench/kernbench-0.50}
+rfile=${KERNBENCH_FILE:-kernbench}
+site_kern=${KERNBENCH_REMOTE_KERN:-https://www.kernel.org/pub/linux/kernel/v3.x}
+rfile_kern=${KERNBENCH_FILE_KERN:-linux-3.15.3.tar.xz}
+
+lfile=kernbench
+lfile_kern=linux.xz
+
+images=`getconfig Images`;
+dstdir="${images}/benchs"
+mkdir -p $dstdir
+
+wget ${site}/${rfile} -O ${dstdir}/${lfile} || \
+    fail "failed downloading the benchmark"
+
+echo >&2 "downloaded $dstdir/$lfile"
+
+wget ${site_kern}/${rfile_kern} -O ${dstdir}/${lfile_kern} || \
+    fail "failed downloading the kernel to be built"
+
+echo >&2 "downloaded $dstdir/$lfile"

new file mode 100755
index 0000000..eea844c
@@ -0,0 +1,86 @@
+#!/usr/bin/perl -w
+# This is part of "osstest", an automated testing framework for Xen.
+# Copyright (C) 2009-2014 Citrix Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use strict qw(vars);
+use DBI;
+use Osstest;
+use Osstest::TestSupport;
+use File::Basename;
+
+tsreadconfig();
+
+# what we expect as argument list is:
+#  host=<somehost> [<someguest>]
+our ($whhost,$gn) = @ARGV;
+$whhost ||= 'host';
+our $ho= selecthost($whhost);
+
+our $gho= $ho;
+if (defined $gn and $gn ne "") {
+  $gho= selectguest($gn, $ho);
+  my $err= guest_check_ip($gho);
+  die "$err $gho->{Name}" if defined $err;
+}
+
+# By default, we expect to find kernbench 0,50, stored in
+# $c{Images}/benchs/kernbench. To use something different, define
+# r{'kernbench_file'}.
+our $kernbench_file= (defined($r{'kernbench_file'})) ? $r{'kernbench_file'} :
+    "$c{Images}/benchs/kernbench";
+
+# We also assume to find the linux kernel archive to be used for the benchmark
+# in $c{Images}benchs/linux.xz. To use something different, define
+# $r{'kernbench_kern_file'}.
+our $kernbench_linux_file= (defined($r{'kernbench_kern_file'})) ?
+    $r{'kernbench_kern_file'} : "$c{Images}/benchs/linux.xz";
+
+# Some deps...
+sub install_deps() {
+  target_install_packages_norec($gho, qw(build-essential time bc));
+}
+
+# Ship the benchmark and the linux kernel archive to the target machine
+sub ship() {
+  logm("Shipping $kernbench_file and $kernbench_linux_file");
+  target_putfile_root($gho, 60, "$kernbench_file", "/root/kernbench");
+  target_putfile_root($gho, 200, "$kernbench_linux_file", "/root/");
+
+  our $kernbench_linux_filename= basename($kernbench_linux_file);
+  logm("Extracting $kernbench_linux_filename");
+  target_cmd_root($gho, <<END, 200);
+      set -ex
+      chmod +x /root/kernbench
+      rm -rf /root/linux-kernbench
+      mkdir /root/linux-kernbench
+      tar xaf /root/$kernbench_linux_filename -C /root/linux-kernbench --strip-components=1
+END
+}
+
+sub prep() {
+  # We use 'alldefconfig', to stress the system a bit more
+  # (default would be 'allnoconfig').
+  target_cmd_root($gho, <<END, 200);
+      set -ex
+      cd /root/linux-kernbench
+      make mrproper
+      make alldefconfig
+END
+}
+
+install_deps();
+ship();
+prep();

new file mode 100755
index 0000000..389f6b3
@@ -0,0 +1,63 @@
+#!/usr/bin/perl -w
+# This is part of "osstest", an automated testing framework for Xen.
+# Copyright (C) 2009-2014 Citrix Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use strict qw(vars);
+use DBI;
+use Osstest;
+use Osstest::TestSupport;
+
+tsreadconfig();
+
+# what we expect as argument list is:
+#  host=<somehost> [<someguest>]
+our ($whhost,$gn) = @ARGV;
+$whhost ||= 'host';
+our $ho= selecthost($whhost);
+
+our $gho= $ho;
+if (defined $gn and $gn ne "") {
+  $gho= selectguest($gn, $ho);
+  my $err= guest_check_ip($gho);
+  die "$err $gho->{Name}" if defined $err;
+}
+
+# Runtime parameters for the benchmark. We expect them to be
+# specified via a runvar. If not, let's provide a safe default.
+if (!defined($r{ 'kernbench_params' })) {
+  store_runvar('kernbench_params',"-n 2");
+}
+our $args= $r{'kernbench_params'};
+our $timeout= (defined($r{'kernbench_timeout'})) ?
+      $r{'kernbench_timeout'} : 12000;
+
+sub run() {
+  logm("Running: /root/linux-kernbench/../kernbench $args (timeout: $timeout)");
+  target_cmd_root($gho, <<END, $timeout);
+      set -ex
+      cd /root/linux-kernbench/
+      rm -f kernbench.log timelog
+      time ../kernbench $args
+END
+}
+
+sub hwinfo() {
+  target_catfile_root_stash($gho, "/proc/cpuinfo");
+  target_catfile_root_stash($gho, "/proc/meminfo");
+}
+
+hwinfo();
+run();

new file mode 100755
index 0000000..dcb279d
@@ -0,0 +1,60 @@
+#!/usr/bin/perl -w
+# This is part of "osstest", an automated testing framework for Xen.
+# Copyright (C) 2009-2013 Citrix Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use strict qw(vars);
+use Osstest;
+use DBI;
+use IO::File;
+use POSIX;
+use Osstest::TestSupport;
+
+tsreadconfig();
+
+# what we expect as argument list is:
+#  host=<somehost> [<someguest>]
+our ($whhost,$gn) = @ARGV;
+$whhost ||= 'host';
+our $ho= selecthost($whhost);
+
+our $gho= $ho;
+if (defined $gn and $gn ne "") {
+  $gho= selectguest($gn, $ho);
+  my $err= guest_check_ip($gho);
+  die "$err $gho->{Name}" if defined $err;
+}
+
+our $lresfile= $r{'kernbench_params'};
+$lresfile =~ tr/ //ds;
+$lresfile .= (defined($r{'kernbench_run_suffix'})) ?
+      $r{'kernbench_run_suffix'} : '';
+$lresfile = "kernbench$lresfile";
+
+# Kernbench stores results in the linux kernel tree it
+# built, in a file called kernbench.log
+sub results() {
+  my $rresults_dir= "/root/linux-kernbench";
+  my $rresfile= "$rresults_dir/kernbench.log";
+
+  my $gho_hostname= target_cmd_output_root($gho, 'hostname');
+  target_cmd_root($gho, "chmod a+r $rresfile", 60);
+
+  logm("Fetching $rresfile from $gho_hostname");
+  target_getfile_root_stash($gho, 60, "$rresfile",
+      "$lresfile");
+}
+
+results();

index 0c5c538..ff45766 100644
@@ -34,6 +34,9 @@ BEGIN {
     @EXPORT      = qw(unixbench_process_results
                       unixbench_print_results
                       unixbench_plot_results
+                      kernbench_process_results
+                      kernbench_print_results
+                      kernbench_plot_results
                       );
     %EXPORT_TAGS = ( );
 

@@ -83,6 +86,15 @@ sub unixbench_print_results ($$) {
   close($h);
 }
 
+our $common_plot_opts= <<EOF;
+set key outside center top horizontal noreverse noenhanced autotitles nobox
+set xtics mirror rotate by -45 out
+set style data histogram
+set style histogram cluster gap 1
+set style fill solid border lt -1
+set boxwidth 1 absolute
+EOF
+
 sub unixbench_plot_results ($$$) {
   my ($dataf,$num_cols,$pfile)= @_;
   my $h= new IO::File "> $pfile.gp" or die "$!";

@@ -91,12 +103,7 @@ sub unixbench_plot_results ($$$) {
 set terminal png enhanced font "/usr/share/fonts/truetype/ttf-dejavu/DejaVuSans.ttf" 8 size 800,600
 set output '$pfile.png'
 set title 'Unixbench INDEXes for $flight.$job'
-set key outside center top horizontal noreverse noenhanced autotitles nobox
-set xtics mirror rotate by -45 out
-set style data histogram
-set style histogram cluster gap 1
-set style fill solid border lt -1
-set boxwidth 1 absolute
+$common_plot_opts
 set bmargin 13
 set rmargin 14
 SKIP_COL=1

@@ -112,4 +119,73 @@ EOF
   logm("WARNING: plotting file with \"$err\"") unless $ok;
 }
 
+sub kernbench_process_results ($$) {
+  my ($results_ref,$rfilen)= @_;
+  my $h= new IO::File "< $rfilen" or die "$!";
+
+  my $run;
+  while (<$h>) {
+    my ($bench,$val,$stdd);
+    if (m/.*load -j ([0-9]*)\s?Run.*$/) {
+      $run= $1 || 0;
+    }
+    if (m/^(\S[a-zA-z\s]*)\s([0-9]+.?[0-9]*)\s\(([0-9]+.?[0-9]*)\)$/) {
+      $bench=$1;
+      next if $bench =~ /Sleeps|Context|Percent/;
+      $val= $2;
+      $stdd= $3;
+      $$results_ref->{"$bench"}{Result}{"$run"}= $val;
+      $$results_ref->{"$bench"}{StdDev}{"$run"}= $stdd;
+    }
+    next;
+  }
+  close($h);
+}
+
+sub kernbench_print_results ($$) {
+  my ($results,$rfilen)= @_;
+  open my $h, "|-", "tee $rfilen" or die "$!";
+
+  printf $h "%-25s","\"What\"";
+  foreach my $i (sort keys $results->{'Elapsed Time'}{'Result'}) {
+    my $col= "kernbench -j";
+    $col .= ($i == 0) ? "" : " $i";
+    printf $h "%-20s","\"$col\""
+    # TODO: Include stddev too in the report
+  }
+  print $h "\n";
+  foreach my $b (keys $results) {
+    printf $h "%-25s","\"$b\"";
+    foreach my $i (sort keys $results->{"$b"}{'Result'}) {
+      printf $h "%-20s",$results->{$b}{'Result'}{$i};
+    }
+    print $h "\n";
+  }
+  close($h);
+}
+
+sub kernbench_plot_results ($$$) {
+  my ($dataf,$num_cols,$pfile)= @_;
+
+  my $h= new IO::File "> $pfile.gp" or die "$!";
+  print $h <<EOF;
+set terminal png enhanced font "/usr/share/fonts/truetype/ttf-dejavu/DejaVuSans.ttf" 8 size 800,600
+set output '$pfile.png'
+set title 'Kernbench Results for $flight.$job'
+$common_plot_opts
+set bmargin 6
+SKIP_COL=1
+NCOL=$num_cols
+HWIDTH=1.0/(NCOL+1.0)
+cols=''
+plot for [c=SKIP_COL+1:SKIP_COL+NCOL] '$dataf' using c:xtic(1) with histograms title columnhead, \\
+        for [c=SKIP_COL+1:SKIP_COL+NCOL] '' every ::1 using 0:c:c with labels notitle offset first -HWIDTH*(NCOL/2.0)+HWIDTH/2.0+(c-(SKIP_COL+1))*HWIDTH, character 2 rotate by 90
+EOF
+  close($h);
+
+  my $gp= can_run('gnuplot') or return;
+  my ($ok,$err)= run( command => "$gp $pfile.gp", verbose => 1 );
+  logm("WARNING: plotting file with \"$err\"") unless $ok;
+}
+
 1;

index dcb279d..113a4ce 100755
@@ -21,6 +21,7 @@ use DBI;
 use IO::File;
 use POSIX;
 use Osstest::TestSupport;
+use Osstest::Benchmarking;
 
 tsreadconfig();
 

@@ -43,9 +44,11 @@ $lresfile .= (defined($r{'kernbench_run_suffix'})) ?
       $r{'kernbench_run_suffix'} : '';
 $lresfile = "kernbench$lresfile";
 
+our $results;
+
 # Kernbench stores results in the linux kernel tree it
 # built, in a file called kernbench.log
-sub results() {
+sub fetch() {
   my $rresults_dir= "/root/linux-kernbench";
   my $rresfile= "$rresults_dir/kernbench.log";
 

@@ -57,4 +60,16 @@ sub results() {
       "$lresfile");
 }
 
-results();
+sub process () {
+  my $resf= "$stash/$gho->{Name}--$lresfile";
+  my $dataf= "$resf-DATA";
+  my $plotf= "$resf-PLOT";
+
+  kernbench_process_results(\$results,$resf);
+  kernbench_print_results($results,$dataf);
+  my $ncols= keys $results->{'Elapsed Time'}{'Result'};
+  kernbench_plot_results($dataf,$ncols,$plotf);
+}
+
+fetch();
+process();

index 81c2040..32c94db 100755
@@ -352,6 +352,35 @@ proc run-job/bench-unixbench-hvm {} {
     bench-unixbench-guest debianhvm
 }
 
+proc bench-kernbench-run {args} {
+    run-ts . = ts-kernbench-build  + host $args
+    run-ts . = ts-kernbench-run    + host $args
+    run-ts . = ts-kernbench-reslts + host $args
+}
+
+proc bench-kernbench-host {} {
+    bench-kernbench-run
+}
+
+proc bench-kernbench-guest {g} {
+    bench-kernbench-run $g
+    run-ts . = ts-guest-stop + host $g
+}
+
+proc need-hosts/bench-kernbench-pv {} { return host }
+proc run-job/bench-kernbench-pv {} {
+    run-ts . = ts-debian-install + host
+    run-ts . = ts-debian-fixup   + host debian
+    run-ts . = ts-guest-start    + host debian
+    bench-kernbench-guest debian
+}
+
+proc need-hosts/bench-kernbench-hvm {} { return host }
+proc run-job/bench-kernbench-hvm {} {
+    run-ts . = ts-debian-hvm-install
+    bench-kernbench-guest debianhvm
+}
+
 #---------- builds ----------
 
 proc need-hosts/build {} { return BUILD }

index cdb22ff..125f244 100755
@@ -56,8 +56,10 @@ test_matrix_branch_filter_callback () {
 }
 
 do_unixbench_tests () {
-  gvcpus=$1
-  gmem=$2
+  gt=$1
+  sched=$2
+  gvcpus=$3
+  gmem=$4
 
   # x86_64 only (for now)
   if [ $xenarch != amd64 ]; then

@@ -73,20 +75,47 @@ do_unixbench_tests () {
   if [ $gvcpus -ge 2 ];then params="-c $(($gvcpus/2))"; fi
   params="$params -c $gvcpus -c $(($gvcpus*2)) -i 6"
 
-  for gt in pv hvm; do
-    for sched in credit credit2; do
-      job_create_test \
-              bench-unixbench-$xenarch-$sched-$gt$gvcpus_suffix$gmem_suffix \
-              bench-unixbench-$gt xl $xenarch $dom0arch $gvcpus_runvars $gmem_runvars \
-              xen_boot_append="sched=$sched" unixbench_params="$params" $debian_runvars \
-              bios=seabios debianhvm_image=debian-7.2.0-amd64-CD-1.iso \
-              all_hostflags=$most_hostflags
-    done
-  done
+  job_create_test \
+          bench-unixbench-$xenarch-$sched-$gt$gvcpus_suffix$gmem_suffix \
+          bench-unixbench-$gt xl $xenarch $dom0arch $gvcpus_runvars $gmem_runvars \
+          xen_boot_append="sched=$sched" unixbench_params="$params" $debian_runvars \
+          bios=seabios debianhvm_image=debian-7.2.0-amd64-CD-1.iso \
+          all_hostflags=$most_hostflags
+}
+
+do_kernbench_tests () {
+  gt=$1
+  sched=$2
+  gvcpus=$3
+  gmem=$4
+
+  # x86_64 only (for now)
+  if [ $xenarch != amd64 ]; then
+    return
+  fi
+  # "homogeneous" tests only (for now)
+  if [ $xenarch != $dom0arch ]; then
+    return
+  fi
+
+  gvcpus_runvars=guests_vcpus=$gvcpus; gvcpus_suffix=-${gvcpus}vcpus
+  gmem_runvars=guests_memory=$gmem; gmem_suffix=-${gmem}ram
+
+  job_create_test \
+          bench-kernbench-$xenarch-$sched-$gt$gvcpus_suffix$gmem_suffix \
+          bench-kernbench-$gt xl $xenarch $dom0arch $gvcpus_runvars $gmem_runvars \
+          xen_boot_append="sched=$sched" kernbench_params="-n 4" $debian_runvars \
+          bios=seabios debianhvm_image=debian-7.2.0-amd64-CD-1.iso \
+          all_hostflags=$most_hostflags
 }
 
 test_matrix_do_one () {
-  do_unixbench_tests 4 4096 # 4 vcpus, 4GB RAM
+  for s in credit credit2; do
+    for t in pv hvm; do
+      do_unixbench_tests $t $s 4 4096 # 4 vcpus, 4GB RAM
+      do_kernbench_tests $t $s 4 4096
+    done
+  done
 }
 
 test_matrix_iterate

index 7cc5be6..251668a 100644
@@ -58,7 +58,7 @@ BEGIN {
                       target_put_guest_image target_editfile
                       target_editfile_root target_file_exists
                       target_catfile_stash target_catfile_root_stash
-                      target_run_apt
+                      target_file_contains target_run_apt
                       target_install_packages target_install_packages_norec
                       target_jobdir target_extract_jobdistpath_subdir
                       target_extract_jobdistpath target_guest_lv_name

@@ -67,6 +67,7 @@ BEGIN {
                       contents_make_cpio file_simple_write_contents
 
                       selecthost get_hostflags get_host_property
+                      get_host_cpus get_host_numanodes get_host_memory
                       get_host_native_linux_console
                       power_state power_cycle power_cycle_time
                       serial_fetch_logs

@@ -519,6 +520,15 @@ sub target_file_exists ($$) {
     die "$rfile $out ?";
 }
 
+sub target_file_contains ($$$) {
+  my ($ho,$rfile,$filecont) = @_;
+  return 0 unless target_file_exists($ho,$rfile);
+  my $out= target_cmd_output($ho, "grep $filecont $rfile");
+  return 1 if ($out ne "");
+  return 0 if ($out eq "");
+  die "$rfile $filecont $out ?";
+}
+
 sub teditfileex {
     my $user= shift @_;
     my $code= pop @_;

@@ -866,6 +876,11 @@ sub selecthost ($) {
     }
     $ho->{Ip}= $ho->{IpStatic};
 
+    #----- HW specs -----
+    $ho->{Cpus} = get_host_property($ho,'cpus');
+    $ho->{Memory} = get_host_property($ho,'memory');
+    $ho->{Nodes} = get_host_property($ho,'nodes');
+
     #----- tftp -----
 
     my $tftpscope = get_host_property($ho, 'TftpScope', $c{TftpDefaultScope});

@@ -937,6 +952,66 @@ sub get_host_method_object ($$$) {
     return $mo;
 }
 
+sub get_host_cpus ($) {
+    my ($ho) = @_;
+
+    # Let's first try if there's an host property defined;
+    # if no, we'll "ask" the host directly.
+    my $cpus= get_host_property($ho,'cpus',undef);
+    return $cpus if defined $cpus;
+
+    # Is the host running Dom0 or baremetal?
+    if (target_file_contains($ho,"/proc/xen/capabilities","control_d")) {
+        $cpus= target_cmd_output_root($ho,
+            "xl info | grep ^nr_cpus | awk '{print \$3}'");
+    } else {
+        $cpus= target_cmd_output_root($ho,
+            "cat /proc/cpuinfo | grep '^processor' | wc -l");
+    }
+
+    return $cpus;
+}
+
+sub get_host_numanodes ($) {
+    my ($ho) = @_;
+
+    # Let's first try if there's an host property defined;
+    # if no, we'll "ask" the host directly.
+    my $nodes= get_host_property($ho,'nodes',undef);
+    return $nodes if defined $nodes;
+
+    # Is the host running Dom0 or baremetal?
+    if (target_file_contains($ho,"/proc/xen/capabilities","control_d")) {
+        $nodes= target_cmd_output_root($ho,
+            "xl info | grep ^nr_nodes | awk '{print \$3}'");
+    } else {
+        $nodes= target_cmd_output_root($ho,
+            "which numactl && numactl --hardware | grep ^available: | awk '{print \$2}'");
+    }
+
+    return $nodes;
+}
+
+sub get_host_memory ($) {
+    my ($ho) = @_;
+
+    # Let's first try if there's an host property defined;
+    # if no, we'll "ask" the host directly.
+    my $mem= get_host_property($ho,'memory',undef);
+    return $mem if defined $mem;
+
+    # Is the host running Dom0 or baremetal?
+    if (target_file_contains($ho,"/proc/xen/capabilities","control_d")) {
+        $mem= target_cmd_output_root($ho,
+            "xl info | grep ^total_memory | awk '{print \$3}'");
+    } else {
+        $mem= target_cmd_output_root($ho,
+            "free -m | grep ^Mem: | awk '{print \$2}'");
+    }
+
+    return $mem;
+}
+
 #---------- stashed files ----------
 
 sub open_unique_stashfile ($) {

index 45d1498..b3880b5 100644
@@ -343,6 +343,21 @@ HostProp_<testbox>_TftpScope
    Defines the Tftp scope (i.e. subnet) where this host resides. See
    "TftpFoo_<scope> and TftpFoo" below.
 
+HostProp_<testbox>_Cpus
+   Tells how many physical CPUs the testbox has. If this is defined,
+   no further investigation is performed to figure out such information
+   and the value provided here is considered reliable and consumed.
+
+HostProp_<testbox>_Memory
+   Tells how much physical memory the testbox has. If this is defined,
+   no further investigation is performed to figure out such information
+   and the value provided here is considered reliable and consumed.
+
+HostProp_<testbox>_Nodes
+   Tells how many NUMA nodes the testbox has. If this is defined,
+   no further investigation is performed to figure out such information
+   and the value provided here is considered reliable and consumed.
+
 HostFlags_<testbox>
    Defines a set of flags for the host. Flags is a list separated by
    whitespace, comma or semi-colon. A flag can be unset by prepending

new file mode 100755
index 0000000..a47d72d
@@ -0,0 +1,35 @@
+#!/bin/bash
+# This is part of "osstest", an automated testing framework for Xen.
+# Copyright (C) 2009-2014 Citrix Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -e
+
+# Is the host running Dom0 or baremetal?
+if [[ -e /proc/xen/capabilities ]] && \
+   [[ `grep ^control_d$ /proc/xen/capabilities` == "control_d" ]]; then
+  cpus=`xl info | grep ^nr_cpus | awk '{print \$3}'`
+  memory=`xl info | grep ^total_memory | awk '{print \$3}'`
+  nodes=`xl info | grep ^nr_nodes | awk '{print \$3}'`
+else
+  cpus=`cat /proc/cpuinfo | grep "^processor" | wc -l`
+  memory=`free -m | grep ^Mem: | awk '{print $2}'`
+  nodes="?"
+  if [[ `which numactl` != "" ]] && [ -x `which numactl` ]; then
+    nodes=`numactl --hardware | grep ^available: | awk '{print $2}'`
+  fi
+fi
+
+echo >&2 "cpus=$cpus / memory=$memory / nodes=$nodes"

new file mode 100755
index 0000000..ffde981
@@ -0,0 +1,87 @@
+#!/usr/bin/perl -w
+# This is part of "osstest", an automated testing framework for Xen.
+# Copyright (C) 2009-2014 Citrix Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use strict qw(vars);
+use DBI;
+use Osstest;
+use Osstest::TestSupport;
+
+tsreadconfig();
+
+our ($whhost,$gn) = @ARGV;
+$whhost ||= 'host';
+$gn ||= 'debianhvm';
+
+our ($ho,$gho) = ts_get_host_guest($whhost,$gn);
+our $toolstack= toolstack()->{Command};
+
+# We want the guest to have:
+#  - the same number of vcpus than the host has pcpus
+#  - (almost) the same amount of memory the host has
+#
+# For both vcpus and memory, a maximum can be specified via
+# runvars.
+sub fixup () {
+  our ($hnodes,$hcpus,$hmem);
+  our ($cfgfile,$bcpus,$bmem,$hfreemem);
+
+  $hcpus= get_host_cpus($ho);
+  $hmem= get_host_memory($ho);
+  die unless (defined $hcpus and defined $hmem);
+
+  $hnodes= get_host_numanodes($ho);
+  if (defined $hnodes and $hnodes > 1) {
+    logm("WARNING: the host has $hnodes NUMA nodes. This may spoil results");
+  }
+
+  $bcpus= (!defined($r{'max_bench_cpus'})) ? $hcpus :
+      ($r{'max_bench_cpus'} > $hcpus) ? $hcpus : $r{'max_bench_cpus'};
+  $bmem= (!defined($r{'max_bench_mem'})) ? $hmem :
+      ($r{'max_bench_mem'} > $hmem) ? $hmem : $r{'max_bench_mem'};
+
+  $hfreemem = host_get_free_memory($ho,$toolstack);
+  if ($hfreemem < $bmem) {
+    $bmem= $hfreemem - 1024;
+    logm("WARNING: Not enough free memory. Using $bmem MB");
+  }
+
+  logm("Will run the benchmark with: $bcpus vcpus and $bmem MB RAM");
+
+  $cfgfile= $r{"$gho->{Guest}_cfgpath"};
+  target_editfile_root($ho, $cfgfile, sub {
+      while (<EI>) {
+        s/^vcpus.*/vcpus=$bcpus/;
+        s/^memory.*/memory=$bmem/;
+	print EO or die $!;
+      }
+  });
+}
+
+sub start () {
+  # Start the guest...
+  guest_umount_lv($ho, $gho);
+  my $cmd= toolstack()->{Command}." create ".
+      $r{ $gho->{Guest}.'_'. toolstack()->{CfgPathVar} };
+  target_cmd_root($ho, $cmd, 30);
+  # ...And verify it's up and running
+  guest_checkrunning($ho, $gho) or die "$gho->{Name} not running";
+  guest_await($gho, target_var($gho,'boot_timeout'));
+  guest_check_up($gho);
+}
+
+fixup();
+start();

index 70afaec..418d9f2 100644
@@ -32,6 +32,9 @@ BEGIN {
     $VERSION     = 1.00;
     @ISA         = qw(Exporter);
     @EXPORT      = qw(debian_boot_setup
+                      setupboot_uboot
+                      setupboot_grub1
+                      setupboot_grub2
                       %preseed_cmds
                       preseed_base
                       preseed_create

@@ -112,7 +115,7 @@ sub bl_getmenu_open ($$$) {
     return $f;
 }
 
-sub setupboot_uboot ($$$) {
+sub setupboot_uboot ($$$$) {
     my ($ho,$want_kernver,$xenhopt,$xenkopt) = @_;
     my $bl= { };
 

@@ -194,7 +197,7 @@ END
     return $bl;
 }
 
-sub setupboot_grub1 ($$$) {
+sub setupboot_grub1 ($$$$) {
     my ($ho,$want_kernver,$xenhopt,$xenkopt) = @_;
     my $bl= { };
 

@@ -274,7 +277,7 @@ sub setupboot_grub1 ($$$) {
     return $bl;
 }
 
-sub setupboot_grub2 ($$$) {
+sub setupboot_grub2 ($$$$) {
     my ($ho,$want_kernver,$xenhopt,$xenkopt) = @_;
     my $bl= { };
 

@@ -387,6 +390,14 @@ END
                 my $v= $k{$k};
                 $v =~ s/\bquiet\b//;
                 $v =~ s/\b(?:console|xencons)=[0-9A-Za-z,]+//;
+                # Get rid of any host/dom0 resource constraining. Idea is:
+                # (1) in the dom0 case, something like that should be achieved
+                # via Xen boot parameters; (2) in any case, if it is important
+                # to have something like these in Linux's boot cmd line by
+                # default, that should be put into the 'linux_boot_append'
+                # runvar, which won't be affected by this.
+                $v =~ s/\bmaxcpus=[0-9A-Za-z,]+//;
+                $v =~ s/\bmem=[0-9A-Za-z,]+//;
                 $v .= " $xenkopt" if $k eq 'GRUB_CMDLINE_LINUX';
                 print ::EO "$k=\"$v\"\n" or die $!;
             }

74 lines not on this page · raw message

index 418d9f2..447d2d2 100644
@@ -87,25 +87,27 @@ sub debian_boot_setup ($$$$;$) {
     my $kern= $bootloader->{GetBootKern}();
     logm("dom0 kernel is $kern");
 
-    system "tar zvtf $distpath->{kern} boot/$kern";
-    $? and die "$distpath->{kern} boot/$kern $?";
-
-    my $kernver= $kern;
-    $kernver =~ s,^/?(?:boot/)?(?:vmlinu[xz]-)?,, or die "$kernver ?";
-    my $kernpath= $kern;
-    $kernpath =~ s,^(?:boot/)?,/boot/,;
-
-    target_cmd_root($ho,
-                    "update-initramfs -k $kernver -c ||".
-                    " update-initramfs -k $kernver -u",
-                    200);
+    if (defined $distpath) {
+        system "tar zvtf $distpath->{kern} boot/$kern";
+        $? and die "$distpath->{kern} boot/$kern $?";
+
+        my $kernver= $kern;
+        $kernver =~ s,^/?(?:boot/)?(?:vmlinu[xz]-)?,, or die "$kernver ?";
+        my $kernpath= $kern;
+        $kernpath =~ s,^(?:boot/)?,/boot/,;
+
+        target_cmd_root($ho,
+                        "update-initramfs -k $kernver -c ||".
+                        " update-initramfs -k $kernver -u",
+                        200);
+
+        store_runvar(target_var_prefix($ho).'xen_kernel_path',$kernpath);
+        store_runvar(target_var_prefix($ho).'xen_kernel_ver',$kernver);
+    }
 
     $bootloader->{PreFinalUpdate}();
 
     $bootloader->{UpdateConfig}($ho);
-
-    store_runvar(target_var_prefix($ho).'xen_kernel_path',$kernpath);
-    store_runvar(target_var_prefix($ho).'xen_kernel_ver',$kernver);
 }
 
 sub bl_getmenu_open ($$$) {

58 lines not on this page · raw message

new file mode 100755
index 0000000..26a67f4
@@ -0,0 +1,39 @@
+#!/usr/bin/perl -w
+# This is part of "osstest", an automated testing framework for Xen.
+# Copyright (C) 2009-2014 Citrix Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use strict qw(vars);
+use DBI;
+use Osstest;
+use Osstest::Debian;
+use Osstest::TestSupport;
+
+tsreadconfig();
+
+our ($whhost) = @ARGV;
+$whhost ||= 'host';
+our $ho= selecthost($whhost);
+
+sub resetboot () {
+  my ($xenhopt,@hooks)= host_bootxen_setup($ho);
+  my $want_kernver = get_runvar('kernel_ver',$r{'kernbuildjob'});
+
+  debian_boot_setup($ho, $want_kernver, $xenhopt, undef, \@hooks);
+
+  logm("host reset to booting Xen");
+}
+
+resetboot();

index 4d34d1f..d8d38fc 100755
@@ -134,43 +134,9 @@ sub adjustconfig () {
 }
 
 sub setupboot () {
-    my $xenhopt= "conswitch=x watchdog";
-
-    my $cons= get_host_property($ho, 'XenSerialConsole', 'com1');
-
-    if ( $cons eq "com1" ) {
-	$xenhopt .= " com1=$c{Baud},8n1 console=com1,vga gdb=com1";
-    } elsif ( $cons eq "dtuart" ) {
-	$xenhopt .= " console=dtuart";
-	my $dtuart= get_host_property($ho, 'XenDTUARTPath', undef);
-	$xenhopt .= " dtuart=$dtuart" if $dtuart;
-    } else {
-	logm("No Xen console device defined for host");
-    }
-    if (toolstack()->{Dom0MemFixed}) {
-        $xenhopt .= " dom0_mem=512M,max:512M";
-    }
-    my $append= $r{xen_boot_append};
-    $xenhopt .= " $append" if defined $append;
-    $append = get_host_property($ho, 'xen-commandline-append', undef);
-    $xenhopt .= " $append" if defined $append;
-
-    my @hooks;
-
-    if (host_involves_pcipassthrough($ho)) {
-        push @hooks, {
-            EditBootOptions => sub {
-                my ($ho,$hopt,$kopt) = @_;
-                $$hopt .= ' iommu=on';
-                my $hide= ' xen-pciback.hide='. join '',map { "($_->{Bdf})" }
-                    host_get_pcipassthrough_devs($ho);
-                logm("pci passthrough: hiding in dom0: $hide");
-                $$kopt .= $hide;
-            }
-        };
-    }
-
+    my ($xenhopt,@hooks)= host_bootxen_setup($ho);
     my $want_kernver = get_runvar('kernel_ver',$r{'kernbuildjob'});
+
     debian_boot_setup($ho, $want_kernver, $xenhopt, \%distpath, \@hooks);
 
     logm("ready to boot Xen");

index b3880b5..1e777df 100644
@@ -200,6 +200,17 @@ bench-$BENCHNAME-$ARCH-<XEN_OPTS>-<GUEST_OPTS>
         <GUEST_OPTS> tells about the guest's configuration (e.g., whether
         it's HVM or PV, number of vCPUs, RAM, etc.).
 
+bench-hostcmp-$BENCHNAME-$ARCH-<XEN_OPTS>-<GUEST_OPTS>
+
+        A benchmarking job, for comparing baremetal and guest performances.
+        This runs benchmark $BENCHNAME on a $ARCH baremetal host, and on
+        PV and HVM guests running on a $ARCH hypervisor and dom0.
+
+        In the remainder of the job's name, <XEN_OPTS> tells something
+        about Xen's configuration (e.g., what scheduler will be used);
+        <GUEST_OPTS> tells about the guest's configuration (e.g., whether
+        it's HVM or PV, number of vCPUs, RAM, etc.).
+
 NB: $ARCH (and $XENARCH etc) are Debian arch names, i386, amd64, armhf.
 
 Standalone Mode

index 125f244..d41cbbd 100755
@@ -109,12 +109,32 @@ do_kernbench_tests () {
           all_hostflags=$most_hostflags
 }
 
+do_hostcmp_unixbench_tests () {
+  sched=$1
+
+  # x86_64 only (for now)
+  if [ $xenarch != amd64 ]; then
+    return
+  fi
+  # "homogeneous" tests only (for now)
+  if [ $xenarch != $dom0arch ]; then
+    return
+  fi
+
+  job_create_test \
+          bench-hostcmp-unixbench-$xenarch-$sched bench-hostcmp-unixbench \
+          xl $xenarch $dom0arch xen_boot_append="sched=$sched" max_bench_cpus=16 \
+          max_bench_mem=24576 $debian_runvars bios=seabios unixbench_params="-i 3" \
+          debbenchhvm_image=debian-7.2.0-amd64-CD-1.iso all_hostflags=$most_hostflags
+}
+
 test_matrix_do_one () {
   for s in credit credit2; do
     for t in pv hvm; do
       do_unixbench_tests $t $s 4 4096 # 4 vcpus, 4GB RAM
       do_kernbench_tests $t $s 4 4096
     done
+    do_hostcmp_unixbench_tests $s
   done
 }
 

index 32c94db..5954032 100755
@@ -30,10 +30,17 @@ proc run-job {job} {
     set anyfailed 0
     jobdb::prepare $job
 
+    set do_leak_check 1
     set nh [need-hosts/$jobinfo(recipe)]
     if {![string compare $nh BUILD]} {
         set need_xen_hosts {}
         set need_build_host 1
+    } elseif {[string match REBOOT* $nh]} {
+        # no leak checks for tests that reboot the host
+        set do_leak_check 0
+        set nh [lreplace $nh 0 0]
+        set need_xen_hosts $nh
+        set need_build_host 0
     } else {
         set need_xen_hosts $nh
         set need_build_host 0

@@ -54,10 +61,10 @@ proc run-job {job} {
     per-host-ts .       xen-install/@     ts-xen-install
     per-host-ts .       xen-boot/@        ts-host-reboot
 
-    per-host-ts .       =(*)             {ts-leak-check basis}
+    if {$do_leak_check} {per-host-ts . =(*) {ts-leak-check basis}         }
 
     if {$ok} { catching-otherwise fail      run-job/$jobinfo(recipe)      }
-    per-host-ts .       =                {ts-leak-check check}
+    if {$do_leak_check} {per-host-ts . = {ts-leak-check check}            }
 
     if {!$need_build_host} {
         per-host-ts !broken capture-logs/@(*) ts-logs-capture

@@ -381,6 +388,28 @@ proc run-job/bench-kernbench-hvm {} {
     bench-kernbench-guest debianhvm
 }
 
+proc need-hosts/bench-hostcmp-unixbench {} { return {REBOOT host} }
+proc run-job/bench-hostcmp-unixbench {} {
+    # Run benchmark in a PV guest
+    set g debbenchpv
+    run-ts . = ts-debian-install           + host $g
+    run-ts . = ts-debian-fixup             + host $g
+    run-ts . = ts-bench-hostcmp-guest-prep + host $g
+    bench-unixbench-guest $g
+    # Run benchmark in an HVM guest
+    set g debbenchhvm
+    run-ts . = ts-debian-hvm-install       + host $g
+    run-ts . = ts-guest-stop               + host $g
+    run-ts . = ts-bench-hostcmp-guest-prep + host $g
+    bench-unixbench-guest $g
+    # Run benchmark on the host
+    run-ts . = ts-bench-hostcmp-host-prep
+    run-ts . = ts-host-reboot
+    bench-unixbench-host
+    run-ts . = ts-bench-hostcmp-post
+    run-ts . = ts-host-reboot
+}
+
 #---------- builds ----------
 
 proc need-hosts/build {} { return BUILD }

index ff45766..301af08 100644
@@ -96,21 +96,31 @@ set boxwidth 1 absolute
 EOF
 
 sub unixbench_plot_results ($$$) {
-  my ($dataf,$num_cols,$pfile)= @_;
-  my $h= new IO::File "> $pfile.gp" or die "$!";
+  my ($dfiles,$num_cols,$pfile)= @_;
+  my $f= keys @$dfiles;
+  my $s= join(' ',@$dfiles);
 
-  printf $h <<EOF;
+  my $h= new IO::File "> $pfile.gp" or die "$!";
+  print $h <<EOF;
 set terminal png enhanced font "/usr/share/fonts/truetype/ttf-dejavu/DejaVuSans.ttf" 8 size 800,600
 set output '$pfile.png'
 set title 'Unixbench INDEXes for $flight.$job'
 $common_plot_opts
 set bmargin 13
 set rmargin 14
+NDATA=$num_cols
+NHOSTS=$f
 SKIP_COL=1
-NCOL=$num_cols
+NCOL=1*(NHOSTS*NDATA)
 HWIDTH=1.0/(NCOL+1.0)
-plot for [c=SKIP_COL+1:SKIP_COL+NCOL] '$dataf' using c:xtic(1) with histograms title columnhead, \\
-        for [c=SKIP_COL+1:SKIP_COL+NCOL]'' every ::1 using 0:c:c with labels notitle offset first -HWIDTH*(NCOL/2.0)+HWIDTH/2.0+(c-(SKIP_COL+1))*HWIDTH, character 2 rotate by 90
+cols=''
+do for [h=0:NHOSTS-1] {
+        do for [c=1+h*(NDATA+SKIP_COL)+SKIP_COL:1+h*(NDATA+SKIP_COL)+SKIP_COL+NDATA-1] {
+          cols = cols . sprintf("\%d ", c);
+        }
+}
+plot for [c in cols] '< paste $s' using int(c):xtic(1) with histograms title columnhead, \\
+        for [i=1:words(cols)] '' every ::1 using 0:int(word(cols,i)):int(word(cols,i)) with labels notitle offset first -HWIDTH*(NCOL/2.0)+HWIDTH/2.0+(i-1)*HWIDTH, character 2 rotate by 90
 EOF
   close($h);
 

index 5954032..b16584a 100755
@@ -406,7 +406,7 @@ proc run-job/bench-hostcmp-unixbench {} {
     run-ts . = ts-bench-hostcmp-host-prep
     run-ts . = ts-host-reboot
     bench-unixbench-host
-    run-ts . = ts-bench-hostcmp-post
+    run-ts . = ts-bench-hostcmp-post       + host unixbench
     run-ts . = ts-host-reboot
 }
 

59 lines not on this page · raw message

index b480d15..6c3e0a3 100755
@@ -69,15 +69,15 @@ END
 
 sub process () {
   my $resf= "$stash/$gho->{Name}--$lresfile";
-  my $dataf= "$resf-DATA";
+  my @dataf= "$resf-DATA";
   my $plotf= "$resf-PLOT";
 
   unixbench_process_results(\$results,$resf);
-  unixbench_print_results($results,$dataf);
+  unixbench_print_results($results,$dataf[0]);
 
   # For plotting we need to know the number of data columns
   my $ncols= keys $results->{'Double-Precision Whetstone'}{Index};
-  unixbench_plot_results($dataf,$ncols,$plotf);
+  unixbench_plot_results(\@dataf,$ncols,$plotf);
 }
 
 fetch();

index d41cbbd..12ac9f7 100755
@@ -109,8 +109,9 @@ do_kernbench_tests () {
           all_hostflags=$most_hostflags
 }
 
-do_hostcmp_unixbench_tests () {
-  sched=$1
+do_bench_hostcmp_tests () {
+  bench=$1
+  sched=$2
 
   # x86_64 only (for now)
   if [ $xenarch != amd64 ]; then

@@ -122,10 +123,11 @@ do_hostcmp_unixbench_tests () {
   fi
 
   job_create_test \
-          bench-hostcmp-unixbench-$xenarch-$sched bench-hostcmp-unixbench \
+          bench-hostcmp-$bench-$xenarch-$sched bench-hostcmp-$bench \
           xl $xenarch $dom0arch xen_boot_append="sched=$sched" max_bench_cpus=16 \
           max_bench_mem=24576 $debian_runvars bios=seabios unixbench_params="-i 3" \
-          debbenchhvm_image=debian-7.2.0-amd64-CD-1.iso all_hostflags=$most_hostflags
+          kernbench_params="-n 3" debbenchhvm_image=debian-7.2.0-amd64-CD-1.iso \
+          all_hostflags=$most_hostflags
 }
 
 test_matrix_do_one () {

@@ -134,7 +136,9 @@ test_matrix_do_one () {
       do_unixbench_tests $t $s 4 4096 # 4 vcpus, 4GB RAM
       do_kernbench_tests $t $s 4 4096
     done
-    do_hostcmp_unixbench_tests $s
+    for b in unixbench kernbench; do
+      do_bench_hostcmp_tests $b $s
+    done
   done
 }
 

index b16584a..1b3f4d6 100755
@@ -388,28 +388,37 @@ proc run-job/bench-kernbench-hvm {} {
     bench-kernbench-guest debianhvm
 }
 
-proc need-hosts/bench-hostcmp-unixbench {} { return {REBOOT host} }
-proc run-job/bench-hostcmp-unixbench {} {
+proc bench-hostcmp {bench} {
     # Run benchmark in a PV guest
     set g debbenchpv
     run-ts . = ts-debian-install           + host $g
     run-ts . = ts-debian-fixup             + host $g
     run-ts . = ts-bench-hostcmp-guest-prep + host $g
-    bench-unixbench-guest $g
+    bench-$bench-guest $g
     # Run benchmark in an HVM guest
     set g debbenchhvm
     run-ts . = ts-debian-hvm-install       + host $g
     run-ts . = ts-guest-stop               + host $g
     run-ts . = ts-bench-hostcmp-guest-prep + host $g
-    bench-unixbench-guest $g
+    bench-$bench-guest $g
     # Run benchmark on the host
     run-ts . = ts-bench-hostcmp-host-prep
     run-ts . = ts-host-reboot
-    bench-unixbench-host
-    run-ts . = ts-bench-hostcmp-post       + host unixbench
+    bench-$bench-host
+    run-ts . = ts-bench-hostcmp-post       + host $bench
     run-ts . = ts-host-reboot
 }
 
+proc need-hosts/bench-hostcmp-unixbench {} { return {REBOOT host} }
+proc run-job/bench-hostcmp-unixbench {} {
+    bench-hostcmp unixbench
+}
+
+proc need-hosts/bench-hostcmp-kernbench {} { return {REBOOT host} }
+proc run-job/bench-hostcmp-kernbench {} {
+    bench-hostcmp kernbench
+}
+
 #---------- builds ----------
 
 proc need-hosts/build {} { return BUILD }

index 301af08..1be9c97 100644
@@ -175,7 +175,9 @@ sub kernbench_print_results ($$) {
 }
 
 sub kernbench_plot_results ($$$) {
-  my ($dataf,$num_cols,$pfile)= @_;
+  my ($dfiles,$num_cols,$pfile)= @_;
+  my $f= keys @$dfiles;
+  my $s= join(' ',@$dfiles);
 
   my $h= new IO::File "> $pfile.gp" or die "$!";
   print $h <<EOF;

@@ -184,12 +186,19 @@ set output '$pfile.png'
 set title 'Kernbench Results for $flight.$job'
 $common_plot_opts
 set bmargin 6
+NDATA=$num_cols
+NHOSTS=$f
 SKIP_COL=1
-NCOL=$num_cols
+NCOL=1*(NHOSTS*NDATA)
 HWIDTH=1.0/(NCOL+1.0)
 cols=''
-plot for [c=SKIP_COL+1:SKIP_COL+NCOL] '$dataf' using c:xtic(1) with histograms title columnhead, \\
-        for [c=SKIP_COL+1:SKIP_COL+NCOL] '' every ::1 using 0:c:c with labels notitle offset first -HWIDTH*(NCOL/2.0)+HWIDTH/2.0+(c-(SKIP_COL+1))*HWIDTH, character 2 rotate by 90
+do for [h=0:NHOSTS-1] {
+        do for [c=1+h*(NDATA+SKIP_COL)+SKIP_COL:1+h*(NDATA+SKIP_COL)+SKIP_COL+NDATA-1] {
+          cols = cols . sprintf("\%d ", c);
+        }
+}
+plot for [c in cols] '< paste $s' using int(c):xtic(1) with histograms title columnhead, \\
+        for [i=1:words(cols)] '' every ::1 using 0:int(word(cols,i)):int(word(cols,i)) with labels notitle offset first -HWIDTH*(NCOL/2.0)+HWIDTH/2.0+(i-1)*HWIDTH, character 2 rotate by 90
 EOF
   close($h);
 

index 383bac0..ee9cf0a 100755
@@ -65,6 +65,7 @@ sub plot_hostcmp () {
     close FH;
   }
   unixbench_plot_results(\@dfiles,$ncols,"$stash/$job-PLOT") if $bn eq "unixbench";
+  kernbench_plot_results(\@dfiles,$ncols,"$stash/$job-PLOT") if $bn eq "kernbench";
 }
 
 sub resetboot () {

index 113a4ce..b9ee393 100755
@@ -62,13 +62,13 @@ sub fetch() {
 
 sub process () {
   my $resf= "$stash/$gho->{Name}--$lresfile";
-  my $dataf= "$resf-DATA";
+  my @dataf= "$resf-DATA";
   my $plotf= "$resf-PLOT";
 
   kernbench_process_results(\$results,$resf);
-  kernbench_print_results($results,$dataf);
+  kernbench_print_results($results,$dataf[0]);
   my $ncols= keys $results->{'Elapsed Time'}{'Result'};
-  kernbench_plot_results($dataf,$ncols,$plotf);
+  kernbench_plot_results(\@dataf,$ncols,$plotf);
 }
 
 fetch();