Immediately retry failing tests
cover Immediately retry failing tests
We discussed this at the Xen Summit. What I do here is immediate retry the jobs with regressions, and then reanalyse the original full flight. If the retries showed the failures were heisenbugs, this will let them though.
This should reduce the negative impact on development, of heisenbugs, but it won't do anything to help keep them out of the tree.
I think this approach was basically already agreed, but I'm CCing the committers here for form's sake.
README.dev | 9 +++---- cr-daily-branch | 58 ++++++++++++++++++++++++++++++++++++++++++--- cr-disk-report | 2 +- cr-try-bisect-adhoc | 2 +- cri-args-hostlists | 28 +++++++++++++++------- cs-bisection-step | 4 ++-- sg-report-flight | 35 +++++++++++++++++++++++---- sg-run-job | 9 ++++++- 8 files changed, 121 insertions(+), 26 deletions(-)
Signature · 1 line
2.20.1
PATCH 1/13 Honour OSSTEST_SIMULATE=2 to actually run dummy flight
--- cri-args-hostlists | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
cri-args-hostlists +2 -2
@@ -68,8 +68,8 @@ fi execute_flight () { case "x$OSSTEST_SIMULATE" in - x|x0) ;; - *) echo SIMULATING - NOT EXECUTING $1 $2 + x|x0|x2) ;; + *) echo SIMULATING $OSSTEST_SIMULATE - NOT EXECUTING $1 $2 return ;; esac
Signature · 1 line
2.20.1
PATCH 2/13 Honour OSSTEST_SIMULATE_FAIL in sg-run-job
In reply to Ian JacksonThis is a Tcl list of globs for <job>.<step>, and allows for simulating particular test failures.
--- sg-run-job | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-)
sg-run-job +8 -1
@@ -406,7 +406,14 @@ proc spawn-ts {iffail testid args} { jobdb::spawn-step-commit $flight $jobinfo(job) $stepno $testid set xprefix {} - if {[var-or-default env(OSSTEST_SIMULATE) 0]} { set xprefix echo } + if {[var-or-default env(OSSTEST_SIMULATE) 0]} { + set xprefix echo + foreach ent [var-or-default env(OSSTEST_SIMULATE_FAIL) {}] { + if {[string match $ent $jobinfo(job).$testid]} { + set xprefix OSSTEST_SIMULATE_FAIL + } + } + } set log [jobdb::step-log-filename $flight $jobinfo(job) $stepno $ts] set redirects {}
Signature · 1 line
2.20.1
PATCH 3/13 sg-report-job-history: eval $DAILY_BRANCH_PREEXEC_HOOK
In reply to Ian JacksonFrom: Ian Jackson <[email protected]>
Put the call to this debugging/testing variable inside an eval. This allows a wider variety of stunts. The one in-tree reference is already compatible with this new semantics.
--- cr-daily-branch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
cr-daily-branch +1 -1
@@ -472,7 +472,7 @@ sgr_args+=" $EXTRA_SGR_ARGS" date >&2 : $flight $branch $OSSTEST_BLESSING $sgr_args -$DAILY_BRANCH_PREEXEC_HOOK +eval "$DAILY_BRANCH_PREEXEC_HOOK" execute_flight $flight $OSSTEST_BLESSING date >&2
Signature · 1 line
2.20.1
PATCH 4/13 cri-args-hostlists: New debug var $OSSTEST_REPORT_JOB_HISTORY_RUN
In reply to Ian JacksonFrom: Ian Jackson <[email protected]>
No effect if this is empty.
--- cri-args-hostlists | 1 + 1 file changed, 1 insertion(+)
cri-args-hostlists +1 -0
@@ -121,6 +121,7 @@ start_email () { date >&2 + $OSSTEST_REPORT_JOB_HISTORY_RUN \ with-lock-ex -w $globallockdir/report-lock \ ./sg-report-job-history --report-processing-start-time \ --html-dir=$job_html_dir --flight=$flight
Signature · 1 line
2.20.1
PATCH 5/13 cri-args-hostlists: Break out report_flight and publish_logs
In reply to Ian JacksonFrom: Ian Jackson <[email protected]>
NFC.
--- cri-args-hostlists | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-)
cri-args-hostlists +15 -6
@@ -128,10 +128,7 @@ start_email () { date >&2 - ./sg-report-flight --report-processing-start-time \ - --html-dir=$flight_html_dir/$flight/ \ - --allow=allow.all --allow=allow.$branch \ - $sgr_args $flight >tmp/$flight.report + report_flight $flight ./cr-fold-long-lines tmp/$flight.report date >&2
@@ -144,11 +141,23 @@ start_email () { date >&2 } +report_flight () { + local flight=$1 + ./sg-report-flight --html-dir=$flight_html_dir/$flight/ \ + --allow=allow.all --allow=allow.$branch \ + $sgr_args $flight >tmp/$flight.report +} + +publish_logs () { + local flight=$1 + ./cr-publish-flight-logs ${OSSTEST_PUSH_HARNESS- --push-harness} \ + $flight >&2 +} + publish_send_email () { local flight=$1 + publish_logs $flight exec >&2 - ./cr-publish-flight-logs ${OSSTEST_PUSH_HARNESS- --push-harness} \ - $flight send_email tmp/$flight.email }
Signature · 1 line
2.20.1
PATCH 6/13 sg-report-flight: Break out printout_flightheader
In reply to Ian JacksonFrom: Ian Jackson <[email protected]>
No functional change.
--- sg-report-flight | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-)
sg-report-flight +11 -4
@@ -783,6 +783,14 @@ sub includes ($) { } } +sub printout_flightheader ($) { + my ($r) = @_; + bodyprint <<END; +flight $r->{Flight} $branch $r->{FlightInfo}{blessing} [$r->{FlightInfo}{intended}] +$c{ReportHtmlPubBaseUrl}/$r->{Flight}/ +END +} + sub printout { my ($r, @failures) = @_; $header_text = '';
@@ -793,10 +801,9 @@ sub printout { $r->{Flight}: $r->{OutcomeSummary} END includes(\@includebeginfiles); - bodyprint <<END; -flight $r->{Flight} $branch $r->{FlightInfo}{blessing} [$r->{FlightInfo}{intended}] -$c{ReportHtmlPubBaseUrl}/$r->{Flight}/ -END + + printout_flightheader($r); + if (defined $r->{Overall}) { bodyprint "\n"; bodyprint $r->{Overall};
Signature · 1 line
2.20.1
PATCH 7/13 sg-report-flight: Provide --refer-to-flight option
In reply to Ian JacksonFrom: Ian Jackson <[email protected]>
This just generates an extra heading and URL at the top of the output. In particular, it doesn't affect the algorithms which calculate regressions.
--- sg-report-flight | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)
sg-report-flight +18 -0
@@ -42,6 +42,7 @@ our $htmldir; our $want_info_headers; our ($branch, $branches_cond_q); our @allows; +our (@refer_to_flights); our (@includebeginfiles,@includefiles); open DEBUG, ">/dev/null";
@@ -66,6 +67,8 @@ while (@ARGV && $ARGV[0] =~ m/^-/) { push @includebeginfiles, $1; } elsif (m/^--include=(.*)$/) { push @includefiles, $1; + } elsif (m/^--refer-to-flight=(.*)$/) { + push @refer_to_flights, $1; } elsif (restrictflight_arg($_)) { # Handled by Executive } elsif (m/^--allow=(.*)$/) {
@@ -504,6 +507,16 @@ END die unless defined $specflight; } } +sub find_refer_to_flights () { + my $ffq = $dbh_tests->prepare("SELECT * FROM flights WHERE flight=?"); + @refer_to_flights = map { + my $flight = $_; + $ffq->execute($flight); + my $row = $ffq->fetchrow_hashref(); + die "refer to flight $flight not found\n" unless $row; + { Flight => $flight, FlightInfo => $row }; + } @refer_to_flights; +} sub examineflight ($) { my ($flight) = @_;
@@ -804,6 +817,10 @@ END printout_flightheader($r); + foreach my $ref_r (@refer_to_flights) { + printout_flightheader($ref_r); + } + if (defined $r->{Overall}) { bodyprint "\n"; bodyprint $r->{Overall};
@@ -1878,6 +1895,7 @@ db_retry($dbh_tests, [], sub { if (defined $mro) { open MRO, "> $mro.new" or die "$mro.new $!"; } + find_refer_to_flights(); findspecflight(); my $fi= examineflight($specflight); my @fails= justifyfailures($fi);
Signature · 1 line
2.20.1
PATCH 8/13 sg-report-flight: Nicer output for --refer-to-flight option
In reply to Ian JacksonSort the flight summary lines together, before the URLs. This makes it considerably easier to read.
--- sg-report-flight | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-)
sg-report-flight +9 -9
@@ -796,12 +796,17 @@ sub includes ($) { } } -sub printout_flightheader ($) { - my ($r) = @_; - bodyprint <<END; +sub printout_flightheaders { + foreach my $r (@_) { + bodyprint <<END; flight $r->{Flight} $branch $r->{FlightInfo}{blessing} [$r->{FlightInfo}{intended}] +END + } + foreach my $r (@_) { + bodyprint <<END; $c{ReportHtmlPubBaseUrl}/$r->{Flight}/ END + } } sub printout {
@@ -814,12 +819,7 @@ sub printout { $r->{Flight}: $r->{OutcomeSummary} END includes(\@includebeginfiles); - - printout_flightheader($r); - - foreach my $ref_r (@refer_to_flights) { - printout_flightheader($ref_r); - } + printout_flightheaders $r, @refer_to_flights; if (defined $r->{Overall}) { bodyprint "\n";
Signature · 1 line
2.20.1
PATCH 9/13 Introduce real-retry blessing
In reply to Ian JacksonFrom: Ian Jackson <[email protected]>
Nothing produces this yet. (There's play-retry as well of course but we don't need to document that really.)
--- README.dev | 9 +++++---- cr-daily-branch | 3 ++- cr-disk-report | 2 +- cr-try-bisect-adhoc | 2 +- cs-bisection-step | 4 ++-- sg-report-flight | 2 +- 6 files changed, 12 insertions(+), 10 deletions(-)
README.dev +5 -4
@@ -381,10 +381,11 @@ These are the principal (intended) blessings: commissioning, and that blessing removed and replaced with `real' when the hosts are ready. - * `real-bisect' and `adhoc-bisect': These are found only as the - blessing of finished flights. (This is achieved by passing - *-bisect to sg-execute-flight.) This allows the archaeologist - tools to distinguish full flights from bisection steps. + * `real-bisect', `real-retry', `adhoc-bisect': These are found only + as the blessing of finished flights. (This is achieved by passing + *-bisect or *-retry to sg-execute-flight.) This allows the + archaeologist tools to distinguish full flights from bisection + steps and retries. The corresponding intended blessing (as found in the `intended' column of the flights table) is `real'. So the hosts used by the
cr-daily-branch +2 -1
@@ -76,7 +76,8 @@ case $branch in treeurl=`./ap-print-url $branch`;; esac -blessings_arg=--blessings=${DAILY_BRANCH_TESTED_BLESSING:-real} +blessings_arg=${DAILY_BRANCH_TESTED_BLESSING:-real} +blessings_arg=--blessings=${blessings_arg},${blessings_arg}-retest sgr_args+=" $blessings_arg" force_baseline='' # Non-empty = indication why we are forcing baseline.
cr-disk-report +1 -1
@@ -38,7 +38,7 @@ our $graphs_px=0; our $graphs_py=0; open DEBUG, ">/dev/null" or die $!; -our @blessings = qw(real real-bisect); +our @blessings = qw(real real-retry real-bisect); # for these blessings column is "<blessing> <branch>" # for other blessings column is "<intended> [<blessing>]"
cr-try-bisect-adhoc +1 -1
@@ -49,7 +49,7 @@ export OSSTEST_BLESSING=adhoc compute_state_callback () { compute_state_core \ - --blessings=real,real-bisect,adhoc-bisect \ + --blessings=real,real-retry,real-bisect,adhoc-bisect \ $bisect "$@" $branch $job $testid }
cs-bisection-step +2 -2
@@ -7,7 +7,7 @@ # usage: # ./cs-bisection-setup [<options>] <branch> <job> <testid> # options, usually: -# --blessings=real,real-bisect +# --blessings=real,real-retry,real-bisect # # First entry in --blessings list is the blessing of the basis # (non-bisection) flights. This should not be the same as the
@@ -45,7 +45,7 @@ use HTML::Entities; use Osstest::Executive; use URI::Escape; -our @blessings= qw(real real-bisect); +our @blessings= qw(real real-retry real-bisect); our @revtuplegenargs= (); our $broken;
sg-report-flight +1 -1
@@ -120,7 +120,7 @@ die if defined $specver{this}{flight}; die if defined $specver{that}{flight} && grep { $_ ne 'flight' } keys %{ $specver{that} }; -push @blessings, 'real', 'real-bisect' unless @blessings; +push @blessings, 'real', 'real-retry', 'real-bisect' unless @blessings; csreadconfig();
Signature · 1 line
2.20.1
PATCH 10/13 cri-args-hostlists: Move flight_html_dir variable
In reply to Ian JacksonThis is only used in report_flight. We are going to want to call report_flight from outside start_email, without having to set that variable ourselves.
The variable isn't actually used in start_email.
--- cri-args-hostlists | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
cri-args-hostlists +1 -1
@@ -113,7 +113,6 @@ start_email () { printf '%s\n' "`getconfig EmailStdHeaders`" printf 'Subject: %s' "${subject_prefix:-[$branch test] }" - local flight_html_dir=$OSSTEST_HTMLPUB_DIR/ local job_html_dir=$OSSTEST_HTML_DIR/ local host_html_dir=$OSSTEST_HTML_DIR/host/
@@ -143,6 +142,7 @@ start_email () { report_flight () { local flight=$1 + local flight_html_dir=$OSSTEST_HTMLPUB_DIR/ ./sg-report-flight --html-dir=$flight_html_dir/$flight/ \ --allow=allow.all --allow=allow.$branch \ $sgr_args $flight >tmp/$flight.report
Signature · 1 line
2.20.1
PATCH 11/13 cr-daily-branch: Immediately retry failing tests
In reply to Ian JacksonFrom: Ian Jackson <[email protected]>
We exclude the self-tests because we don't want to miss breakage, and the Xen smoke tests because they will be run again RSN anyway.
--- cr-daily-branch | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-)
cr-daily-branch +47 -1
@@ -472,12 +472,58 @@ esac sgr_args+=" $EXTRA_SGR_ARGS" date >&2 +original_start=`date +%s` + : $flight $branch $OSSTEST_BLESSING $sgr_args eval "$DAILY_BRANCH_PREEXEC_HOOK" execute_flight $flight $OSSTEST_BLESSING date >&2 -start_email $flight $branch "$sgr_args" "$subject_prefix" +default_immediate_retry=$wantpush + +case "$branch" in +*smoke*) default_immediate_retry=false ;; +osstest) default_immediate_retry=false ;; +*) ;; +esac + +: ${OSSTEST_IMMEDIATE_RETRY:-$default_immediate_retry} + +while true; do + start_email $flight $branch "$sgr_args" "$subject_prefix" + if grep '^tolerable$' $mrof >/dev/null 2>&1; then break; fi + if ! $OSSTEST_IMMEDIATE_RETRY; then break; fi + OSSTEST_IMMEDIATE_RETRY=false + retry_jobs=$( + perl <$mrof -wne ' + next unless m/^regression (\S+) /; + my $j = $1; + next if $j =~ m/^build/; + $r{$j}++; + END { + print "copy-jobs '$flight' $_ " + foreach sort keys %r; + }' + ) + if [ "x$retry_jobs" = x ]; then break; fi + + rflight=$( + ./cs-adjust-flight new:$OSSTEST_BLESSING \ + branch-set $branch \ + $retry_jobs + ) + + ./mg-adjust-flight-makexrefs -v $rflight \ + --branch=$branch --revision-osstest=$narness_rev \ + '^build-*' --debug --blessings=real + + export OSSTEST_RESOURCE_WAITSTART=$original_start + execute_flight $rflight $OSSTEST_BLESSING-retest + report_flight $rflight + publish_logs $rflight + + sgr_args+=" --refer-to-flight=$rflight" +done push=false if grep '^tolerable$' $mrof >/dev/null 2>&1; then push=$wantpush; fi
Signature · 1 line
2.20.1
PATCH 12/13 Honour OSSTEST_SIMULATE_FAIL_RETRY for immediate retries
In reply to Ian JacksonThis is primarily useful for debugging the immediate-retry logic, but it seems churlish to delete it again.
--- cr-daily-branch | 4 ++++ 1 file changed, 4 insertions(+)
cr-daily-branch +4 -0
@@ -517,6 +517,10 @@ while true; do --branch=$branch --revision-osstest=$narness_rev \ '^build-*' --debug --blessings=real + if [ "${OSSTEST_SIMULATE_FAIL_RETRY+y}" = y ]; then + export OSSTEST_SIMULATE_FAIL="${OSSTEST_SIMULATE_FAIL_RETRY}" + fi + export OSSTEST_RESOURCE_WAITSTART=$original_start execute_flight $rflight $OSSTEST_BLESSING-retest report_flight $rflight
Signature · 1 line
2.20.1
PATCH 13/13 cr-daily-branch: Do not do immediate retry of failing xtf flights
In reply to Ian Jackson--- cr-daily-branch | 1 + 1 file changed, 1 insertion(+)
cr-daily-branch +1 -0
@@ -484,6 +484,7 @@ default_immediate_retry=$wantpush case "$branch" in *smoke*) default_immediate_retry=false ;; osstest) default_immediate_retry=false ;; +xtf*) default_immediate_retry=false ;; *) ;; esac
Signature · 1 line
2.20.1