mg-repro-flight: Provide --rebuild to make variant build jobs too
cover mg-repro-flight: Provide --rebuild to make variant build jobs too
It is annoying that mg-repro-flight cannot run a build for you too. Fix this.
This is on xenbits in
https://xenbits.xen.org/git-http/people/iwj/osstest.git
xenbits.xen.org:/home/iwj/ext/osstest.git
etc. as the branch
wip.repro-flight-builds.v2
This version seems to actually work.
Ian Jackson (11):
mg-execute-flight: Save an mro in tmp/
cs-adjust-flight: Break out copy_jobs_*
cs-adjust-flight: Use db_prepare and honour -D -D for sql dumps
mg-repro-setup: Improve a doc message slightly
mg-repro-setup: Break out compute_adjusts
mg-repro-setup: Move logging setup to later
mg-repro-setup: Allow arguments to badusage
mg-repro-setup: Move flight creation up before task creation
mg-repro-flight: Provide --rebuild to make variant build jobs too
mg-repro-setup: Detect options (-...) in wrong place
mg-allocate: Don't try allocating the same resource for two specs
cs-adjust-flight | 149 ++++++++++++++++++++++++++++++++++++++++++++++++------ mg-allocate | 10 ++++ mg-execute-flight | 1 + mg-repro-setup | 127 +++++++++++++++++++++++++++++++++++++++------- 4 files changed, 253 insertions(+), 34 deletions(-)
Signature · 7 lines
2.11.0 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH 1/11 mg-execute-flight: Save an mro in tmp/
This may be useful for some things. For example, it will be used in just a moment.
--- mg-execute-flight | 1 + 1 file changed, 1 insertion(+)
mg-execute-flight +1 -0
@@ -81,6 +81,7 @@ echo >&5 "flight $flight reporting" ./sg-report-flight $basisflightopt \ --html-dir=$OSSTEST_HTMLPUB_DIR/$flight/ \ + --machine-readable-output=tmp/$flight.mro \ $flight >tmp/$flight.report subject=`sed -n 1p <tmp/$flight.report`
Signature · 7 lines
2.11.0 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH 2/11 cs-adjust-flight: Break out copy_jobs_*
In reply to Ian JacksonNo functional change.
--- cs-adjust-flight | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-)
cs-adjust-flight +14 -6
@@ -194,10 +194,8 @@ sub do_rm_job ($) { verbose "$dstflight.$job job deleted\n" if $count; } -sub copy_jobs ($$) { - my ($srcflight, $jobs) = @_; - prep_rm_jobs(); - my @job_copy_qs = map { $dbh_tests->prepare($_) } +sub copy_jobs_qs () { + map { $dbh_tests->prepare($_) } ("INSERT INTO jobs (flight, job, recipe, status)". " SELECT ?, job, recipe, 'queued'". " FROM jobs".
@@ -206,12 +204,22 @@ sub copy_jobs ($$) { " SELECT ?, job, name, val, 'f'". " FROM runvars". " WHERE flight = ? AND job = ? AND NOT synth"); +} +sub copy_jobs_do ($$) { + my ($qs, $srcflight, $srcjob) = @_; + $_->execute($dstflight, $srcflight, $job) foreach @$qs; + verbose "$dstflight.$job job created from $srcflight.$job\n"; +} + +sub copy_jobs ($$) { + my ($srcflight, $jobs) = @_; + prep_rm_jobs(); + my @job_copy_qs = copy_jobs_qs; for_jobs($dstflight, $jobs, \&do_rm_job, 1); for_jobs($srcflight, $jobs, \&do_rm_job); for_jobs($srcflight, $jobs, sub { my ($job) = @_; - $_->execute($dstflight, $srcflight, $job) foreach @job_copy_qs; - verbose "$dstflight.$job job created from $srcflight.$job\n"; + copy_jobs_do(\@job_copy_qs, $srcflight, $job); }); }
Signature · 7 lines
2.11.0 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH 3/11 cs-adjust-flight: Use db_prepare and honour -D -D for sql dumps
In reply to Ian JacksonThis makes debugging it easier. No functional change with zero or one occurrences of -D.
--- cs-adjust-flight | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-)
cs-adjust-flight +14 -12
@@ -75,6 +75,7 @@ csreadconfig(); our $debug = 0; open VERBOSE, ">/dev/null" or die $!; +open DEBUG, ">/dev/null" or die $!; our $verbose_buffer; while (@ARGV && $ARGV[0] =~ m/^\-/) {
@@ -83,6 +84,7 @@ while (@ARGV && $ARGV[0] =~ m/^\-/) { if (m/^-v$/) { open VERBOSE, ">&STDERR" or die $!; } elsif (m/^-D$/) { + open DEBUG, ">&STDERR" if $debug; $debug = 1; } else { die "$_ ?";
@@ -138,9 +140,9 @@ sub for_things ($$$$$$$) { my ($spec, $fn, $ifnone, $table, $keycol, $basecond, $basecondvals) = @_; - my $thing_q = $dbh_tests->prepare + my $thing_q = db_prepare ("SELECT * FROM $table WHERE $basecond AND $keycol = ?"); - my $things_q = $dbh_tests->prepare + my $things_q = db_prepare ("SELECT * FROM $table WHERE $basecond"); my ($specfn,$exact) = notspec_exfn("FOR_THINGS $table.$keycol",$spec);
@@ -182,7 +184,7 @@ our @job_rm_qs; sub prep_rm_jobs () { return if @job_rm_qs; - @job_rm_qs = map { $dbh_tests->prepare($_) } + @job_rm_qs = map { db_prepare($_) } ("DELETE FROM runvars WHERE flight = ? AND job = ?", "DELETE FROM jobs WHERE flight = ? AND job = ?"); }
@@ -195,7 +197,7 @@ sub do_rm_job ($) { } sub copy_jobs_qs () { - map { $dbh_tests->prepare($_) } + map { db_prepare($_) } ("INSERT INTO jobs (flight, job, recipe, status)". " SELECT ?, job, recipe, 'queued'". " FROM jobs".
@@ -205,8 +207,8 @@ sub copy_jobs_qs () { " FROM runvars". " WHERE flight = ? AND job = ? AND NOT synth"); } -sub copy_jobs_do ($$) { - my ($qs, $srcflight, $srcjob) = @_; +sub copy_jobs_do ($$$) { + my ($qs, $srcflight, $job) = @_; $_->execute($dstflight, $srcflight, $job) foreach @$qs; verbose "$dstflight.$job job created from $srcflight.$job\n"; }
@@ -253,9 +255,9 @@ sub change__copy_jobs { copy_jobs($srcflight, $jobs); } -our $runvar_rm_q = $dbh_tests->prepare +our $runvar_rm_q = db_prepare ("DELETE FROM runvars WHERE flight = ? AND job = ? AND name = ?"); -our $runvar_insert_q = $dbh_tests->prepare +our $runvar_insert_q = db_prepare ("INSERT INTO runvars (flight, job, name, val, synth)". " VALUES (?, ?, ?, ?, 'f')");
@@ -430,7 +432,7 @@ sub change__job_status { die unless @changes >= 1; my $jobs = shift @changes; - my $q = $dbh_tests->prepare(<<END); + my $q = db_prepare(<<END); SELECT status FROM jobs WHERE flight = ? AND job = ?
@@ -449,12 +451,12 @@ sub change__jobs_rename { my $jobs = shift @changes; my $perlop = shift @changes; - my $jobq = $dbh_tests->prepare(<<END); + my $jobq = db_prepare(<<END); UPDATE jobs SET job = ? WHERE flight = ? AND job = ? END - my $runvarq = $dbh_tests->prepare(<<END); + my $runvarq = db_prepare(<<END); UPDATE runvars SET job = ? WHERE flight = ? AND job = ?
@@ -472,7 +474,7 @@ sub changes () { if ($dstflight =~ m/^\d+$/) { my $steps_q = - $dbh_tests->prepare("SELECT * FROM steps WHERE flight = ?"); + db_prepare("SELECT * FROM steps WHERE flight = ?"); $steps_q->execute($dstflight); my $step = $steps_q->fetchrow_hashref(); if ($step) {
Signature · 7 lines
2.11.0 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH 4/11 mg-repro-setup: Improve a doc message slightly
In reply to Ian Jackson--- mg-repro-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
mg-repro-setup +1 -1
@@ -33,7 +33,7 @@ usage () { cat <<END (default <ident> is \`host') OPTIONs - -t<duration> estimated duration (default = 28d) + -t<duration> estimated overall allocation lifetime (default = 28d) --rogue bypass queuing system and allocate now --capture do not suppress log capture -r<var>=<value> set runvar
Signature · 7 lines
2.11.0 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH 5/11 mg-repro-setup: Break out compute_adjusts
In reply to Ian JacksonNo functional change.
--- mg-repro-setup | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-)
mg-repro-setup +11 -9
@@ -54,7 +54,6 @@ set -e -o posix mgexecflags=() adjustsets=() -adjusts=() allocs=() logfile=tmp/mg-repro-setup.log duration=28d
@@ -104,14 +103,17 @@ adjrunvar () { delrunvar "$1" adjusts+=(runvar-set "$job" "$1" "$2") } - -for arg in "${adjustsets[@]}"; do - case "$arg" in - !*|^*) delrunvar "${arg#?}" ;; - *=*) adjrunvar "${arg%%=*}" "${arg#*=}" ;; - *) bad-adjuistset-pattern ;; - esac -done +compute_adjusts () { + adjusts=() + for arg in "$@"; do + case "$arg" in + !*|^*) delrunvar "${arg#?}" ;; + *=*) adjrunvar "${arg%%=*}" "${arg#*=}" ;; + *) bad-adjuistset-pattern ;; + esac + done +} +compute_adjusts "${adjustsets[@]}" while [ $# -ne 0 ]; do arg=$1; shift
Signature · 7 lines
2.11.0 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH 6/11 mg-repro-setup: Move logging setup to later
In reply to Ian JacksonNo functional change.
--- mg-repro-setup | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-)
mg-repro-setup +7 -7
@@ -115,6 +115,13 @@ compute_adjusts () { } compute_adjusts "${adjustsets[@]}" +progressf () { printf >&2 "$@"; } +progress () { progressf "%s\n" "$1"; } + +progress "logging to $logfile" +savelog "$logfile" +exec 3>"$logfile" + while [ $# -ne 0 ]; do arg=$1; shift
@@ -160,13 +167,6 @@ while [ $# -ne 0 ]; do esac done -progressf () { printf >&2 "$@"; } -progress () { progressf "%s\n" "$1"; } - -progress "logging to $logfile" -savelog "$logfile" -exec 3>"$logfile" - OSSTEST_TASK=$(perl -e ' use Osstest; use Osstest::Executive;
Signature · 7 lines
2.11.0 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH 7/11 mg-repro-setup: Allow arguments to badusage
In reply to Ian JacksonNo functional change with existing call sites.
--- mg-repro-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
mg-repro-setup +1 -1
@@ -46,7 +46,7 @@ END } -badusage () { echo >&2 "bad usage"; usage >&2; exit 126; } +badusage () { echo >&2 "bad usage$*"; usage >&2; exit 126; } set -e -o posix
Signature · 7 lines
2.11.0 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH 8/11 mg-repro-setup: Move flight creation up before task creation
In reply to Ian JacksonNo significant functional change.
--- mg-repro-setup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
mg-repro-setup +3 -3
@@ -167,6 +167,9 @@ while [ $# -ne 0 ]; do esac done +flight=$(./cs-adjust-flight new:$blessing) +progress "new flight is $flight" + OSSTEST_TASK=$(perl -e ' use Osstest; use Osstest::Executive;
@@ -176,9 +179,6 @@ OSSTEST_TASK=$(perl -e ' ') export OSSTEST_TASK -flight=$(./cs-adjust-flight new:$blessing) -progress "new flight is $flight" - if [ "${alloc_idents[*]}" ]; then progress "allocating ${alloc_idents[*]} ..." alloc_output=tmp/$flight.allocations
Signature · 7 lines
2.11.0 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH 9/11 mg-repro-flight: Provide --rebuild to make variant build jobs too
In reply to Ian JacksonThis allows a single command to repro a particular job with a variety of different source code.
The implementation technique is:
- run the build job in a separate flight, so that it can run with a separate task which gives its host up after the build
- do much of the heavy lifting of runvar fiddling etc. in a new helper routine in cs-adjust-flight
---
v2: Add a missing `continue' (without which everything goes quite wrong)
--- cs-adjust-flight | 109 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ mg-repro-setup | 85 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 193 insertions(+), 1 deletion(-)
cs-adjust-flight +109 -0
@@ -469,6 +469,115 @@ END $runvarq->execute($newjob, $dstflight, $oldjob); }) } + +sub change__repro_buildjobs { + # helper for mg-repro-setup + # arguments to this change are + # EXAMPLE-FLIGHT EXAMPLE-JOB THING... + # where THING is + # .BUILDJOB =BUILDJOBRUNVAR +TREE=URL#REVISION + # (we eat all until it doesn't start with . = +) + # stdout output is + # list of runvar settings for repro job + die unless @changes >= 2; + my $eflight = shift @changes; + my $ejob = shift @changes; + my @refspecs; + my @varspecs; + my %treespecs; + while (@changes && $changes[0] =~ m/^[.=+]/) { + local $_ = shift @changes; + if (m/^[.=]/) { + push @refspecs, $_; + } elsif (m/^\+(.*?)\=(.*)\#(.*)$/) { + die $1 if $treespecs{$1}; + $treespecs{$1} = [$2,$3]; + } elsif (m/^-r/) { + push @varspecs, $_; + } else { + die "bad .BUILDJOB / =BUILDJOBRUNVAR / +TREE=URL#REVISION): $_\n"; + } + } + my $testq = db_prepare(<<END); +SELECT name, val FROM runvars WHERE flight=? AND job=? AND name like '%job'; +END + my $buildq_txt = <<END; +SELECT name FROM runvars WHERE flight=? AND job=? AND ('f' +END + my @buildq_xvars; + foreach my $t (sort keys %treespecs) { + $buildq_txt .= " OR name=?"; + push @buildq_xvars, "tree_$t"; + } + $buildq_txt .= ")"; + my $buildq = db_prepare($buildq_txt); + + my %bjobs; + + $testq->execute($eflight, $ejob); + while (my ($refvar, $bjobref) = $testq->fetchrow_array()) { + my ($bflight, $bjob) = flight_otherjob($eflight,$bjobref); + if (@refspecs) { + my $y = 0; + foreach my $rs (@refspecs) { + if ($rs eq ".$bjob" || $rs eq "=$refvar") { + $y = 1; + last; + } + } + next unless $y; + } + $buildq->execute($bflight,$bjob,@buildq_xvars); + my %got; + while (my ($treevar) = $buildq->fetchrow_array()) { + $treevar =~ m/^tree_/ or die "$treevar ?"; + $got{$'} = 1; + next unless $treespecs{$'}; + $treespecs{$'}[2]++; + } + next unless %got || @refspecs; + + $bjobs{$bjob} //= { Template => $bflight }; + $bjobs{$bjob}{Template} eq $bflight or + die "inconsistent $bjob: $bjobs{$bjob}{Template} != $bflight"; + push @{ $bjobs{$bjob}{Refs} }, $refvar; + $bjobs{$bjob}{Trees}{$_} = 1 foreach keys %got; + } + + foreach my $tree (sort keys %treespecs) { + die "unused tree/revision adjustment $tree" + .(@refspecs + ? " (no tree_$tree var in any of " + .(join ' ', map { "$bjobs{$_}{Template}.$_" } + sort keys %bjobs).")" + : '') + unless $treespecs{$tree}[2]; + } + + my @copy_jobs_qs = copy_jobs_qs(); + foreach my $bjob (sort keys %bjobs) { + copy_jobs_do(\@copy_jobs_qs, $bjobs{$bjob}{Template}, $bjob); + foreach my $tree (sort keys %{ $bjobs{$bjob}{Trees} }) { + runvar_set($bjob, "tree_$tree", $treespecs{$tree}[0]); + runvar_set($bjob, "revision_$tree", $treespecs{$tree}[1]); + } + foreach (@varspecs) { + if (m/^-r!|^-r^/) { + $runvar_rm_q->execute($dstflight, $bjob, $'); + } elsif (m/^-r(.*?)=/) { + runvar_set($bjob, $1, $'); + } else { + die "bad -r $_ ?"; + } + } + foreach my $refvar (@{ $bjobs{$bjob}{Refs} }) { + die "$bjob $refvar $& ?" if + "$bjob.$refvar" =~ m{[^+=_./,:0-9a-z-]}i; + print "runvar-set . $refvar $dstflight.$bjob\n" or die $!; + } + } +} + sub changes () { debug("CHANGES...\n");
mg-repro-setup +84 -1
113 lines not on this page · raw message
Signature · 7 lines
2.11.0 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH 10/11 mg-repro-setup: Detect options (-...) in wrong place
In reply to Ian JacksonWithout this they tend to be interpreted as HOSTSPECs leading to lossage.
---
v2: New patch
--- mg-repro-setup | 4 ++++ 1 file changed, 4 insertions(+)
mg-repro-setup +4 -0
@@ -197,6 +197,10 @@ while [ $# -ne 0 ]; do continue ;; + -*) + badusage ': misplaced option, expected REBUILD or HOSTSPEC' + ;; + none:) # provided so we can repro a job with no hosts ;;
Signature · 7 lines
2.11.0 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel
PATCH 11/11 mg-allocate: Don't try allocating the same resource for two specs
In reply to Ian JacksonThis is obviously wrong. In fact it does not work (we bomb out in the allocation).
---
v2: New patch
--- mg-allocate | 10 ++++++++++ 1 file changed, 10 insertions(+)
mg-allocate +10 -0
@@ -437,6 +437,16 @@ sub plan () { map { [ @$possreqs, $_ ] } @reqlist; } @possmatrix; } + @possmatrix = grep { + my $possreqs = $_; + my %got; + my $ok=1; + foreach (@$possreqs) { + next if !$got{$_->{Reso}}++; + $ok=0; last; + } + $ok; + } @possmatrix; my $planned; my @reqlist;
Signature · 7 lines
2.11.0 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xenproject.org/mailman/listinfo/xen-devel