* Log evaluation errors etc. in the DB.

This commit is contained in:
Eelco Dolstra 2008-11-25 13:27:57 +00:00
parent 6efe0c37f3
commit a8f748e547
21 changed files with 122 additions and 72 deletions

View file

@ -56,8 +56,9 @@ sub index :Path :Args(0) {
# Get the latest finished build for each unique job.
$c->stash->{latestBuilds} = [$c->model('DB::Builds')->search(undef,
{ join => 'resultInfo'
, where => "finished != 0 and timestamp = (select max(timestamp) from Builds where project == me.project and attrName == me.attrName and finished != 0)"
, order_by => "project, attrname"
, where => "finished != 0 and timestamp = (select max(timestamp) from Builds " .
"where project == me.project and attrName == me.attrName and finished != 0 and system == me.system)"
, order_by => "project, attrname, system"
})];
}

View file

@ -8,8 +8,8 @@ use base 'DBIx::Class::Schema';
__PACKAGE__->load_classes;
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:8AQc7Q1I5DrtSjFp722iMA
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:iYwU4nHj9C2HkHwpYlVygQ
# You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -36,8 +36,8 @@ __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
__PACKAGE__->belongs_to("dependency", "Hydra::Schema::Builds", { id => "dependency" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:NSUTUwG2qbvbFkZxezeSEA
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:H8JAWhQt1ehZTEEPH2KGBw
# You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -33,8 +33,8 @@ __PACKAGE__->set_primary_key("build", "productnr");
__PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9b3ZNy73Cjat0gr5nQHIpA
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:EAo0rZSn+5Q5OY7Y1xjuyA
# You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -27,8 +27,8 @@ __PACKAGE__->set_primary_key("id");
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:uTiPHxrs6S19Laa/qeb7CA
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:5R03Kdy5ujkr7s9OfCNbTA
# You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -70,8 +70,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Ky0EF25kaZIaY+ofkP0ohw
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:G1KBKGvEgnTcrgDkUnG1UQ
__PACKAGE__->has_many(dependents => 'Hydra::Schema::Buildinputs', 'dependency');

View file

@ -23,8 +23,8 @@ __PACKAGE__->set_primary_key("id");
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:kt5gBt14ay7/DiIaQpKopA
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:MkMR6pE6mrN2BQL18ROYfA
# You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -35,8 +35,8 @@ __PACKAGE__->set_primary_key("id", "stepnr");
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ljGW5oOaJQ/uUKQx80W5SA
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:JbawGso3rboFeQMX6XfgXQ
# You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -31,8 +31,8 @@ __PACKAGE__->belongs_to(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:mG0ka8XxC3ZEs/KgGKe5Hg
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:7+oJOQegjLhp5JeuCS0KPw
# You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -43,8 +43,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:trcxVA3lLfgTC0TCHL3mVw
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:BCzeQkhJYieHoL4ppjdJvw
# You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -18,6 +18,10 @@ __PACKAGE__->add_columns(
{ data_type => "text", is_nullable => 0, size => undef },
"nixexprpath",
{ data_type => "text", is_nullable => 0, size => undef },
"errormsg",
{ data_type => "text", is_nullable => 0, size => undef },
"errortime",
{ data_type => "integer", is_nullable => 0, size => undef },
);
__PACKAGE__->set_primary_key("project", "name");
__PACKAGE__->has_many(
@ -44,8 +48,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:MYl8lWfWLCIAGSulR3m5zw
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:kEcdBc9XKVhz6zO/XEkAxg
# You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -30,8 +30,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:tLlYQAkX5CeUR2GJir8ifg
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:rRYzLbd9BlZAQdOjT/GQcw
# You can replace this text with custom content, and it will be preserved on regeneration

View file

@ -3,6 +3,7 @@
use strict;
use XML::Simple;
use Hydra::Schema;
use IPC::Run;
my $db = Hydra::Schema->connect("dbi:SQLite:dbname=hydra.sqlite", "", "", {});
@ -16,6 +17,13 @@ sub isValidPath {
}
sub captureStdoutStderr {
my $stdin = ""; my $stdout; my $stderr;
my $res = IPC::Run::run(\@_, \$stdin, \$stdout, \$stderr);
return ($res, $stdout, $stderr);
}
sub getStorePathHash {
my ($storePath) = @_;
my $hash = `nix-store --query --hash $storePath`
@ -36,10 +44,11 @@ sub fetchInput {
if ($type eq "path") {
my $uri = $alt->value;
print "copying input ", $input->name, " from $uri\n";
my $storePath = `nix-store --add "$uri"`
or die "cannot copy path $uri to the Nix store";
chomp $storePath;
print " copied to $storePath\n";
$$inputInfo{$input->name} =
{ type => $type
@ -64,13 +73,16 @@ sub checkJob {
my ($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs) = @_;
# Instantiate the store derivation.
my $drvPath = `nix-instantiate $nixExprPath --attr $jobName $extraArgs`
or die "cannot evaluate the Nix expression containing the job definitions: $?";
(my $res, my $drvPath, my $stderr) = captureStdoutStderr(
"nix-instantiate", $nixExprPath, "--attr", $jobName, @{$extraArgs});
die "cannot evaluate the Nix expression for job `$jobName':\n$stderr" unless $res;
chomp $drvPath;
# Call nix-env --xml to get info about this job (drvPath, outPath, meta attributes, ...).
my $infoXml = `nix-env -f $nixExprPath --query --available "*" --attr-path --out-path --drv-path --meta --xml --system-filter "*" --attr $jobName $extraArgs`
or die "cannot get information about the job: $?";
# Call nix-env --xml to get info about this job (drvPath, outPath, meta attributes, ...).
($res, my $infoXml, $stderr) = captureStdoutStderr(
qw(nix-env --query --available * --attr-path --out-path --drv-path --meta --xml --system-filter *),
"-f", $nixExprPath, "--attr", $jobName, @{$extraArgs});
die "cannot get information about the job `$jobName':\n$stderr" unless $res;
my $info = XMLin($infoXml, ForceArray => 1, KeyAttr => ['attrPath', 'name'])
or die "cannot parse XML output";
@ -87,11 +99,11 @@ sub checkJob {
{ project => $project->name, jobset => $jobset->name
, attrname => $jobName, outPath => $outPath })) > 0)
{
print " already scheduled/done\n";
print "already scheduled/done\n";
return;
}
print " adding to queue\n";
print "adding to queue\n";
my $build = $db->resultset('Builds')->create(
{ finished => 0
@ -132,6 +144,18 @@ sub checkJob {
};
sub setJobsetError {
my ($jobset, $errorMsg) = @_;
eval {
$db->txn_do(sub {
$jobset->errormsg($errorMsg);
$jobset->errortime(time);
$jobset->update;
});
};
}
sub checkJobAlternatives {
my ($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs, $argsNeeded, $n) = @_;
@ -139,12 +163,15 @@ sub checkJobAlternatives {
eval {
checkJob($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs);
};
warn $@ if $@;
if ($@) {
print "error evaluating job `", $jobName, "': $@";
setJobsetError($jobset, $@);
}
return;
}
my $argName = @{$argsNeeded}[$n];
print " finding alternatives for argument $argName\n";
#print "finding alternatives for argument $argName\n";
my ($input) = $jobset->jobsetinputs->search({name => $argName});
@ -153,18 +180,17 @@ sub checkJobAlternatives {
if (defined $input) {
foreach my $alt ($input->jobsetinputalts) {
print " INPUT ", $input->name, " (type ", $input->type, ") alt ", $alt->altnr, "\n";
#print "input ", $input->name, " (type ", $input->type, ") alt ", $alt->altnr, "\n";
fetchInput($input, $alt, $inputInfo); # !!! caching
my $newArgs = "";
my @newArgs = @{$extraArgs};
if (defined $inputInfo->{$argName}->{storePath}) {
# !!! escaping
$newArgs = " --arg $argName '{path = " . $inputInfo->{$argName}->{storePath} . ";}'";
push @newArgs, "--arg", $argName, "{path = " . $inputInfo->{$argName}->{storePath} . ";}";
} elsif (defined $inputInfo->{$argName}->{value}) {
$newArgs = " --argstr $argName '" . $inputInfo->{$argName}->{value} . "'";
push @newArgs, "--argstr", $argName, $inputInfo->{$argName}->{value};
}
checkJobAlternatives(
$project, $jobset, $inputInfo, $nixExprPath,
$jobName, $jobExpr, $extraArgs . $newArgs, $argsNeeded, $n + 1);
$jobName, $jobExpr, \@newArgs, $argsNeeded, $n + 1);
}
}
@ -192,15 +218,16 @@ sub checkJobAlternatives {
, id => $prevBuild->id
};
my @newArgs = @{$extraArgs};
push @newArgs, "--arg", $argName, "{path = " . $prevBuild->outpath . ";}";
checkJobAlternatives(
$project, $jobset, $inputInfo, $nixExprPath,
$jobName, $jobExpr,
$extraArgs . " --arg $argName '{path = " . $prevBuild->outpath . ";}'",
$argsNeeded, $n + 1);
$jobName, $jobExpr, \@newArgs, $argsNeeded, $n + 1);
}
}
sub checkJobSet {
my ($project, $jobset) = @_;
my $inputInfo = {};
@ -216,10 +243,11 @@ sub checkJobSet {
# Evaluate the Nix expression.
my $nixExprPath = $inputInfo->{$jobset->nixexprinput}->{storePath} . "/" . $jobset->nixexprpath;
print " EVALUATING $nixExprPath\n";
my $jobsXml = `nix-instantiate $nixExprPath --eval-only --strict --xml`
or die "cannot evaluate the Nix expression containing the jobs: $?";
print "evaluating $nixExprPath\n";
(my $res, my $jobsXml, my $stderr) = captureStdoutStderr(
"nix-instantiate", $nixExprPath, "--eval-only", "--strict", "--xml");
die "cannot evaluate the Nix expression containing the jobs:\n$stderr" unless $res;
my $jobs = XMLin($jobsXml,
ForceArray => ['value', 'attr'],
@ -235,7 +263,7 @@ sub checkJobSet {
# function, then fill in the function arguments with the
# (alternative) values supplied in the jobsetinputs table.
foreach my $jobName (keys(%{$jobs->{attrs}->{attr}})) {
print " JOB $jobName\n";
print "considering job $jobName\n";
my @argsNeeded = ();
@ -244,7 +272,7 @@ sub checkJobSet {
# !!! fix the case where there is only 1 attr, XML::Simple fucks up as usual
if (defined $jobExpr->{function}->{attrspat}) {
foreach my $argName (keys(%{$jobExpr->{function}->{attrspat}->{attr}})) {
print " needs input $argName\n";
#print "needs input $argName\n";
push @argsNeeded, $argName;
}
}
@ -252,7 +280,7 @@ sub checkJobSet {
eval {
checkJobAlternatives(
$project, $jobset, {}, $nixExprPath,
$jobName, $jobExpr, "", \@argsNeeded, 0);
$jobName, $jobExpr, [], \@argsNeeded, 0);
};
warn $@ if $@;
}
@ -262,14 +290,17 @@ sub checkJobSet {
sub checkJobs {
foreach my $project ($db->resultset('Projects')->search({enabled => 1})) {
print "PROJECT ", $project->name, "\n";
print "considering project ", $project->name, "\n";
foreach my $jobset ($project->jobsets->all) {
print " JOBSET ", $jobset->name, "\n";
print "considering jobset ", $jobset->name, " in ", $project->name, "\n";
eval {
checkJobSet($project, $jobset);
};
warn $@ if $@;
}
};
if ($@) {
print "error evaluating jobset ", $jobset->name, ": $@";
setProjectError($jobset, $@);
}
}
}
}

View file

@ -1,7 +1,6 @@
[% WRAPPER layout.tt title="Hydra Overview" %]
[% PROCESS common.tt %]
[% USE HTML %]
[% USE date %]
[% USE mibs=format("%.2f") %]
<h1>
@ -25,7 +24,7 @@
</tr>
<tr>
<th>Time added:</th>
<td>[% date.format(build.timestamp, '%Y-%m-%d %H:%M:%S') %]</td>
<td>[% PROCESS renderDateTime timestamp = build.timestamp %]</td>
</tr>
<tr>
<th>Status:</th>
@ -87,11 +86,11 @@
[% IF build.finished %]
<tr>
<th>Build started:</th>
<td>[% IF build.resultInfo.starttime %][% date.format(build.resultInfo.starttime, '%Y-%m-%d %H:%M:%S') %][% ELSE %]<em>(cached build)</em>[% END %]</td>
<td>[% IF build.resultInfo.starttime %][% PROCESS renderDateTime timestamp = build.resultInfo.starttime %][% ELSE %]<em>(cached build)</em>[% END %]</td>
</tr>
<tr>
<th>Build finished:</th>
<td>[% IF build.resultInfo.stoptime %][% date.format(build.resultInfo.stoptime, '%Y-%m-%d %H:%M:%S') %][% ELSE %]<em>(cached build)</em>[% END %]</td>
<td>[% IF build.resultInfo.stoptime %][% PROCESS renderDateTime timestamp = build.resultInfo.stoptime %][% ELSE %]<em>(cached build)</em>[% END %]</td>
</tr>
<tr>
<th>Duration (seconds):</th>
@ -348,7 +347,7 @@
<td><a href="[% c.uri_for('/build' input.build.id) %]">Job <tt>[% input.build.project.name %]:[% input.build.attrname %]</tt> build [% input.build.id %]</a></td>
<td><tt>[% input.name %]</tt></td>
<td><tt>[% input.build.system %]</tt></td>
<td>[% date.format(input.build.timestamp, '%Y-%m-%d %H:%M:%S') %]</td>
<td>[% PROCESS renderDateTime timestamp = input.build.timestamp %]</td>
</tr>
[% END -%]
</tbody>

View file

@ -7,3 +7,9 @@
, "build" = "Build output"
}
%]
[% USE date %]
[% BLOCK renderDateTime %]
[% date.format(timestamp, '%Y-%m-%d %H:%M:%S') -%]
[% END %]

View file

@ -67,13 +67,4 @@
</table>
<h2>Projects</h2>
<ul>
[% FOREACH project IN projects -%]
<li><a href="[% c.uri_for('/project' project.name) %]"><tt>[% project.name %]</tt></a></li>
[% END -%]
</ul>
[% END %]

View file

@ -119,4 +119,4 @@
</div>
</body>
</html>
</html>

View file

@ -88,6 +88,15 @@
<tt>[% INCLUDE maybeEditString param="jobset-$baseName-nixexprinput" value=jobset.nixexprinput extraClass="shortString" %]</tt>
</td>
</tr>
[% IF !edit && jobset.errormsg %]
<tr>
<th>Last evaluation error:</th>
<td>
<em>On [% PROCESS renderDateTime timestamp = jobset.errortime %]:</em>
<pre class="multiLineMsg error">[% HTML.escape(jobset.errormsg) %]</pre>
</td>
</tr>
[% END %]
</table>
<h4>Inputs</h4>

View file

@ -44,6 +44,7 @@ td {
}
th {
vertical-align: top;
background: #ffffc0;
}
@ -169,6 +170,12 @@ div.buildlog {
padding: 0.3em;
}
.multiLineMsg {
padding: 0em;
margin: 0em;
white-space: pre-wrap;
}
ul.productList {
list-style: none;
padding-left: 1em;

View file

@ -86,4 +86,4 @@ em.storeref:hover span.popup {
code {
white-space: pre-wrap;
}
}

View file

@ -170,6 +170,8 @@ create table Jobsets (
description text,
nixExprInput text not null, -- name of the jobsetInput containing the Nix expression
nixExprPath text not null, -- relative path of the Nix expression
errorMsg text, -- used to signal the last evaluation error etc. for this jobset
errorTime integer, -- timestamp associated with errorMsg
primary key (project, name),
foreign key (project) references Projects(name) on delete cascade, -- ignored by sqlite
foreign key (project, name, nixExprInput) references JobsetInputs(project, job, name)