ma27/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

* hydra_scheduler: use eval-jobs.

Browse source
This commit is contained in:
Eelco Dolstra 2009-03-09 13:04:46 +00:00
parent 875f57857e
commit 8c58448afc
2 changed files with 75 additions and 173 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/c/eval-jobs.cc
View file

@ -45,7 +45,6 @@ static void tryJobAlts(EvalState & state, XMLWriter & doc,
if (!matchFormal(ATgetFirst(formals), name, def2)) abort(); if (!matchFormal(ATgetFirst(formals), name, def2)) abort();
if ((values = (ATermList) argsLeft.get(name))) { if ((values = (ATermList) argsLeft.get(name))) {
int n = 0; int n = 0;
for (ATermIterator i(ATreverse(values)); i; ++i, ++n) { for (ATermIterator i(ATreverse(values)); i; ++i, ++n) {
ATermMap actualArgs2(actualArgs); ATermMap actualArgs2(actualArgs);
@ -56,8 +55,8 @@ static void tryJobAlts(EvalState & state, XMLWriter & doc,
argsLeft2.remove(name); argsLeft2.remove(name);
tryJobAlts(state, doc, argsUsed2, argsLeft2, attrPath, fun, ATgetNext(formals), actualArgs2); tryJobAlts(state, doc, argsUsed2, argsLeft2, attrPath, fun, ATgetNext(formals), actualArgs2);
} }
} }
else else
throw TypeError(format("job `%1%' requires an argument named `%2%'") throw TypeError(format("job `%1%' requires an argument named `%2%'")
% attrPath % aterm2String(name)); % attrPath % aterm2String(name));
@ -100,7 +99,8 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc,
XMLAttrs xmlAttrs; XMLAttrs xmlAttrs;
Path outPath, drvPath; Path outPath, drvPath;
xmlAttrs["name"] = attrPath; xmlAttrs["jobName"] = attrPath;
xmlAttrs["nixName"] = drv.name;
xmlAttrs["system"] = drv.system; xmlAttrs["system"] = drv.system;
xmlAttrs["drvPath"] = drv.queryDrvPath(state); xmlAttrs["drvPath"] = drv.queryDrvPath(state);
xmlAttrs["outPath"] = drv.queryOutPath(state); xmlAttrs["outPath"] = drv.queryOutPath(state);
@ -108,6 +108,7 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc,
xmlAttrs["longDescription"] = drv.queryMetaInfo(state, "longDescription"); xmlAttrs["longDescription"] = drv.queryMetaInfo(state, "longDescription");
xmlAttrs["license"] = drv.queryMetaInfo(state, "license"); xmlAttrs["license"] = drv.queryMetaInfo(state, "license");
xmlAttrs["homepage"] = drv.queryMetaInfo(state, "homepage"); xmlAttrs["homepage"] = drv.queryMetaInfo(state, "homepage");
xmlAttrs["schedulingPriority"] = drv.queryMetaInfo(state, "schedulingPriority");
XMLOpenElement _(doc, "job", xmlAttrs); XMLOpenElement _(doc, "job", xmlAttrs);
showArgsUsed(doc, argsUsed); showArgsUsed(doc, argsUsed);
@ -175,6 +176,8 @@ void run(Strings args)
releaseExpr = arg; releaseExpr = arg;
} }
if (releaseExpr == "") throw UsageError("no expression specified");
store = openStore(); store = openStore();
Expr e = parseExprFromFile(state, releaseExpr); Expr e = parseExprFromFile(state, releaseExpr);

239
src/script/hydra_scheduler.pl
View file

@ -1,6 +1,7 @@
#! /var/run/current-system/sw/bin/perl -w #! /var/run/current-system/sw/bin/perl -w
use strict; use strict;
use feature 'switch';
use XML::Simple; use XML::Simple;
use Hydra::Schema; use Hydra::Schema;
use Hydra::Helper::Nix; use Hydra::Helper::Nix;
@ -32,8 +33,8 @@ sub getStorePathHash {
} }
sub fetchInput { sub fetchInputAlt {
my ($input, $alt, $inputInfo) = @_; my ($input, $alt) = @_;
my $type = $input->type; my $type = $input->type;
if ($type eq "path") { if ($type eq "path") {
@ -89,7 +90,7 @@ sub fetchInput {
} }
} }
$$inputInfo{$input->name} = return
{ type => $type { type => $type
, uri => $uri , uri => $uri
, storePath => $storePath , storePath => $storePath
@ -140,8 +141,8 @@ sub fetchInput {
}); });
}); });
} }
$$inputInfo{$input->name} = return
{ type => $type { type => $type
, uri => $uri , uri => $uri
, storePath => $storePath , storePath => $storePath
@ -152,12 +153,12 @@ sub fetchInput {
elsif ($type eq "string") { elsif ($type eq "string") {
die unless defined $alt->value; die unless defined $alt->value;
$$inputInfo{$input->name} = {type => $type, value => $alt->value}; return {type => $type, value => $alt->value};
} }
elsif ($type eq "boolean") { elsif ($type eq "boolean") {
die unless defined $alt->value && ($alt->value eq "true" || $alt->value eq "false"); die unless defined $alt->value && ($alt->value eq "true" || $alt->value eq "false");
$$inputInfo{$input->name} = {type => $type, value => $alt->value}; return {type => $type, value => $alt->value};
} }
else { else {
@ -166,41 +167,26 @@ sub fetchInput {
} }
sub fetchInputs {
my ($jobset, $inputInfo) = @_;
foreach my $input ($jobset->jobsetinputs->all) {
foreach my $alt ($input->jobsetinputalts->all) {
push @{$$inputInfo{$input->name}}, fetchInputAlt($input, $alt);
}
}
}
sub checkJob { sub checkJob {
my ($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs) = @_; my ($project, $jobset, $inputInfo, $job) = @_;
# Instantiate the store derivation.
(my $res, my $drvPath, my $stderr) = captureStdoutStderr(
"nix-instantiate", $nixExprPath, "--attr", $jobName, @{$extraArgs});
die "cannot evaluate the Nix expression for job `$jobName':\n$stderr" unless $res;
chomp $drvPath;
# Call nix-env --xml to get info about this job (drvPath, outPath, meta attributes, ...). my $jobName = $job->{jobName};
($res, my $infoXml, $stderr) = captureStdoutStderr( my $drvPath = $job->{drvPath};
qw(nix-env --query --available * --attr-path --out-path --drv-path --meta --xml --system-filter *),
"-f", $nixExprPath, "--attr", $jobName, @{$extraArgs});
die "cannot get information about the job `$jobName':\n$stderr" unless $res;
my $info = XMLin($infoXml, ForceArray => 1, KeyAttr => ['attrPath', 'name'])
or die "cannot parse XML output";
my $job = $info->{item}->{$jobName};
die if !defined $job;
my $description = defined $job->{meta}->{description} ? $job->{meta}->{description}->{value} : "";
my $longDescription = defined $job->{meta}->{longDescription} ? $job->{meta}->{longDescription}->{value} : "";
my $license = defined $job->{meta}->{license} ? $job->{meta}->{license}->{value} : "";
my $homepage = defined $job->{meta}->{homepage} ? $job->{meta}->{homepage}->{value} : "";
die unless $job->{drvPath} eq $drvPath;
my $outPath = $job->{outPath}; my $outPath = $job->{outPath};
my $priority = 100; my $priority = 100;
if (defined $job->{meta}->{schedulingPriority} && $priority = int($job->{schedulingPriority})
$job->{meta}->{schedulingPriority}->{value} =~ /^\d+$/) if $job->{schedulingPriority} =~ /^\d+$/;
{
$priority = int($job->{meta}->{schedulingPriority}->{value});
}
$db->txn_do(sub { $db->txn_do(sub {
if (scalar($db->resultset('Builds')->search( if (scalar($db->resultset('Builds')->search(
@ -219,10 +205,10 @@ sub checkJob {
, project => $project->name , project => $project->name
, jobset => $jobset->name , jobset => $jobset->name
, attrname => $jobName , attrname => $jobName
, description => $description , description => $job->{description}
, longdescription => $longDescription , longdescription => $job->{longDescription}
, license => $license , license => $job->{license}
, nixname => $job->{name} , nixname => $job->{nixName}
, drvpath => $drvPath , drvpath => $drvPath
, outpath => $outPath , outpath => $outPath
, system => $job->{system} , system => $job->{system}
@ -235,11 +221,11 @@ sub checkJob {
, locker => "" , locker => ""
}); });
foreach my $inputName (keys %{$inputInfo}) { foreach my $arg (@{$job->{arg}}) {
my $input = $inputInfo->{$inputName}; my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}] or die "invalid input";
$db->resultset('BuildInputs')->create( $db->resultset('BuildInputs')->create(
{ build => $build->id { build => $build->id
, name => $inputName , name => $arg->{name}
, type => $input->{type} , type => $input->{type}
, uri => $input->{uri} , uri => $input->{uri}
, revision => $input->{revision} , revision => $input->{revision}
@ -249,7 +235,7 @@ sub checkJob {
, sha256hash => $input->{sha256hash} , sha256hash => $input->{sha256hash}
}); });
} }
# !!! this should really by done by nix-instantiate to prevent a GC race. # !!! this should really by done by nix-instantiate to prevent a GC race.
registerRoot $drvPath; registerRoot $drvPath;
}); });
@ -268,90 +254,31 @@ sub setJobsetError {
} }
sub checkJobAlternatives { sub inputsToArgs {
my ($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs, $argsNeeded, $n) = @_; my ($inputInfo) = @_;
my @res = ();
if ($n >= scalar @{$argsNeeded}) { foreach my $input (keys %{$inputInfo}) {
eval { foreach my $alt (@{$inputInfo->{$input}}) {
checkJob($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs); given ($alt->{type}) {
}; when ("string") {
if ($@) { push @res, "--argstr", $input, $alt->{value};
print "error evaluating job `", $jobName, "': $@"; }
setJobsetError($jobset, $@); when ("boolean") {
} push @res, "--arg", $input, $alt->{value};
return; }
} when (["svn", "path"]) {
push @res, "--arg", $input, (
my $argName = @{$argsNeeded}[$n]; "{ outPath = builtins.storePath " . $alt->{storePath} . "" .
#print "finding alternatives for argument $argName\n"; "; rev = \"" . $alt->{revision} . "\"" .
";}"
my ($input) = $jobset->jobsetinputs->search({name => $argName}); );
}
my %newInputInfo = %{$inputInfo}; $inputInfo = \%newInputInfo; # clone
if (defined $input) {
foreach my $alt ($input->jobsetinputalts) {
#print "input ", $input->name, " (type ", $input->type, ") alt ", $alt->altnr, "\n";
fetchInput($input, $alt, $inputInfo);
my @newArgs = @{$extraArgs};
if (defined $inputInfo->{$argName}->{storePath}) {
push @newArgs, "--arg", $argName,
"{path = builtins.storePath " . $inputInfo->{$argName}->{storePath} . ";" .
" outPath = builtins.storePath " . $inputInfo->{$argName}->{storePath} . ";" .
" rev = \"" . $inputInfo->{$argName}->{revision} . "\";}";
} elsif ($inputInfo->{$argName}->{type} eq "string") {
push @newArgs, "--argstr", $argName, $inputInfo->{$argName}->{value};
} elsif ($inputInfo->{$argName}->{type} eq "boolean") {
push @newArgs, "--arg", $argName, $inputInfo->{$argName}->{value};
} }
checkJobAlternatives(
$project, $jobset, $inputInfo, $nixExprPath,
$jobName, $jobExpr, \@newArgs, $argsNeeded, $n + 1);
} }
} }
else { return @res;
(my $prevBuild) = $db->resultset('Builds')->search(
{finished => 1, project => $project->name, jobset => $jobset->name, attrname => $argName, buildStatus => 0},
{join => 'resultInfo', order_by => "timestamp DESC", rows => 1});
if (!defined $prevBuild) {
# !!! reschedule?
die "missing input `$argName'";
}
# The argument name matches a previously built job in this
# jobset. Pick the most recent build. !!! refine the
# selection criteria: e.g., most recent successful build.
if (!isValidPath($prevBuild->outpath)) {
die "input path " . $prevBuild->outpath . " has been garbage-collected";
}
$$inputInfo{$argName} =
{ type => "build"
, storePath => $prevBuild->outpath
, id => $prevBuild->id
};
my $pkgNameRE = "(?:(?:[A-Za-z0-9]|(?:-[^0-9]))+)";
my $versionRE = "(?:[A-Za-z0-9\.\-]+)";
my $relName = ($prevBuild->resultInfo->releasename or $prevBuild->nixname);
my $version = $2 if $relName =~ /^($pkgNameRE)-($versionRE)$/;
my @newArgs = @{$extraArgs};
push @newArgs, "--arg", $argName,
"{ path = builtins.storePath " . $prevBuild->outpath . "; " .
" outPath = builtins.storePath " . $prevBuild->outpath . "; " .
($version ? " version = \"$version\"; " : "") . # !!! escape
"}";
checkJobAlternatives(
$project, $jobset, $inputInfo, $nixExprPath,
$jobName, $jobExpr, \@newArgs, $argsNeeded, $n + 1);
}
} }
@ -364,61 +291,33 @@ sub checkJobSet {
$jobset->update; $jobset->update;
}); });
# Fetch the input containing the Nix expression. # Fetch all values for all inputs.
(my $exprInput) = $jobset->jobsetinputs->search({name => $jobset->nixexprinput}); fetchInputs($jobset, $inputInfo);
die "No input named " . $jobset->nixexprinput unless defined $exprInput;
die "Multiple alternatives for the Nix expression input not supported yet" # Evaluate the job expression.
if scalar($exprInput->jobsetinputalts) != 1; my $nixExprPath = $inputInfo->{$jobset->nixexprinput}->[0]->{storePath}
or die "cannot find the input containing the job expression";
fetchInput($exprInput, $exprInput->jobsetinputalts->first, $inputInfo); $nixExprPath .= "/" . $jobset->nixexprpath;
# Evaluate the Nix expression.
my $nixExprPath = $inputInfo->{$jobset->nixexprinput}->{storePath} . "/" . $jobset->nixexprpath;
print "evaluating $nixExprPath\n";
(my $res, my $jobsXml, my $stderr) = captureStdoutStderr( (my $res, my $jobsXml, my $stderr) = captureStdoutStderr(
"nix-instantiate", $nixExprPath, "--eval-only", "--strict", "--xml"); "eval-jobs", $nixExprPath, inputsToArgs($inputInfo));
die "cannot evaluate the Nix expression containing the jobs:\n$stderr" unless $res; die "cannot evaluate the Nix expression containing the jobs:\n$stderr" unless $res;
my $jobs = XMLin($jobsXml, my $jobs = XMLin($jobsXml,
ForceArray => ['value', 'attr'], ForceArray => ['error', 'job', 'arg'],
KeyAttr => ['name'], KeyAttr => [],
SuppressEmpty => '', SuppressEmpty => '')
ValueAttr => [value => 'value'])
or die "cannot parse XML output"; or die "cannot parse XML output";
die unless defined $jobs->{attrs}; # Store the errors messages for jobs that failed to evaluate.
foreach my $error (@{$jobs->{error}}) {
print "error at " . $error->{location} . ": " . $error->{msg} . "\n";
}
# Iterate over the attributes listed in the Nix expression and # Schedule each successfully evaluated job.
# perform the builds described by them. If an attribute is a foreach my $job (@{$jobs->{job}}) {
# function, then fill in the function arguments with the print "considering job " . $job->{jobName} . "\n";
# (alternative) values supplied in the jobsetinputs table. checkJob($project, $jobset, $inputInfo, $job);
foreach my $jobName (keys(%{$jobs->{attrs}->{attr}})) {
print "considering job $jobName\n";
my @argsNeeded = ();
my $jobExpr = $jobs->{attrs}->{attr}->{$jobName};
# !!! fix the case where there is only 1 attr, XML::Simple fucks up as usual
if (defined $jobExpr->{function}->{attrspat}) {
foreach my $argName (keys(%{$jobExpr->{function}->{attrspat}->{attr}})) {
#print "needs input $argName\n";
push @argsNeeded, $argName;
}
}
eval {
checkJobAlternatives(
$project, $jobset, {}, $nixExprPath,
$jobName, $jobExpr, [], \@argsNeeded, 0);
};
if ($@) {
print "error checking job ", $jobName, ": $@";
setJobsetError($jobset, $@);
}
} }
} }