delroth/hydra
0
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

* hydra_scheduler: use eval-jobs.

Browse source
This commit is contained in:
Eelco Dolstra 2009-03-09 13:04:46 +00:00
parent 875f57857e
commit 8c58448afc
2 changed files with 75 additions and 173 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/c/eval-jobs.cc
View file

@ -45,7 +45,6 @@ static void tryJobAlts(EvalState & state, XMLWriter & doc,
if (!matchFormal(ATgetFirst(formals), name, def2)) abort();
if ((values = (ATermList) argsLeft.get(name))) {
int n = 0;
for (ATermIterator i(ATreverse(values)); i; ++i, ++n) {
ATermMap actualArgs2(actualArgs);
@ -56,8 +55,8 @@ static void tryJobAlts(EvalState & state, XMLWriter & doc,
argsLeft2.remove(name);
tryJobAlts(state, doc, argsUsed2, argsLeft2, attrPath, fun, ATgetNext(formals), actualArgs2);
}
}
else
throw TypeError(format("job `%1%' requires an argument named `%2%'")
% attrPath % aterm2String(name));
@ -100,7 +99,8 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc,
XMLAttrs xmlAttrs;
Path outPath, drvPath;
xmlAttrs["name"] = attrPath;
xmlAttrs["jobName"] = attrPath;
xmlAttrs["nixName"] = drv.name;
xmlAttrs["system"] = drv.system;
xmlAttrs["drvPath"] = drv.queryDrvPath(state);
xmlAttrs["outPath"] = drv.queryOutPath(state);
@ -108,6 +108,7 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc,
xmlAttrs["longDescription"] = drv.queryMetaInfo(state, "longDescription");
xmlAttrs["license"] = drv.queryMetaInfo(state, "license");
xmlAttrs["homepage"] = drv.queryMetaInfo(state, "homepage");
xmlAttrs["schedulingPriority"] = drv.queryMetaInfo(state, "schedulingPriority");
XMLOpenElement _(doc, "job", xmlAttrs);
showArgsUsed(doc, argsUsed);
@ -175,6 +176,8 @@ void run(Strings args)
releaseExpr = arg;
}
if (releaseExpr == "") throw UsageError("no expression specified");
store = openStore();
Expr e = parseExprFromFile(state, releaseExpr);

239
src/script/hydra_scheduler.pl
View file

@ -1,6 +1,7 @@
#! /var/run/current-system/sw/bin/perl -w
use strict;
use feature 'switch';
use XML::Simple;
use Hydra::Schema;
use Hydra::Helper::Nix;
@ -32,8 +33,8 @@ sub getStorePathHash {
}
sub fetchInput {
my ($input, $alt, $inputInfo) = @_;
sub fetchInputAlt {
my ($input, $alt) = @_;
my $type = $input->type;
if ($type eq "path") {
@ -89,7 +90,7 @@ sub fetchInput {
}
}
$$inputInfo{$input->name} =
return
{ type => $type
, uri => $uri
, storePath => $storePath
@ -140,8 +141,8 @@ sub fetchInput {
});
});
}
$$inputInfo{$input->name} =
return
{ type => $type
, uri => $uri
, storePath => $storePath
@ -152,12 +153,12 @@ sub fetchInput {
elsif ($type eq "string") {
die unless defined $alt->value;
$$inputInfo{$input->name} = {type => $type, value => $alt->value};
return {type => $type, value => $alt->value};
}
elsif ($type eq "boolean") {
die unless defined $alt->value && ($alt->value eq "true" || $alt->value eq "false");
$$inputInfo{$input->name} = {type => $type, value => $alt->value};
return {type => $type, value => $alt->value};
}
else {
@ -166,41 +167,26 @@ sub fetchInput {
}
sub fetchInputs {
my ($jobset, $inputInfo) = @_;
foreach my $input ($jobset->jobsetinputs->all) {
foreach my $alt ($input->jobsetinputalts->all) {
push @{$$inputInfo{$input->name}}, fetchInputAlt($input, $alt);
}
}
}
sub checkJob {
my ($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs) = @_;
# Instantiate the store derivation.
(my $res, my $drvPath, my $stderr) = captureStdoutStderr(
"nix-instantiate", $nixExprPath, "--attr", $jobName, @{$extraArgs});
die "cannot evaluate the Nix expression for job `$jobName':\n$stderr" unless $res;
chomp $drvPath;
my ($project, $jobset, $inputInfo, $job) = @_;
# Call nix-env --xml to get info about this job (drvPath, outPath, meta attributes, ...).
($res, my $infoXml, $stderr) = captureStdoutStderr(
qw(nix-env --query --available * --attr-path --out-path --drv-path --meta --xml --system-filter *),
"-f", $nixExprPath, "--attr", $jobName, @{$extraArgs});
die "cannot get information about the job `$jobName':\n$stderr" unless $res;
my $info = XMLin($infoXml, ForceArray => 1, KeyAttr => ['attrPath', 'name'])
or die "cannot parse XML output";
my $job = $info->{item}->{$jobName};
die if !defined $job;
my $description = defined $job->{meta}->{description} ? $job->{meta}->{description}->{value} : "";
my $longDescription = defined $job->{meta}->{longDescription} ? $job->{meta}->{longDescription}->{value} : "";
my $license = defined $job->{meta}->{license} ? $job->{meta}->{license}->{value} : "";
my $homepage = defined $job->{meta}->{homepage} ? $job->{meta}->{homepage}->{value} : "";
die unless $job->{drvPath} eq $drvPath;
my $jobName = $job->{jobName};
my $drvPath = $job->{drvPath};
my $outPath = $job->{outPath};
my $priority = 100;
if (defined $job->{meta}->{schedulingPriority} &&
$job->{meta}->{schedulingPriority}->{value} =~ /^\d+$/)
{
$priority = int($job->{meta}->{schedulingPriority}->{value});
}
$priority = int($job->{schedulingPriority})
if $job->{schedulingPriority} =~ /^\d+$/;
$db->txn_do(sub {
if (scalar($db->resultset('Builds')->search(
@ -219,10 +205,10 @@ sub checkJob {
, project => $project->name
, jobset => $jobset->name
, attrname => $jobName
, description => $description
, longdescription => $longDescription
, license => $license
, nixname => $job->{name}
, description => $job->{description}
, longdescription => $job->{longDescription}
, license => $job->{license}
, nixname => $job->{nixName}
, drvpath => $drvPath
, outpath => $outPath
, system => $job->{system}
@ -235,11 +221,11 @@ sub checkJob {
, locker => ""
});
foreach my $inputName (keys %{$inputInfo}) {
my $input = $inputInfo->{$inputName};
foreach my $arg (@{$job->{arg}}) {
my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}] or die "invalid input";
$db->resultset('BuildInputs')->create(
{ build => $build->id
, name => $inputName
, name => $arg->{name}
, type => $input->{type}
, uri => $input->{uri}
, revision => $input->{revision}
@ -249,7 +235,7 @@ sub checkJob {
, sha256hash => $input->{sha256hash}
});
}
# !!! this should really by done by nix-instantiate to prevent a GC race.
registerRoot $drvPath;
});
@ -268,90 +254,31 @@ sub setJobsetError {
}
sub checkJobAlternatives {
my ($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs, $argsNeeded, $n) = @_;
sub inputsToArgs {
my ($inputInfo) = @_;
my @res = ();
if ($n >= scalar @{$argsNeeded}) {
eval {
checkJob($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs);
};
if ($@) {
print "error evaluating job `", $jobName, "': $@";
setJobsetError($jobset, $@);
}
return;
}
my $argName = @{$argsNeeded}[$n];
#print "finding alternatives for argument $argName\n";
my ($input) = $jobset->jobsetinputs->search({name => $argName});
my %newInputInfo = %{$inputInfo}; $inputInfo = \%newInputInfo; # clone
if (defined $input) {
foreach my $alt ($input->jobsetinputalts) {
#print "input ", $input->name, " (type ", $input->type, ") alt ", $alt->altnr, "\n";
fetchInput($input, $alt, $inputInfo);
my @newArgs = @{$extraArgs};
if (defined $inputInfo->{$argName}->{storePath}) {
push @newArgs, "--arg", $argName,
"{path = builtins.storePath " . $inputInfo->{$argName}->{storePath} . ";" .
" outPath = builtins.storePath " . $inputInfo->{$argName}->{storePath} . ";" .
" rev = \"" . $inputInfo->{$argName}->{revision} . "\";}";
} elsif ($inputInfo->{$argName}->{type} eq "string") {
push @newArgs, "--argstr", $argName, $inputInfo->{$argName}->{value};
} elsif ($inputInfo->{$argName}->{type} eq "boolean") {
push @newArgs, "--arg", $argName, $inputInfo->{$argName}->{value};
foreach my $input (keys %{$inputInfo}) {
foreach my $alt (@{$inputInfo->{$input}}) {
given ($alt->{type}) {
when ("string") {
push @res, "--argstr", $input, $alt->{value};
}
when ("boolean") {
push @res, "--arg", $input, $alt->{value};
}
when (["svn", "path"]) {
push @res, "--arg", $input, (
"{ outPath = builtins.storePath " . $alt->{storePath} . "" .
"; rev = \"" . $alt->{revision} . "\"" .
";}"
);
}
}
checkJobAlternatives(
$project, $jobset, $inputInfo, $nixExprPath,
$jobName, $jobExpr, \@newArgs, $argsNeeded, $n + 1);
}
}
else {
(my $prevBuild) = $db->resultset('Builds')->search(
{finished => 1, project => $project->name, jobset => $jobset->name, attrname => $argName, buildStatus => 0},
{join => 'resultInfo', order_by => "timestamp DESC", rows => 1});
if (!defined $prevBuild) {
# !!! reschedule?
die "missing input `$argName'";
}
# The argument name matches a previously built job in this
# jobset. Pick the most recent build. !!! refine the
# selection criteria: e.g., most recent successful build.
if (!isValidPath($prevBuild->outpath)) {
die "input path " . $prevBuild->outpath . " has been garbage-collected";
}
$$inputInfo{$argName} =
{ type => "build"
, storePath => $prevBuild->outpath
, id => $prevBuild->id
};
my $pkgNameRE = "(?:(?:[A-Za-z0-9]|(?:-[^0-9]))+)";
my $versionRE = "(?:[A-Za-z0-9\.\-]+)";
my $relName = ($prevBuild->resultInfo->releasename or $prevBuild->nixname);
my $version = $2 if $relName =~ /^($pkgNameRE)-($versionRE)$/;
my @newArgs = @{$extraArgs};
push @newArgs, "--arg", $argName,
"{ path = builtins.storePath " . $prevBuild->outpath . "; " .
" outPath = builtins.storePath " . $prevBuild->outpath . "; " .
($version ? " version = \"$version\"; " : "") . # !!! escape
"}";
checkJobAlternatives(
$project, $jobset, $inputInfo, $nixExprPath,
$jobName, $jobExpr, \@newArgs, $argsNeeded, $n + 1);
}
return @res;
}
@ -364,61 +291,33 @@ sub checkJobSet {
$jobset->update;
});
# Fetch the input containing the Nix expression.
(my $exprInput) = $jobset->jobsetinputs->search({name => $jobset->nixexprinput});
die "No input named " . $jobset->nixexprinput unless defined $exprInput;
# Fetch all values for all inputs.
fetchInputs($jobset, $inputInfo);
die "Multiple alternatives for the Nix expression input not supported yet"
if scalar($exprInput->jobsetinputalts) != 1;
fetchInput($exprInput, $exprInput->jobsetinputalts->first, $inputInfo);
# Evaluate the Nix expression.
my $nixExprPath = $inputInfo->{$jobset->nixexprinput}->{storePath} . "/" . $jobset->nixexprpath;
print "evaluating $nixExprPath\n";
# Evaluate the job expression.
my $nixExprPath = $inputInfo->{$jobset->nixexprinput}->[0]->{storePath}
or die "cannot find the input containing the job expression";
$nixExprPath .= "/" . $jobset->nixexprpath;
(my $res, my $jobsXml, my $stderr) = captureStdoutStderr(
"nix-instantiate", $nixExprPath, "--eval-only", "--strict", "--xml");
"eval-jobs", $nixExprPath, inputsToArgs($inputInfo));
die "cannot evaluate the Nix expression containing the jobs:\n$stderr" unless $res;
my $jobs = XMLin($jobsXml,
ForceArray => ['value', 'attr'],
KeyAttr => ['name'],
SuppressEmpty => '',
ValueAttr => [value => 'value'])
ForceArray => ['error', 'job', 'arg'],
KeyAttr => [],
SuppressEmpty => '')
or die "cannot parse XML output";
die unless defined $jobs->{attrs};
# Store the errors messages for jobs that failed to evaluate.
foreach my $error (@{$jobs->{error}}) {
print "error at " . $error->{location} . ": " . $error->{msg} . "\n";
}
# Iterate over the attributes listed in the Nix expression and
# perform the builds described by them. If an attribute is a
# function, then fill in the function arguments with the
# (alternative) values supplied in the jobsetinputs table.
foreach my $jobName (keys(%{$jobs->{attrs}->{attr}})) {
print "considering job $jobName\n";
my @argsNeeded = ();
my $jobExpr = $jobs->{attrs}->{attr}->{$jobName};
# !!! fix the case where there is only 1 attr, XML::Simple fucks up as usual
if (defined $jobExpr->{function}->{attrspat}) {
foreach my $argName (keys(%{$jobExpr->{function}->{attrspat}->{attr}})) {
#print "needs input $argName\n";
push @argsNeeded, $argName;
}
}
eval {
checkJobAlternatives(
$project, $jobset, {}, $nixExprPath,
$jobName, $jobExpr, [], \@argsNeeded, 0);
};
if ($@) {
print "error checking job ", $jobName, ": $@";
setJobsetError($jobset, $@);
}
# Schedule each successfully evaluated job.
foreach my $job (@{$jobs->{job}}) {
print "considering job " . $job->{jobName} . "\n";
checkJob($project, $jobset, $inputInfo, $job);
}
}