* hydra_scheduler: use eval-jobs.

src/c/eval-jobs.cc

@@ -45,7 +45,6 @@ static void tryJobAlts(EvalState & state, XMLWriter & doc,
     if (!matchFormal(ATgetFirst(formals), name, def2)) abort();
     
     if ((values = (ATermList) argsLeft.get(name))) {
-
         int n = 0;
         for (ATermIterator i(ATreverse(values)); i; ++i, ++n) {
             ATermMap actualArgs2(actualArgs);
@@ -56,8 +55,8 @@ static void tryJobAlts(EvalState & state, XMLWriter & doc,
             argsLeft2.remove(name);
             tryJobAlts(state, doc, argsUsed2, argsLeft2, attrPath, fun, ATgetNext(formals), actualArgs2);
         }
-        
     }
+    
     else
         throw TypeError(format("job `%1%' requires an argument named `%2%'")
             % attrPath % aterm2String(name));
@@ -100,7 +99,8 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc,
             XMLAttrs xmlAttrs;
             Path outPath, drvPath;
 
-            xmlAttrs["name"] = attrPath;
+            xmlAttrs["jobName"] = attrPath;
+            xmlAttrs["nixName"] = drv.name;
             xmlAttrs["system"] = drv.system;
             xmlAttrs["drvPath"] = drv.queryDrvPath(state);
             xmlAttrs["outPath"] = drv.queryOutPath(state);
@@ -108,6 +108,7 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc,
             xmlAttrs["longDescription"] = drv.queryMetaInfo(state, "longDescription");
             xmlAttrs["license"] = drv.queryMetaInfo(state, "license");
             xmlAttrs["homepage"] = drv.queryMetaInfo(state, "homepage");
+            xmlAttrs["schedulingPriority"] = drv.queryMetaInfo(state, "schedulingPriority");
         
             XMLOpenElement _(doc, "job", xmlAttrs);
             showArgsUsed(doc, argsUsed);
@@ -175,6 +176,8 @@ void run(Strings args)
             releaseExpr = arg;
     }
 
+    if (releaseExpr == "") throw UsageError("no expression specified");
+    
     store = openStore();
 
     Expr e = parseExprFromFile(state, releaseExpr);

src/script/hydra_scheduler.pl

@@ -1,6 +1,7 @@
 #! /var/run/current-system/sw/bin/perl -w
 
 use strict;
+use feature 'switch';
 use XML::Simple;
 use Hydra::Schema;
 use Hydra::Helper::Nix;
@@ -32,8 +33,8 @@ sub getStorePathHash {
 }
 
 
-sub fetchInput {
+sub fetchInputAlt {
-    my ($input, $alt, $inputInfo) = @_;
+    my ($input, $alt) = @_;
     my $type = $input->type;
 
     if ($type eq "path") {
@@ -89,7 +90,7 @@ sub fetchInput {
             }
         }
 
-        $$inputInfo{$input->name} =
+        return
             { type => $type
             , uri => $uri
             , storePath => $storePath
@@ -141,7 +142,7 @@ sub fetchInput {
             });
         }
 
-        $$inputInfo{$input->name} =
+        return 
             { type => $type
             , uri => $uri
             , storePath => $storePath
@@ -152,12 +153,12 @@ sub fetchInput {
     
     elsif ($type eq "string") {
         die unless defined $alt->value;
-        $$inputInfo{$input->name} = {type => $type, value => $alt->value};
+        return {type => $type, value => $alt->value};
     }
     
     elsif ($type eq "boolean") {
         die unless defined $alt->value && ($alt->value eq "true" || $alt->value eq "false");
-        $$inputInfo{$input->name} = {type => $type, value => $alt->value};
+        return {type => $type, value => $alt->value};
     }
     
     else {
@@ -166,41 +167,26 @@ sub fetchInput {
 }
 
 
+sub fetchInputs {
+    my ($jobset, $inputInfo) = @_;
+    foreach my $input ($jobset->jobsetinputs->all) {
+        foreach my $alt ($input->jobsetinputalts->all) {
+            push @{$$inputInfo{$input->name}}, fetchInputAlt($input, $alt);
+        }
+    }
+}
+
+
 sub checkJob {
-    my ($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs) = @_;
+    my ($project, $jobset, $inputInfo, $job) = @_;
 
-    # Instantiate the store derivation.
+    my $jobName = $job->{jobName};
-    (my $res, my $drvPath, my $stderr) = captureStdoutStderr(
+    my $drvPath = $job->{drvPath};
-        "nix-instantiate", $nixExprPath, "--attr", $jobName, @{$extraArgs});
-    die "cannot evaluate the Nix expression for job `$jobName':\n$stderr" unless $res;
-    chomp $drvPath;
-
-    # Call nix-env --xml to get info about this job (drvPath, outPath, meta attributes, ...). 
-    ($res, my $infoXml, $stderr) = captureStdoutStderr(
-        qw(nix-env --query --available * --attr-path --out-path --drv-path --meta --xml --system-filter *),
-        "-f", $nixExprPath, "--attr", $jobName, @{$extraArgs});
-    die "cannot get information about the job `$jobName':\n$stderr" unless $res;
-
-    my $info = XMLin($infoXml, ForceArray => 1, KeyAttr => ['attrPath', 'name'])
-        or die "cannot parse XML output";
-
-    my $job = $info->{item}->{$jobName};
-    die if !defined $job;
-        
-    my $description = defined $job->{meta}->{description} ? $job->{meta}->{description}->{value} : "";
-    my $longDescription = defined $job->{meta}->{longDescription} ? $job->{meta}->{longDescription}->{value} : "";
-    my $license = defined $job->{meta}->{license} ? $job->{meta}->{license}->{value} : "";
-    my $homepage = defined $job->{meta}->{homepage} ? $job->{meta}->{homepage}->{value} : "";
-    
-    die unless $job->{drvPath} eq $drvPath;
     my $outPath = $job->{outPath};
 
     my $priority = 100;
-    if (defined $job->{meta}->{schedulingPriority} &&
+    $priority = int($job->{schedulingPriority})
-        $job->{meta}->{schedulingPriority}->{value} =~ /^\d+$/)
+        if $job->{schedulingPriority} =~ /^\d+$/;
-    {
-        $priority = int($job->{meta}->{schedulingPriority}->{value});
-    }
 
     $db->txn_do(sub {
         if (scalar($db->resultset('Builds')->search(
@@ -219,10 +205,10 @@ sub checkJob {
             , project => $project->name
             , jobset => $jobset->name
             , attrname => $jobName
-            , description => $description
+            , description => $job->{description}
-            , longdescription => $longDescription
+            , longdescription => $job->{longDescription}
-            , license => $license
+            , license => $job->{license}
-            , nixname => $job->{name}
+            , nixname => $job->{nixName}
             , drvpath => $drvPath
             , outpath => $outPath
             , system => $job->{system}
@@ -235,11 +221,11 @@ sub checkJob {
             , locker => ""
             });
 
-        foreach my $inputName (keys %{$inputInfo}) {
+        foreach my $arg (@{$job->{arg}}) {
-            my $input = $inputInfo->{$inputName};
+            my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}] or die "invalid input";
             $db->resultset('BuildInputs')->create(
                 { build => $build->id
-                , name => $inputName
+                , name => $arg->{name}
                 , type => $input->{type}
                 , uri => $input->{uri}
                 , revision => $input->{revision}
@@ -268,90 +254,31 @@ sub setJobsetError {
 }
 
 
-sub checkJobAlternatives {
+sub inputsToArgs {
-    my ($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs, $argsNeeded, $n) = @_;
+    my ($inputInfo) = @_;
+    my @res = ();
 
-    if ($n >= scalar @{$argsNeeded}) {
+    foreach my $input (keys %{$inputInfo}) {
-        eval {
+        foreach my $alt (@{$inputInfo->{$input}}) {
-            checkJob($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs);
+            given ($alt->{type}) {
-        };
+                when ("string") {
-        if ($@) {
+                    push @res, "--argstr", $input, $alt->{value};
-            print "error evaluating job `", $jobName, "': $@";
-            setJobsetError($jobset, $@);
                 }
-        return;
+                when ("boolean") {
+                    push @res, "--arg", $input, $alt->{value};
+                }
+                when (["svn", "path"]) {
+                    push @res, "--arg", $input, (
+                        "{ outPath = builtins.storePath " . $alt->{storePath} . "" .
+                        "; rev = \"" . $alt->{revision} . "\"" .
+                        ";}"
+                    );
                 }
-
-    my $argName = @{$argsNeeded}[$n];
-    #print "finding alternatives for argument $argName\n";
-
-    my ($input) = $jobset->jobsetinputs->search({name => $argName});
-
-    my %newInputInfo = %{$inputInfo}; $inputInfo = \%newInputInfo; # clone
-
-    if (defined $input) {
-        
-        foreach my $alt ($input->jobsetinputalts) {
-            #print "input ", $input->name, " (type ", $input->type, ") alt ", $alt->altnr, "\n";
-            fetchInput($input, $alt, $inputInfo);
-            my @newArgs = @{$extraArgs};
-            if (defined $inputInfo->{$argName}->{storePath}) {
-                push @newArgs, "--arg", $argName,
-                    "{path = builtins.storePath " . $inputInfo->{$argName}->{storePath} . ";" .
-                    " outPath = builtins.storePath " . $inputInfo->{$argName}->{storePath} . ";" .
-                    " rev = \"" . $inputInfo->{$argName}->{revision} . "\";}";
-            } elsif ($inputInfo->{$argName}->{type} eq "string") {
-                push @newArgs, "--argstr", $argName, $inputInfo->{$argName}->{value};
-            } elsif ($inputInfo->{$argName}->{type} eq "boolean") {
-                push @newArgs, "--arg", $argName, $inputInfo->{$argName}->{value};
             }
-            checkJobAlternatives(
-                $project, $jobset, $inputInfo, $nixExprPath,
-                $jobName, $jobExpr, \@newArgs, $argsNeeded, $n + 1);
         }
     }
 
-    else {
+    return @res;
-
-        (my $prevBuild) = $db->resultset('Builds')->search(
-            {finished => 1, project => $project->name, jobset => $jobset->name, attrname => $argName, buildStatus => 0},
-            {join => 'resultInfo', order_by => "timestamp DESC", rows => 1});
-
-        if (!defined $prevBuild) {
-            # !!! reschedule?
-            die "missing input `$argName'";
-        }
-
-        # The argument name matches a previously built job in this
-        # jobset.  Pick the most recent build.  !!! refine the
-        # selection criteria: e.g., most recent successful build.
-        if (!isValidPath($prevBuild->outpath)) {
-            die "input path " . $prevBuild->outpath . " has been garbage-collected";
-        }
-                    
-        $$inputInfo{$argName} =
-            { type => "build"
-            , storePath => $prevBuild->outpath
-            , id => $prevBuild->id
-            };
-
-        my $pkgNameRE = "(?:(?:[A-Za-z0-9]|(?:-[^0-9]))+)";
-        my $versionRE = "(?:[A-Za-z0-9\.\-]+)";
-
-        my $relName = ($prevBuild->resultInfo->releasename or $prevBuild->nixname);
-        my $version = $2 if $relName =~ /^($pkgNameRE)-($versionRE)$/;
-
-        my @newArgs = @{$extraArgs};
-        push @newArgs, "--arg", $argName,
-            "{ path = builtins.storePath " . $prevBuild->outpath . "; " .
-            "  outPath = builtins.storePath " . $prevBuild->outpath . "; " .
-            ($version ? "  version = \"$version\"; " : "") . # !!! escape
-            "}";
-        
-        checkJobAlternatives(
-            $project, $jobset, $inputInfo, $nixExprPath,
-            $jobName, $jobExpr, \@newArgs, $argsNeeded, $n + 1);
-    }
 }
 
 
@@ -364,61 +291,33 @@ sub checkJobSet {
         $jobset->update;
     });
 
-    # Fetch the input containing the Nix expression.
+    # Fetch all values for all inputs.
-    (my $exprInput) = $jobset->jobsetinputs->search({name => $jobset->nixexprinput});
+    fetchInputs($jobset, $inputInfo);
-    die "No input named " . $jobset->nixexprinput unless defined $exprInput;
 
-    die "Multiple alternatives for the Nix expression input not supported yet"
+    # Evaluate the job expression.
-        if scalar($exprInput->jobsetinputalts) != 1;
+    my $nixExprPath = $inputInfo->{$jobset->nixexprinput}->[0]->{storePath}
-
+        or die "cannot find the input containing the job expression";
-    fetchInput($exprInput, $exprInput->jobsetinputalts->first, $inputInfo);
+    $nixExprPath .= "/" . $jobset->nixexprpath;
-
-    # Evaluate the Nix expression.
-    my $nixExprPath = $inputInfo->{$jobset->nixexprinput}->{storePath} . "/" . $jobset->nixexprpath;
-
-    print "evaluating $nixExprPath\n";
 
     (my $res, my $jobsXml, my $stderr) = captureStdoutStderr(
-        "nix-instantiate", $nixExprPath, "--eval-only", "--strict", "--xml");
+        "eval-jobs", $nixExprPath, inputsToArgs($inputInfo));
     die "cannot evaluate the Nix expression containing the jobs:\n$stderr" unless $res;
 
     my $jobs = XMLin($jobsXml,
-                     ForceArray => ['value', 'attr'],
+                     ForceArray => ['error', 'job', 'arg'],
-                     KeyAttr => ['name'],
+                     KeyAttr => [],
-                     SuppressEmpty => '',
+                     SuppressEmpty => '')
-                     ValueAttr => [value => 'value'])
         or die "cannot parse XML output";
 
-    die unless defined $jobs->{attrs};
+    # Store the errors messages for jobs that failed to evaluate.
-
+    foreach my $error (@{$jobs->{error}}) {
-    # Iterate over the attributes listed in the Nix expression and
+        print "error at " . $error->{location} . ": " . $error->{msg} . "\n";
-    # perform the builds described by them.  If an attribute is a
-    # function, then fill in the function arguments with the
-    # (alternative) values supplied in the jobsetinputs table.
-    foreach my $jobName (keys(%{$jobs->{attrs}->{attr}})) {
-        print "considering job $jobName\n";
-
-        my @argsNeeded = ();
-        
-        my $jobExpr = $jobs->{attrs}->{attr}->{$jobName};
-
-        # !!! fix the case where there is only 1 attr, XML::Simple fucks up as usual
-        if (defined $jobExpr->{function}->{attrspat}) {
-            foreach my $argName (keys(%{$jobExpr->{function}->{attrspat}->{attr}})) {
-                #print "needs input $argName\n";
-                push @argsNeeded, $argName;
-            }            
     }
 
-        eval {
+    # Schedule each successfully evaluated job.
-            checkJobAlternatives(
+    foreach my $job (@{$jobs->{job}}) {
-                $project, $jobset, {}, $nixExprPath,
+        print "considering job " . $job->{jobName} . "\n";
-                $jobName, $jobExpr, [], \@argsNeeded, 0);
+        checkJob($project, $jobset, $inputInfo, $job);
-        };
-        if ($@) {
-            print "error checking job ", $jobName, ": $@";
-            setJobsetError($jobset, $@);
-        }
     }
 }