forked from lix-project/hydra
hydra-eval-jobs: Use JSON instead of XML
XML::Simple is pretty slow - reading the output for the Nixpkgs jobset takes half a minute or so. JSON is pretty much instantaneous.
This commit is contained in:
Eelco Dolstra
parent
d9a5143fcb
commit
09a96c642a
|
|
@ -8,7 +8,7 @@
|
|||
#include "eval.hh"
|
||||
#include "eval-inline.hh"
|
||||
#include "util.hh"
|
||||
#include "xml-writer.hh"
|
||||
#include "value-to-json.hh"
|
||||
#include "get-drvs.hh"
|
||||
#include "common-opts.hh"
|
||||
#include "globals.hh"
|
||||
|
|
@ -23,11 +23,11 @@ typedef std::list<Value *, traceable_allocator<Value *> > ValueList;
|
|||
typedef std::map<Symbol, ValueList> AutoArgs;
|
||||
|
||||
|
||||
static void findJobs(EvalState & state, XMLWriter & doc,
|
||||
static void findJobs(EvalState & state, JSONObject & top,
|
||||
const AutoArgs & argsLeft, Value & v, const string & attrPath);
|
||||
|
||||
|
||||
static void tryJobAlts(EvalState & state, XMLWriter & doc,
|
||||
static void tryJobAlts(EvalState & state, JSONObject & top,
|
||||
const AutoArgs & argsLeft, const string & attrPath, Value & fun,
|
||||
Formals::Formals_::iterator cur,
|
||||
Formals::Formals_::iterator last,
|
||||
|
|
@ -38,7 +38,7 @@ static void tryJobAlts(EvalState & state, XMLWriter & doc,
|
|||
state.mkAttrs(*arg, 0);
|
||||
arg->attrs = &actualArgs;
|
||||
mkApp(v, fun, *arg);
|
||||
findJobs(state, doc, argsLeft, v, attrPath);
|
||||
findJobs(state, top, argsLeft, v, attrPath);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -50,7 +50,7 @@ static void tryJobAlts(EvalState & state, XMLWriter & doc,
|
|||
if (!cur->def)
|
||||
throw TypeError(format("job `%1%' requires an argument named `%2%'")
|
||||
% attrPath % cur->name);
|
||||
tryJobAlts(state, doc, argsLeft, attrPath, fun, next, last, actualArgs);
|
||||
tryJobAlts(state, top, argsLeft, attrPath, fun, next, last, actualArgs);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -63,7 +63,7 @@ static void tryJobAlts(EvalState & state, XMLWriter & doc,
|
|||
actualArgs2.push_back(Attr(cur->name, *i));
|
||||
actualArgs2.sort(); // !!! inefficient
|
||||
argsLeft2.erase(cur->name);
|
||||
tryJobAlts(state, doc, argsLeft2, attrPath, fun, next, last, actualArgs2);
|
||||
tryJobAlts(state, top, argsLeft2, attrPath, fun, next, last, actualArgs2);
|
||||
++n;
|
||||
}
|
||||
}
|
||||
|
|
@ -93,7 +93,7 @@ static string queryMetaStrings(EvalState & state, DrvInfo & drv, const string &
|
|||
}
|
||||
|
||||
|
||||
static void findJobsWrapped(EvalState & state, XMLWriter & doc,
|
||||
static void findJobsWrapped(EvalState & state, JSONObject & top,
|
||||
const AutoArgs & argsLeft, Value & v, const string & attrPath)
|
||||
{
|
||||
debug(format("at path `%1%'") % attrPath);
|
||||
|
|
@ -107,7 +107,6 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc,
|
|||
DrvInfo drv(state);
|
||||
|
||||
if (getDerivation(state, v, drv, false)) {
|
||||
XMLAttrs xmlAttrs;
|
||||
Path drvPath;
|
||||
|
||||
DrvInfo::Outputs outputs = drv.queryOutputs();
|
||||
|
|
@ -115,21 +114,19 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc,
|
|||
if (drv.system == "unknown")
|
||||
throw EvalError("derivation must have a ‘system’ attribute");
|
||||
|
||||
xmlAttrs["jobName"] = attrPath;
|
||||
xmlAttrs["nixName"] = drv.name;
|
||||
xmlAttrs["system"] = drv.system;
|
||||
xmlAttrs["drvPath"] = drvPath = drv.queryDrvPath();
|
||||
xmlAttrs["description"] = drv.queryMetaString("description");
|
||||
xmlAttrs["longDescription"] = drv.queryMetaString("longDescription");
|
||||
xmlAttrs["license"] = queryMetaStrings(state, drv, "license");
|
||||
xmlAttrs["homepage"] = drv.queryMetaString("homepage");
|
||||
xmlAttrs["maintainers"] = queryMetaStrings(state, drv, "maintainers");
|
||||
|
||||
xmlAttrs["schedulingPriority"] = int2String(drv.queryMetaInt("schedulingPriority", 100));
|
||||
|
||||
xmlAttrs["timeout"] = int2String(drv.queryMetaInt("timeout", 36000));
|
||||
|
||||
xmlAttrs["maxSilent"] = int2String(drv.queryMetaInt("maxSilent", 7200));
|
||||
top.attr(attrPath);
|
||||
JSONObject res(top.str);
|
||||
res.attr("nixName", drv.name);
|
||||
res.attr("system", drv.system);
|
||||
res.attr("drvPath", drvPath = drv.queryDrvPath());
|
||||
res.attr("description", drv.queryMetaString("description"));
|
||||
res.attr("longDescription", drv.queryMetaString("longDescription"));
|
||||
res.attr("license", queryMetaStrings(state, drv, "license"));
|
||||
res.attr("homepage", drv.queryMetaString("homepage"));
|
||||
res.attr("maintainers", queryMetaStrings(state, drv, "maintainers"));
|
||||
res.attr("schedulingPriority", drv.queryMetaInt("schedulingPriority", 100));
|
||||
res.attr("timeout", drv.queryMetaInt("timeout", 36000));
|
||||
res.attr("maxSilent", drv.queryMetaInt("maxSilent", 7200));
|
||||
|
||||
/* If this is an aggregate, then get its constituents. */
|
||||
Bindings::iterator a = v.attrs->find(state.symbols.create("_hydraAggregate"));
|
||||
|
|
@ -145,7 +142,7 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc,
|
|||
size_t index = i->find("!", 1);
|
||||
drvs.insert(string(*i, index + 1));
|
||||
}
|
||||
xmlAttrs["constituents"] = concatStringsSep(" ", drvs);
|
||||
res.attr("constituents", concatStringsSep(" ", drvs));
|
||||
}
|
||||
|
||||
/* Register the derivation as a GC root. !!! This
|
||||
|
|
@ -156,20 +153,16 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc,
|
|||
if (!pathExists(root)) addPermRoot(*store, drvPath, root, false);
|
||||
}
|
||||
|
||||
XMLOpenElement _(doc, "job", xmlAttrs);
|
||||
|
||||
foreach (DrvInfo::Outputs::iterator, j, outputs) {
|
||||
XMLAttrs attrs2;
|
||||
attrs2["name"] = j->first;
|
||||
attrs2["path"] = j->second;
|
||||
doc.writeEmptyElement("output", attrs2);
|
||||
}
|
||||
res.attr("outputs");
|
||||
JSONObject res2(res.str);
|
||||
for (auto & j : outputs)
|
||||
res2.attr(j.first, j.second);
|
||||
}
|
||||
|
||||
else {
|
||||
if (!state.isDerivation(v)) {
|
||||
foreach (Bindings::iterator, i, *v.attrs)
|
||||
findJobs(state, doc, argsLeft, *i->value,
|
||||
findJobs(state, top, argsLeft, *i->value,
|
||||
(attrPath.empty() ? "" : attrPath + ".") + (string) i->name);
|
||||
}
|
||||
}
|
||||
|
|
@ -177,7 +170,7 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc,
|
|||
|
||||
else if (v.type == tLambda && v.lambda.fun->matchAttrs) {
|
||||
Bindings & tmp(*state.allocBindings(0));
|
||||
tryJobAlts(state, doc, argsLeft, attrPath, v,
|
||||
tryJobAlts(state, top, argsLeft, attrPath, v,
|
||||
v.lambda.fun->formals->formals.begin(),
|
||||
v.lambda.fun->formals->formals.end(),
|
||||
tmp);
|
||||
|
|
@ -192,16 +185,16 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc,
|
|||
}
|
||||
|
||||
|
||||
static void findJobs(EvalState & state, XMLWriter & doc,
|
||||
static void findJobs(EvalState & state, JSONObject & top,
|
||||
const AutoArgs & argsLeft, Value & v, const string & attrPath)
|
||||
{
|
||||
try {
|
||||
findJobsWrapped(state, doc, argsLeft, v, attrPath);
|
||||
findJobsWrapped(state, top, argsLeft, v, attrPath);
|
||||
} catch (EvalError & e) {
|
||||
XMLAttrs xmlAttrs;
|
||||
xmlAttrs["location"] = attrPath;
|
||||
xmlAttrs["msg"] = e.msg();
|
||||
XMLOpenElement _(doc, "error", xmlAttrs);
|
||||
top.attr(attrPath);
|
||||
JSONObject res(top.str);
|
||||
res.attr("error", e.msg());
|
||||
top.str << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -266,9 +259,8 @@ int main(int argc, char * * argv)
|
|||
Value v;
|
||||
state.evalFile(releaseExpr, v);
|
||||
|
||||
XMLWriter doc(true, std::cout);
|
||||
XMLOpenElement root(doc, "jobs");
|
||||
findJobs(state, doc, autoArgs, v, "");
|
||||
JSONObject json(std::cout);
|
||||
findJobs(state, json, autoArgs, v, "");
|
||||
|
||||
state.printStats();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ package Hydra::Helper::AddBuilds;
|
|||
use strict;
|
||||
use feature 'switch';
|
||||
use utf8;
|
||||
use XML::Simple;
|
||||
use JSON;
|
||||
use IPC::Run;
|
||||
use Nix::Store;
|
||||
use Nix::Config;
|
||||
|
|
@ -335,42 +335,15 @@ sub evalJobs {
|
|||
my $evaluator = ($exprType eq "guile") ? "hydra-eval-guile-jobs" : "hydra-eval-jobs";
|
||||
print STDERR "evaluator ${evaluator}\n";
|
||||
|
||||
(my $res, my $jobsXml, my $stderr) = captureStdoutStderr(10800,
|
||||
(my $res, my $jobsJSON, my $stderr) = captureStdoutStderr(10800,
|
||||
$evaluator, $nixExprFullPath, "--gc-roots-dir", getGCRootsDir, "-j", 1, inputsToArgs($inputInfo, $exprType));
|
||||
if ($res) {
|
||||
die "$evaluator returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8))
|
||||
. ":\n" . ($stderr ? $stderr : "(no output)\n");
|
||||
}
|
||||
. ":\n" . ($stderr ? $stderr : "(no output)\n")
|
||||
if $res;
|
||||
|
||||
print STDERR "$stderr";
|
||||
|
||||
my $jobs = XMLin(
|
||||
$jobsXml,
|
||||
ForceArray => ['error', 'job', 'arg', 'output'],
|
||||
KeyAttr => { output => "+name" },
|
||||
SuppressEmpty => '')
|
||||
or die "cannot parse XML output";
|
||||
|
||||
my %jobNames;
|
||||
my $errors;
|
||||
my @filteredJobs = ();
|
||||
foreach my $job (@{$jobs->{job}}) {
|
||||
# Ensure that there is only one job with the given
|
||||
# name. FIXME: this check will become unnecessary after we
|
||||
# remove support for multiple values per jobset input.
|
||||
if (defined $jobNames{$job->{jobName}}) {
|
||||
$errors .= "error: there are multiple jobs named ‘$job->{jobName}’\n\n";
|
||||
next;
|
||||
}
|
||||
$jobNames{$job->{jobName}} = 1;
|
||||
push @filteredJobs, $job;
|
||||
}
|
||||
$jobs->{job} = \@filteredJobs;
|
||||
|
||||
# Handle utf-8 characters in error messages. No idea why this works.
|
||||
utf8::decode($_->{msg}) foreach @{$jobs->{error}};
|
||||
|
||||
return ($jobs, $nixExprInput, $errors);
|
||||
return (decode_json($jobsJSON), $nixExprInput);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -464,22 +437,18 @@ sub getPrevJobsetEval {
|
|||
sub checkBuild {
|
||||
my ($db, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildMap, $prevEval, $jobOutPathMap, $plugins) = @_;
|
||||
|
||||
my @outputNames = sort keys %{$buildInfo->{output}};
|
||||
my @outputNames = sort keys %{$buildInfo->{outputs}};
|
||||
die unless scalar @outputNames;
|
||||
|
||||
# In various checks we can use an arbitrary output (the first)
|
||||
# rather than all outputs, since if one output is the same, the
|
||||
# others will be as well.
|
||||
my $firstOutputName = $outputNames[0];
|
||||
my $firstOutputPath = $buildInfo->{output}->{$firstOutputName}->{path};
|
||||
my $firstOutputPath = $buildInfo->{outputs}->{$firstOutputName};
|
||||
|
||||
my $jobName = $buildInfo->{jobName} or die;
|
||||
my $drvPath = $buildInfo->{drvPath} or die;
|
||||
|
||||
my $priority = 100;
|
||||
$priority = int($buildInfo->{schedulingPriority})
|
||||
if $buildInfo->{schedulingPriority} =~ /^\d+$/;
|
||||
|
||||
my $build;
|
||||
|
||||
txn_do($db, sub {
|
||||
|
|
@ -531,7 +500,7 @@ sub checkBuild {
|
|||
my $buildStatus;
|
||||
my $releaseName;
|
||||
foreach my $name (@outputNames) {
|
||||
my $path = $buildInfo->{output}->{$name}->{path};
|
||||
my $path = $buildInfo->{outputs}->{$name};
|
||||
if (isValidPath($path)) {
|
||||
if (-f "$path/nix-support/failed") {
|
||||
$buildStatus = 6;
|
||||
|
|
@ -573,13 +542,13 @@ sub checkBuild {
|
|||
, system => $buildInfo->{system}
|
||||
, nixexprinput => $jobset->nixexprinput
|
||||
, nixexprpath => $jobset->nixexprpath
|
||||
, priority => $priority
|
||||
, priority => $buildInfo->{schedulingPriority}
|
||||
, busy => 0
|
||||
, locker => ""
|
||||
, %extraFlags
|
||||
});
|
||||
|
||||
$build->buildoutputs->create({ name => $_, path => $buildInfo->{output}->{$_}->{path} })
|
||||
$build->buildoutputs->create({ name => $_, path => $buildInfo->{outputs}->{$_} })
|
||||
foreach @outputNames;
|
||||
|
||||
$buildMap->{$build->id} = { id => $build->id, jobName => $jobName, new => 1, drvPath => $drvPath };
|
||||
|
|
|
|||
|
|
@ -144,9 +144,11 @@ sub checkJobsetWrapped {
|
|||
|
||||
# Evaluate the job expression.
|
||||
my $evalStart = time;
|
||||
my ($jobs, $nixExprInput, $msg) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath);
|
||||
my ($jobs, $nixExprInput) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath);
|
||||
my $evalStop = time;
|
||||
|
||||
$jobs->{$_}->{jobName} = $_ for keys %{$jobs};
|
||||
|
||||
my $jobOutPathMap = {};
|
||||
|
||||
txn_do($db, sub {
|
||||
|
|
@ -160,8 +162,8 @@ sub checkJobsetWrapped {
|
|||
|
||||
# Schedule each successfully evaluated job.
|
||||
my %buildMap;
|
||||
foreach my $job (permute @{$jobs->{job}}) {
|
||||
next if $job->{jobName} eq "";
|
||||
foreach my $job (permute(values %{$jobs})) {
|
||||
next if defined $job->{error};
|
||||
print STDERR " considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n";
|
||||
checkBuild($db, $jobset, $inputInfo, $nixExprInput, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins);
|
||||
}
|
||||
|
|
@ -244,10 +246,12 @@ sub checkJobsetWrapped {
|
|||
});
|
||||
|
||||
# Store the error messages for jobs that failed to evaluate.
|
||||
foreach my $error (@{$jobs->{error}}) {
|
||||
my $msg = "";
|
||||
foreach my $job (values %{$jobs}) {
|
||||
next unless defined $job->{error};
|
||||
$msg .=
|
||||
($error->{location} ne "" ? "in job ‘$error->{location}’" : "at top-level") .
|
||||
":\n" . $error->{msg} . "\n\n";
|
||||
($job->{jobName} ne "" ? "in job ‘$job->{jobName}’" : "at top-level") .
|
||||
":\n" . $job->{error} . "\n\n";
|
||||
}
|
||||
setJobsetError($jobset, $msg);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue