ma27/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

hydra-eval-jobs: Use JSON instead of XML

Browse source 
XML::Simple is pretty slow - reading the output for the Nixpkgs jobset
takes half a minute or so. JSON is pretty much instantaneous.
This commit is contained in:
Eelco Dolstra 2014-09-30 00:20:54 +02:00
parent d9a5143fcb
commit 09a96c642a
3 changed files with 56 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

78
src/c/hydra-eval-jobs.cc
View file

@ -8,7 +8,7 @@
#include "eval.hh" #include "eval.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "util.hh" #include "util.hh"
#include "xml-writer.hh" #include "value-to-json.hh"
#include "get-drvs.hh" #include "get-drvs.hh"
#include "common-opts.hh" #include "common-opts.hh"
#include "globals.hh" #include "globals.hh"
@ -23,11 +23,11 @@ typedef std::list<Value *, traceable_allocator<Value *> > ValueList;
typedef std::map<Symbol, ValueList> AutoArgs; typedef std::map<Symbol, ValueList> AutoArgs;
static void findJobs(EvalState & state, XMLWriter & doc, static void findJobs(EvalState & state, JSONObject & top,
const AutoArgs & argsLeft, Value & v, const string & attrPath); const AutoArgs & argsLeft, Value & v, const string & attrPath);
static void tryJobAlts(EvalState & state, XMLWriter & doc, static void tryJobAlts(EvalState & state, JSONObject & top,
const AutoArgs & argsLeft, const string & attrPath, Value & fun, const AutoArgs & argsLeft, const string & attrPath, Value & fun,
Formals::Formals_::iterator cur, Formals::Formals_::iterator cur,
Formals::Formals_::iterator last, Formals::Formals_::iterator last,
@ -38,7 +38,7 @@ static void tryJobAlts(EvalState & state, XMLWriter & doc,
state.mkAttrs(*arg, 0); state.mkAttrs(*arg, 0);
arg->attrs = &actualArgs; arg->attrs = &actualArgs;
mkApp(v, fun, *arg); mkApp(v, fun, *arg);
findJobs(state, doc, argsLeft, v, attrPath); findJobs(state, top, argsLeft, v, attrPath);
return; return;
} }
@ -50,7 +50,7 @@ static void tryJobAlts(EvalState & state, XMLWriter & doc,
if (!cur->def) if (!cur->def)
throw TypeError(format("job `%1%' requires an argument named `%2%'") throw TypeError(format("job `%1%' requires an argument named `%2%'")
% attrPath % cur->name); % attrPath % cur->name);
tryJobAlts(state, doc, argsLeft, attrPath, fun, next, last, actualArgs); tryJobAlts(state, top, argsLeft, attrPath, fun, next, last, actualArgs);
return; return;
} }
@ -63,7 +63,7 @@ static void tryJobAlts(EvalState & state, XMLWriter & doc,
actualArgs2.push_back(Attr(cur->name, *i)); actualArgs2.push_back(Attr(cur->name, *i));
actualArgs2.sort(); // !!! inefficient actualArgs2.sort(); // !!! inefficient
argsLeft2.erase(cur->name); argsLeft2.erase(cur->name);
tryJobAlts(state, doc, argsLeft2, attrPath, fun, next, last, actualArgs2); tryJobAlts(state, top, argsLeft2, attrPath, fun, next, last, actualArgs2);
++n; ++n;
} }
} }
@ -93,7 +93,7 @@ static string queryMetaStrings(EvalState & state, DrvInfo & drv, const string &
} }
static void findJobsWrapped(EvalState & state, XMLWriter & doc, static void findJobsWrapped(EvalState & state, JSONObject & top,
const AutoArgs & argsLeft, Value & v, const string & attrPath) const AutoArgs & argsLeft, Value & v, const string & attrPath)
{ {
debug(format("at path `%1%'") % attrPath); debug(format("at path `%1%'") % attrPath);
@ -107,7 +107,6 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc,
DrvInfo drv(state); DrvInfo drv(state);
if (getDerivation(state, v, drv, false)) { if (getDerivation(state, v, drv, false)) {
XMLAttrs xmlAttrs;
Path drvPath; Path drvPath;
DrvInfo::Outputs outputs = drv.queryOutputs(); DrvInfo::Outputs outputs = drv.queryOutputs();
@ -115,21 +114,19 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc,
if (drv.system == "unknown") if (drv.system == "unknown")
throw EvalError("derivation must have a ‘system’ attribute"); throw EvalError("derivation must have a ‘system’ attribute");
xmlAttrs["jobName"] = attrPath; top.attr(attrPath);
xmlAttrs["nixName"] = drv.name; JSONObject res(top.str);
xmlAttrs["system"] = drv.system; res.attr("nixName", drv.name);
xmlAttrs["drvPath"] = drvPath = drv.queryDrvPath(); res.attr("system", drv.system);
xmlAttrs["description"] = drv.queryMetaString("description"); res.attr("drvPath", drvPath = drv.queryDrvPath());
xmlAttrs["longDescription"] = drv.queryMetaString("longDescription"); res.attr("description", drv.queryMetaString("description"));
xmlAttrs["license"] = queryMetaStrings(state, drv, "license"); res.attr("longDescription", drv.queryMetaString("longDescription"));
xmlAttrs["homepage"] = drv.queryMetaString("homepage"); res.attr("license", queryMetaStrings(state, drv, "license"));
xmlAttrs["maintainers"] = queryMetaStrings(state, drv, "maintainers"); res.attr("homepage", drv.queryMetaString("homepage"));
res.attr("maintainers", queryMetaStrings(state, drv, "maintainers"));
xmlAttrs["schedulingPriority"] = int2String(drv.queryMetaInt("schedulingPriority", 100)); res.attr("schedulingPriority", drv.queryMetaInt("schedulingPriority", 100));
res.attr("timeout", drv.queryMetaInt("timeout", 36000));
xmlAttrs["timeout"] = int2String(drv.queryMetaInt("timeout", 36000)); res.attr("maxSilent", drv.queryMetaInt("maxSilent", 7200));
xmlAttrs["maxSilent"] = int2String(drv.queryMetaInt("maxSilent", 7200));
/* If this is an aggregate, then get its constituents. */ /* If this is an aggregate, then get its constituents. */
Bindings::iterator a = v.attrs->find(state.symbols.create("_hydraAggregate")); Bindings::iterator a = v.attrs->find(state.symbols.create("_hydraAggregate"));
@ -145,7 +142,7 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc,
size_t index = i->find("!", 1); size_t index = i->find("!", 1);
drvs.insert(string(*i, index + 1)); drvs.insert(string(*i, index + 1));
} }
xmlAttrs["constituents"] = concatStringsSep(" ", drvs); res.attr("constituents", concatStringsSep(" ", drvs));
} }
/* Register the derivation as a GC root. !!! This /* Register the derivation as a GC root. !!! This
@ -156,20 +153,16 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc,
if (!pathExists(root)) addPermRoot(*store, drvPath, root, false); if (!pathExists(root)) addPermRoot(*store, drvPath, root, false);
} }
XMLOpenElement _(doc, "job", xmlAttrs); res.attr("outputs");
JSONObject res2(res.str);
foreach (DrvInfo::Outputs::iterator, j, outputs) { for (auto & j : outputs)
XMLAttrs attrs2; res2.attr(j.first, j.second);
attrs2["name"] = j->first;
attrs2["path"] = j->second;
doc.writeEmptyElement("output", attrs2);
}
} }
else { else {
if (!state.isDerivation(v)) { if (!state.isDerivation(v)) {
foreach (Bindings::iterator, i, *v.attrs) foreach (Bindings::iterator, i, *v.attrs)
findJobs(state, doc, argsLeft, *i->value, findJobs(state, top, argsLeft, *i->value,
(attrPath.empty() ? "" : attrPath + ".") + (string) i->name); (attrPath.empty() ? "" : attrPath + ".") + (string) i->name);
} }
} }
@ -177,7 +170,7 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc,
else if (v.type == tLambda && v.lambda.fun->matchAttrs) { else if (v.type == tLambda && v.lambda.fun->matchAttrs) {
Bindings & tmp(*state.allocBindings(0)); Bindings & tmp(*state.allocBindings(0));
tryJobAlts(state, doc, argsLeft, attrPath, v, tryJobAlts(state, top, argsLeft, attrPath, v,
v.lambda.fun->formals->formals.begin(), v.lambda.fun->formals->formals.begin(),
v.lambda.fun->formals->formals.end(), v.lambda.fun->formals->formals.end(),
tmp); tmp);
@ -192,16 +185,16 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc,
} }
static void findJobs(EvalState & state, XMLWriter & doc, static void findJobs(EvalState & state, JSONObject & top,
const AutoArgs & argsLeft, Value & v, const string & attrPath) const AutoArgs & argsLeft, Value & v, const string & attrPath)
{ {
try { try {
findJobsWrapped(state, doc, argsLeft, v, attrPath); findJobsWrapped(state, top, argsLeft, v, attrPath);
} catch (EvalError & e) { } catch (EvalError & e) {
XMLAttrs xmlAttrs; top.attr(attrPath);
xmlAttrs["location"] = attrPath; JSONObject res(top.str);
xmlAttrs["msg"] = e.msg(); res.attr("error", e.msg());
XMLOpenElement _(doc, "error", xmlAttrs); top.str << "\n";
} }
} }
@ -266,9 +259,8 @@ int main(int argc, char * * argv)
Value v; Value v;
state.evalFile(releaseExpr, v); state.evalFile(releaseExpr, v);
XMLWriter doc(true, std::cout); JSONObject json(std::cout);
XMLOpenElement root(doc, "jobs"); findJobs(state, json, autoArgs, v, "");
findJobs(state, doc, autoArgs, v, "");
state.printStats(); state.printStats();
}); });

53
src/lib/Hydra/Helper/AddBuilds.pm
View file

@ -3,7 +3,7 @@ package Hydra::Helper::AddBuilds;
use strict; use strict;
use feature 'switch'; use feature 'switch';
use utf8; use utf8;
use XML::Simple; use JSON;
use IPC::Run; use IPC::Run;
use Nix::Store; use Nix::Store;
use Nix::Config; use Nix::Config;
@ -335,42 +335,15 @@ sub evalJobs {
my $evaluator = ($exprType eq "guile") ? "hydra-eval-guile-jobs" : "hydra-eval-jobs"; my $evaluator = ($exprType eq "guile") ? "hydra-eval-guile-jobs" : "hydra-eval-jobs";
print STDERR "evaluator ${evaluator}\n"; print STDERR "evaluator ${evaluator}\n";
(my $res, my $jobsXml, my $stderr) = captureStdoutStderr(10800, (my $res, my $jobsJSON, my $stderr) = captureStdoutStderr(10800,
$evaluator, $nixExprFullPath, "--gc-roots-dir", getGCRootsDir, "-j", 1, inputsToArgs($inputInfo, $exprType)); $evaluator, $nixExprFullPath, "--gc-roots-dir", getGCRootsDir, "-j", 1, inputsToArgs($inputInfo, $exprType));
if ($res) { die "$evaluator returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8))
die "$evaluator returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8)) . ":\n" . ($stderr ? $stderr : "(no output)\n")
. ":\n" . ($stderr ? $stderr : "(no output)\n"); if $res;
}
print STDERR "$stderr"; print STDERR "$stderr";
my $jobs = XMLin( return (decode_json($jobsJSON), $nixExprInput);
$jobsXml,
ForceArray => ['error', 'job', 'arg', 'output'],
KeyAttr => { output => "+name" },
SuppressEmpty => '')
or die "cannot parse XML output";
my %jobNames;
my $errors;
my @filteredJobs = ();
foreach my $job (@{$jobs->{job}}) {
# Ensure that there is only one job with the given
# name. FIXME: this check will become unnecessary after we
# remove support for multiple values per jobset input.
if (defined $jobNames{$job->{jobName}}) {
$errors .= "error: there are multiple jobs named ‘$job->{jobName}’\n\n";
next;
}
$jobNames{$job->{jobName}} = 1;
push @filteredJobs, $job;
}
$jobs->{job} = \@filteredJobs;
# Handle utf-8 characters in error messages. No idea why this works.
utf8::decode($_->{msg}) foreach @{$jobs->{error}};
return ($jobs, $nixExprInput, $errors);
} }
@ -464,22 +437,18 @@ sub getPrevJobsetEval {
sub checkBuild { sub checkBuild {
my ($db, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildMap, $prevEval, $jobOutPathMap, $plugins) = @_; my ($db, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildMap, $prevEval, $jobOutPathMap, $plugins) = @_;
my @outputNames = sort keys %{$buildInfo->{output}}; my @outputNames = sort keys %{$buildInfo->{outputs}};
die unless scalar @outputNames; die unless scalar @outputNames;
# In various checks we can use an arbitrary output (the first) # In various checks we can use an arbitrary output (the first)
# rather than all outputs, since if one output is the same, the # rather than all outputs, since if one output is the same, the
# others will be as well. # others will be as well.
my $firstOutputName = $outputNames[0]; my $firstOutputName = $outputNames[0];
my $firstOutputPath = $buildInfo->{output}->{$firstOutputName}->{path}; my $firstOutputPath = $buildInfo->{outputs}->{$firstOutputName};
my $jobName = $buildInfo->{jobName} or die; my $jobName = $buildInfo->{jobName} or die;
my $drvPath = $buildInfo->{drvPath} or die; my $drvPath = $buildInfo->{drvPath} or die;
my $priority = 100;
$priority = int($buildInfo->{schedulingPriority})
if $buildInfo->{schedulingPriority} =~ /^\d+$/;
my $build; my $build;
txn_do($db, sub { txn_do($db, sub {
@ -531,7 +500,7 @@ sub checkBuild {
my $buildStatus; my $buildStatus;
my $releaseName; my $releaseName;
foreach my $name (@outputNames) { foreach my $name (@outputNames) {
my $path = $buildInfo->{output}->{$name}->{path}; my $path = $buildInfo->{outputs}->{$name};
if (isValidPath($path)) { if (isValidPath($path)) {
if (-f "$path/nix-support/failed") { if (-f "$path/nix-support/failed") {
$buildStatus = 6; $buildStatus = 6;
@ -573,13 +542,13 @@ sub checkBuild {
, system => $buildInfo->{system} , system => $buildInfo->{system}
, nixexprinput => $jobset->nixexprinput , nixexprinput => $jobset->nixexprinput
, nixexprpath => $jobset->nixexprpath , nixexprpath => $jobset->nixexprpath
, priority => $priority , priority => $buildInfo->{schedulingPriority}
, busy => 0 , busy => 0
, locker => "" , locker => ""
, %extraFlags , %extraFlags
}); });
$build->buildoutputs->create({ name => $_, path => $buildInfo->{output}->{$_}->{path} }) $build->buildoutputs->create({ name => $_, path => $buildInfo->{outputs}->{$_} })
foreach @outputNames; foreach @outputNames;
$buildMap->{$build->id} = { id => $build->id, jobName => $jobName, new => 1, drvPath => $drvPath }; $buildMap->{$build->id} = { id => $build->id, jobName => $jobName, new => 1, drvPath => $drvPath };

16
src/script/hydra-evaluator
View file

@ -144,9 +144,11 @@ sub checkJobsetWrapped {
# Evaluate the job expression. # Evaluate the job expression.
my $evalStart = time; my $evalStart = time;
my ($jobs, $nixExprInput, $msg) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath); my ($jobs, $nixExprInput) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath);
my $evalStop = time; my $evalStop = time;
$jobs->{$_}->{jobName} = $_ for keys %{$jobs};
my $jobOutPathMap = {}; my $jobOutPathMap = {};
txn_do($db, sub { txn_do($db, sub {
@ -160,8 +162,8 @@ sub checkJobsetWrapped {
# Schedule each successfully evaluated job. # Schedule each successfully evaluated job.
my %buildMap; my %buildMap;
foreach my $job (permute @{$jobs->{job}}) { foreach my $job (permute(values %{$jobs})) {
next if $job->{jobName} eq ""; next if defined $job->{error};
print STDERR " considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n"; print STDERR " considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n";
checkBuild($db, $jobset, $inputInfo, $nixExprInput, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins); checkBuild($db, $jobset, $inputInfo, $nixExprInput, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins);
} }
@ -244,10 +246,12 @@ sub checkJobsetWrapped {
}); });
# Store the error messages for jobs that failed to evaluate. # Store the error messages for jobs that failed to evaluate.
foreach my $error (@{$jobs->{error}}) { my $msg = "";
foreach my $job (values %{$jobs}) {
next unless defined $job->{error};
$msg .= $msg .=
($error->{location} ne "" ? "in job ‘$error->{location}’" : "at top-level") . ($job->{jobName} ne "" ? "in job ‘$job->{jobName}’" : "at top-level") .
":\n" . $error->{msg} . "\n\n"; ":\n" . $job->{error} . "\n\n";
} }
setJobsetError($jobset, $msg); setJobsetError($jobset, $msg);
} }