From 09a96c642a4766668a44fafc6362b0f69f3b4660 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 30 Sep 2014 00:20:54 +0200 Subject: [PATCH] hydra-eval-jobs: Use JSON instead of XML XML::Simple is pretty slow - reading the output for the Nixpkgs jobset takes half a minute or so. JSON is pretty much instantaneous. --- src/c/hydra-eval-jobs.cc | 78 ++++++++++++++----------------- src/lib/Hydra/Helper/AddBuilds.pm | 53 +++++---------------- src/script/hydra-evaluator | 16 ++++--- 3 files changed, 56 insertions(+), 91 deletions(-) diff --git a/src/c/hydra-eval-jobs.cc b/src/c/hydra-eval-jobs.cc index 1eb949be..75a16847 100644 --- a/src/c/hydra-eval-jobs.cc +++ b/src/c/hydra-eval-jobs.cc @@ -8,7 +8,7 @@ #include "eval.hh" #include "eval-inline.hh" #include "util.hh" -#include "xml-writer.hh" +#include "value-to-json.hh" #include "get-drvs.hh" #include "common-opts.hh" #include "globals.hh" @@ -23,11 +23,11 @@ typedef std::list > ValueList; typedef std::map AutoArgs; -static void findJobs(EvalState & state, XMLWriter & doc, +static void findJobs(EvalState & state, JSONObject & top, const AutoArgs & argsLeft, Value & v, const string & attrPath); -static void tryJobAlts(EvalState & state, XMLWriter & doc, +static void tryJobAlts(EvalState & state, JSONObject & top, const AutoArgs & argsLeft, const string & attrPath, Value & fun, Formals::Formals_::iterator cur, Formals::Formals_::iterator last, @@ -38,7 +38,7 @@ static void tryJobAlts(EvalState & state, XMLWriter & doc, state.mkAttrs(*arg, 0); arg->attrs = &actualArgs; mkApp(v, fun, *arg); - findJobs(state, doc, argsLeft, v, attrPath); + findJobs(state, top, argsLeft, v, attrPath); return; } @@ -50,7 +50,7 @@ static void tryJobAlts(EvalState & state, XMLWriter & doc, if (!cur->def) throw TypeError(format("job `%1%' requires an argument named `%2%'") % attrPath % cur->name); - tryJobAlts(state, doc, argsLeft, attrPath, fun, next, last, actualArgs); + tryJobAlts(state, top, argsLeft, attrPath, fun, next, last, actualArgs); return; } @@ -63,7 +63,7 @@ static void tryJobAlts(EvalState & state, XMLWriter & doc, actualArgs2.push_back(Attr(cur->name, *i)); actualArgs2.sort(); // !!! inefficient argsLeft2.erase(cur->name); - tryJobAlts(state, doc, argsLeft2, attrPath, fun, next, last, actualArgs2); + tryJobAlts(state, top, argsLeft2, attrPath, fun, next, last, actualArgs2); ++n; } } @@ -93,7 +93,7 @@ static string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & } -static void findJobsWrapped(EvalState & state, XMLWriter & doc, +static void findJobsWrapped(EvalState & state, JSONObject & top, const AutoArgs & argsLeft, Value & v, const string & attrPath) { debug(format("at path `%1%'") % attrPath); @@ -107,7 +107,6 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc, DrvInfo drv(state); if (getDerivation(state, v, drv, false)) { - XMLAttrs xmlAttrs; Path drvPath; DrvInfo::Outputs outputs = drv.queryOutputs(); @@ -115,21 +114,19 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc, if (drv.system == "unknown") throw EvalError("derivation must have a ‘system’ attribute"); - xmlAttrs["jobName"] = attrPath; - xmlAttrs["nixName"] = drv.name; - xmlAttrs["system"] = drv.system; - xmlAttrs["drvPath"] = drvPath = drv.queryDrvPath(); - xmlAttrs["description"] = drv.queryMetaString("description"); - xmlAttrs["longDescription"] = drv.queryMetaString("longDescription"); - xmlAttrs["license"] = queryMetaStrings(state, drv, "license"); - xmlAttrs["homepage"] = drv.queryMetaString("homepage"); - xmlAttrs["maintainers"] = queryMetaStrings(state, drv, "maintainers"); - - xmlAttrs["schedulingPriority"] = int2String(drv.queryMetaInt("schedulingPriority", 100)); - - xmlAttrs["timeout"] = int2String(drv.queryMetaInt("timeout", 36000)); - - xmlAttrs["maxSilent"] = int2String(drv.queryMetaInt("maxSilent", 7200)); + top.attr(attrPath); + JSONObject res(top.str); + res.attr("nixName", drv.name); + res.attr("system", drv.system); + res.attr("drvPath", drvPath = drv.queryDrvPath()); + res.attr("description", drv.queryMetaString("description")); + res.attr("longDescription", drv.queryMetaString("longDescription")); + res.attr("license", queryMetaStrings(state, drv, "license")); + res.attr("homepage", drv.queryMetaString("homepage")); + res.attr("maintainers", queryMetaStrings(state, drv, "maintainers")); + res.attr("schedulingPriority", drv.queryMetaInt("schedulingPriority", 100)); + res.attr("timeout", drv.queryMetaInt("timeout", 36000)); + res.attr("maxSilent", drv.queryMetaInt("maxSilent", 7200)); /* If this is an aggregate, then get its constituents. */ Bindings::iterator a = v.attrs->find(state.symbols.create("_hydraAggregate")); @@ -145,7 +142,7 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc, size_t index = i->find("!", 1); drvs.insert(string(*i, index + 1)); } - xmlAttrs["constituents"] = concatStringsSep(" ", drvs); + res.attr("constituents", concatStringsSep(" ", drvs)); } /* Register the derivation as a GC root. !!! This @@ -156,20 +153,16 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc, if (!pathExists(root)) addPermRoot(*store, drvPath, root, false); } - XMLOpenElement _(doc, "job", xmlAttrs); - - foreach (DrvInfo::Outputs::iterator, j, outputs) { - XMLAttrs attrs2; - attrs2["name"] = j->first; - attrs2["path"] = j->second; - doc.writeEmptyElement("output", attrs2); - } + res.attr("outputs"); + JSONObject res2(res.str); + for (auto & j : outputs) + res2.attr(j.first, j.second); } else { if (!state.isDerivation(v)) { foreach (Bindings::iterator, i, *v.attrs) - findJobs(state, doc, argsLeft, *i->value, + findJobs(state, top, argsLeft, *i->value, (attrPath.empty() ? "" : attrPath + ".") + (string) i->name); } } @@ -177,7 +170,7 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc, else if (v.type == tLambda && v.lambda.fun->matchAttrs) { Bindings & tmp(*state.allocBindings(0)); - tryJobAlts(state, doc, argsLeft, attrPath, v, + tryJobAlts(state, top, argsLeft, attrPath, v, v.lambda.fun->formals->formals.begin(), v.lambda.fun->formals->formals.end(), tmp); @@ -192,16 +185,16 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc, } -static void findJobs(EvalState & state, XMLWriter & doc, +static void findJobs(EvalState & state, JSONObject & top, const AutoArgs & argsLeft, Value & v, const string & attrPath) { try { - findJobsWrapped(state, doc, argsLeft, v, attrPath); + findJobsWrapped(state, top, argsLeft, v, attrPath); } catch (EvalError & e) { - XMLAttrs xmlAttrs; - xmlAttrs["location"] = attrPath; - xmlAttrs["msg"] = e.msg(); - XMLOpenElement _(doc, "error", xmlAttrs); + top.attr(attrPath); + JSONObject res(top.str); + res.attr("error", e.msg()); + top.str << "\n"; } } @@ -266,9 +259,8 @@ int main(int argc, char * * argv) Value v; state.evalFile(releaseExpr, v); - XMLWriter doc(true, std::cout); - XMLOpenElement root(doc, "jobs"); - findJobs(state, doc, autoArgs, v, ""); + JSONObject json(std::cout); + findJobs(state, json, autoArgs, v, ""); state.printStats(); }); diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index 843a4ff6..b85bc4f8 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -3,7 +3,7 @@ package Hydra::Helper::AddBuilds; use strict; use feature 'switch'; use utf8; -use XML::Simple; +use JSON; use IPC::Run; use Nix::Store; use Nix::Config; @@ -335,42 +335,15 @@ sub evalJobs { my $evaluator = ($exprType eq "guile") ? "hydra-eval-guile-jobs" : "hydra-eval-jobs"; print STDERR "evaluator ${evaluator}\n"; - (my $res, my $jobsXml, my $stderr) = captureStdoutStderr(10800, + (my $res, my $jobsJSON, my $stderr) = captureStdoutStderr(10800, $evaluator, $nixExprFullPath, "--gc-roots-dir", getGCRootsDir, "-j", 1, inputsToArgs($inputInfo, $exprType)); - if ($res) { - die "$evaluator returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8)) - . ":\n" . ($stderr ? $stderr : "(no output)\n"); - } + die "$evaluator returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8)) + . ":\n" . ($stderr ? $stderr : "(no output)\n") + if $res; print STDERR "$stderr"; - my $jobs = XMLin( - $jobsXml, - ForceArray => ['error', 'job', 'arg', 'output'], - KeyAttr => { output => "+name" }, - SuppressEmpty => '') - or die "cannot parse XML output"; - - my %jobNames; - my $errors; - my @filteredJobs = (); - foreach my $job (@{$jobs->{job}}) { - # Ensure that there is only one job with the given - # name. FIXME: this check will become unnecessary after we - # remove support for multiple values per jobset input. - if (defined $jobNames{$job->{jobName}}) { - $errors .= "error: there are multiple jobs named ‘$job->{jobName}’\n\n"; - next; - } - $jobNames{$job->{jobName}} = 1; - push @filteredJobs, $job; - } - $jobs->{job} = \@filteredJobs; - - # Handle utf-8 characters in error messages. No idea why this works. - utf8::decode($_->{msg}) foreach @{$jobs->{error}}; - - return ($jobs, $nixExprInput, $errors); + return (decode_json($jobsJSON), $nixExprInput); } @@ -464,22 +437,18 @@ sub getPrevJobsetEval { sub checkBuild { my ($db, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildMap, $prevEval, $jobOutPathMap, $plugins) = @_; - my @outputNames = sort keys %{$buildInfo->{output}}; + my @outputNames = sort keys %{$buildInfo->{outputs}}; die unless scalar @outputNames; # In various checks we can use an arbitrary output (the first) # rather than all outputs, since if one output is the same, the # others will be as well. my $firstOutputName = $outputNames[0]; - my $firstOutputPath = $buildInfo->{output}->{$firstOutputName}->{path}; + my $firstOutputPath = $buildInfo->{outputs}->{$firstOutputName}; my $jobName = $buildInfo->{jobName} or die; my $drvPath = $buildInfo->{drvPath} or die; - my $priority = 100; - $priority = int($buildInfo->{schedulingPriority}) - if $buildInfo->{schedulingPriority} =~ /^\d+$/; - my $build; txn_do($db, sub { @@ -531,7 +500,7 @@ sub checkBuild { my $buildStatus; my $releaseName; foreach my $name (@outputNames) { - my $path = $buildInfo->{output}->{$name}->{path}; + my $path = $buildInfo->{outputs}->{$name}; if (isValidPath($path)) { if (-f "$path/nix-support/failed") { $buildStatus = 6; @@ -573,13 +542,13 @@ sub checkBuild { , system => $buildInfo->{system} , nixexprinput => $jobset->nixexprinput , nixexprpath => $jobset->nixexprpath - , priority => $priority + , priority => $buildInfo->{schedulingPriority} , busy => 0 , locker => "" , %extraFlags }); - $build->buildoutputs->create({ name => $_, path => $buildInfo->{output}->{$_}->{path} }) + $build->buildoutputs->create({ name => $_, path => $buildInfo->{outputs}->{$_} }) foreach @outputNames; $buildMap->{$build->id} = { id => $build->id, jobName => $jobName, new => 1, drvPath => $drvPath }; diff --git a/src/script/hydra-evaluator b/src/script/hydra-evaluator index 8f367e77..96d910a0 100755 --- a/src/script/hydra-evaluator +++ b/src/script/hydra-evaluator @@ -144,9 +144,11 @@ sub checkJobsetWrapped { # Evaluate the job expression. my $evalStart = time; - my ($jobs, $nixExprInput, $msg) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath); + my ($jobs, $nixExprInput) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath); my $evalStop = time; + $jobs->{$_}->{jobName} = $_ for keys %{$jobs}; + my $jobOutPathMap = {}; txn_do($db, sub { @@ -160,8 +162,8 @@ sub checkJobsetWrapped { # Schedule each successfully evaluated job. my %buildMap; - foreach my $job (permute @{$jobs->{job}}) { - next if $job->{jobName} eq ""; + foreach my $job (permute(values %{$jobs})) { + next if defined $job->{error}; print STDERR " considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n"; checkBuild($db, $jobset, $inputInfo, $nixExprInput, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins); } @@ -244,10 +246,12 @@ sub checkJobsetWrapped { }); # Store the error messages for jobs that failed to evaluate. - foreach my $error (@{$jobs->{error}}) { + my $msg = ""; + foreach my $job (values %{$jobs}) { + next unless defined $job->{error}; $msg .= - ($error->{location} ne "" ? "in job ‘$error->{location}’" : "at top-level") . - ":\n" . $error->{msg} . "\n\n"; + ($job->{jobName} ne "" ? "in job ‘$job->{jobName}’" : "at top-level") . + ":\n" . $job->{error} . "\n\n"; } setJobsetError($jobset, $msg); }