From 68e689cacecf8e94153dc9462778eb18888781ca Mon Sep 17 00:00:00 2001 From: Samuel Dionne-Riel Date: Sun, 25 Oct 2020 18:50:54 -0400 Subject: [PATCH 1/2] hydra-eval-jobs: Identify unexpected errors in handling aggregate jobs The vague "[json.exception.type_error.302] type must be string, but is null" is **absolutely** unhelpful in the way Hydra currently handles it on evaluation. This is handling *unexpected* errors only; the following commit will handle the specific instance of the previously mentioned error. --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 78 +++++++++++++++----------- 1 file changed, 45 insertions(+), 33 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 600d4670..1cbc6fd8 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -442,45 +442,57 @@ int main(int argc, char * * argv) for (auto i = state->jobs.begin(); i != state->jobs.end(); ++i) { auto jobName = i.key(); auto & job = i.value(); + // For the error message + std::string lastTriedJobName = i.key(); auto named = job.find("namedConstituents"); if (named == job.end()) continue; - if (myArgs.dryRun) { - for (std::string jobName2 : *named) { - auto job2 = state->jobs.find(jobName2); - if (job2 == state->jobs.end()) - throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); - std::string drvPath2 = (*job2)["drvPath"]; - job["constituents"].push_back(drvPath2); + try { + if (myArgs.dryRun) { + for (std::string jobName2 : *named) { + lastTriedJobName = jobName2; + auto job2 = state->jobs.find(jobName2); + if (job2 == state->jobs.end()) + throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); + std::string drvPath2 = (*job2)["drvPath"]; + job["constituents"].push_back(drvPath2); + } + } else { + auto drvPath = store->parseStorePath((std::string) job["drvPath"]); + auto drv = store->readDerivation(drvPath); + + for (std::string jobName2 : *named) { + lastTriedJobName = jobName2; + auto job2 = state->jobs.find(jobName2); + if (job2 == state->jobs.end()) + throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); + auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]); + auto drv2 = store->readDerivation(drvPath2); + job["constituents"].push_back(store->printStorePath(drvPath2)); + drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first}; + } + + std::string drvName(drvPath.name()); + assert(hasSuffix(drvName, drvExtension)); + drvName.resize(drvName.size() - drvExtension.size()); + auto h = std::get(hashDerivationModulo(*store, drv, true)); + auto outPath = store->makeOutputPath("out", h, drvName); + drv.env["out"] = store->printStorePath(outPath); + drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = outPath } }); + auto newDrvPath = store->printStorePath(writeDerivation(*store, drv)); + + debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath); + + job["drvPath"] = newDrvPath; + job["outputs"]["out"] = store->printStorePath(outPath); } - } else { - auto drvPath = store->parseStorePath((std::string) job["drvPath"]); - auto drv = store->readDerivation(drvPath); + } catch (std::exception & e) { + // Print more information to help debugging. + printError("Unexpected error in hydra-eval-jobs when handling job '%s', when producing aggregate job '%s':", lastTriedJobName, jobName); - for (std::string jobName2 : *named) { - auto job2 = state->jobs.find(jobName2); - if (job2 == state->jobs.end()) - throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); - auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]); - auto drv2 = store->readDerivation(drvPath2); - job["constituents"].push_back(store->printStorePath(drvPath2)); - drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first}; - } - - std::string drvName(drvPath.name()); - assert(hasSuffix(drvName, drvExtension)); - drvName.resize(drvName.size() - drvExtension.size()); - auto h = std::get(hashDerivationModulo(*store, drv, true)); - auto outPath = store->makeOutputPath("out", h, drvName); - drv.env["out"] = store->printStorePath(outPath); - drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = outPath } }); - auto newDrvPath = store->printStorePath(writeDerivation(*store, drv)); - - debug("rewrote aggregate derivation %s -> %s", store->printStorePath(drvPath), newDrvPath); - - job["drvPath"] = newDrvPath; - job["outputs"]["out"] = store->printStorePath(outPath); + // And throw the original exception! + throw; } job.erase("namedConstituents"); From b5140c1da17b1af8ca531d84c0e7f3b05b7cbe60 Mon Sep 17 00:00:00 2001 From: Samuel Dionne-Riel Date: Sun, 25 Oct 2020 19:15:16 -0400 Subject: [PATCH 2/2] hydra-eval-jobs: Transmit original Nix error when handling aggregate jobs It might happen that a job from the aggregate returned an error! This is what the vague "[json.exception.type_error.302] type must be string, but is null" was all about in this instance; there was no `drvPath` to stringify! So we now actively watch for errors and copy them to the aggregate job. --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 1cbc6fd8..54936da9 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -467,10 +467,18 @@ int main(int argc, char * * argv) auto job2 = state->jobs.find(jobName2); if (job2 == state->jobs.end()) throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); - auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]); - auto drv2 = store->readDerivation(drvPath2); - job["constituents"].push_back(store->printStorePath(drvPath2)); - drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first}; + + if ((*job2).find("error") != (*job2).end()) { + if (job.find("error") == job.end()) { + job["error"] = fmt("Errors aggregating aggregate job '%1%'.\n", jobName); + } + job["error"] = fmt("While handling '%1%': %2%\n", jobName2, (std::string) (*job2)["error"]); + } else { + auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]); + auto drv2 = store->readDerivation(drvPath2); + job["constituents"].push_back(store->printStorePath(drvPath2)); + drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first}; + } } std::string drvName(drvPath.name());