From 2394140843264f972f8c4ea27608227eaac884c9 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 26 Sep 2020 23:36:37 +0200 Subject: [PATCH 1/2] flake.lock: Update Flake input changes: * Updated 'nix': 'github:NixOS/nix/2a8017092025a7c108ca1a829a8b4f0bcf7ee2c0' -> 'github:NixOS/nix/8b4a542d1767e0df7b3c0902b766f34352cb0958' * Updated 'nix/lowdown-src': 'github:edolstra/lowdown/c7a4e715af1e233080842db82d15b261cb74cb28' -> 'github:kristapsdz/lowdown/1705b4a26fbf065d9574dce47a94e8c7c79e052f' * Updated 'nixpkgs': 'github:NixOS/nixpkgs/252bfe0107587d40092057f338e9ffcf7bbd90cb' -> 'github:NixOS/nixpkgs/5659cb448e9b615d642c5fe52779c2223e72f7eb' --- flake.lock | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/flake.lock b/flake.lock index 9086f0b6..e20be7c3 100644 --- a/flake.lock +++ b/flake.lock @@ -3,16 +3,15 @@ "lowdown-src": { "flake": false, "locked": { - "lastModified": 1598296217, - "narHash": "sha256-ha7lyNY1d8m+osmDpPc9f/bfZ3ZC1IVIXwfyklSWg8I=", - "owner": "edolstra", + "lastModified": 1598695561, + "narHash": "sha256-gyH/5j+h/nWw0W8AcR2WKvNBUsiQ7QuxqSJNXAwV+8E=", + "owner": "kristapsdz", "repo": "lowdown", - "rev": "c7a4e715af1e233080842db82d15b261cb74cb28", + "rev": "1705b4a26fbf065d9574dce47a94e8c7c79e052f", "type": "github" }, "original": { - "owner": "edolstra", - "ref": "no-structs-in-anonymous-unions", + "owner": "kristapsdz", "repo": "lowdown", "type": "github" } @@ -23,11 +22,11 @@ "nixpkgs": "nixpkgs" }, "locked": { - "lastModified": 1600161919, - "narHash": "sha256-kfxiAdd/CK09M5i2WeBbTIxDenAdlqDLkhsrMZy8qQo=", + "lastModified": 1601119984, + "narHash": "sha256-OBwnd2GzAXb0VtOuovgj3mv/Ax8Jyqelwh6j/TeMD5U=", "owner": "NixOS", "repo": "nix", - "rev": "2a8017092025a7c108ca1a829a8b4f0bcf7ee2c0", + "rev": "8b4a542d1767e0df7b3c0902b766f34352cb0958", "type": "github" }, "original": { @@ -52,11 +51,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1599903709, - "narHash": "sha256-tinVKY7zRdSXmVOp7BAulNtIb2iP5nvNp5UAxihKXNI=", + "lastModified": 1600952868, + "narHash": "sha256-U1HfLLrV7gs2u0hU1YjU4aDGbTC1uYtxyNu2TWXKXMY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "252bfe0107587d40092057f338e9ffcf7bbd90cb", + "rev": "5659cb448e9b615d642c5fe52779c2223e72f7eb", "type": "github" }, "original": { From 9cc76f6d69aa69ff47697f3b7698f8d95f7dbbac Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 26 Sep 2020 23:37:39 +0200 Subject: [PATCH 2/2] Fix build with latest Nix Recently a few internal APIs have changed[1]. The `outputPaths` function has been removed and a lot of data structures are modeled with `std::optional` which broke compilation. This patch updates the code in `hydra-queue-runner` accordingly to make sure that Hydra compiles again. [1] https://github.com/NixOS/nix/pull/3883 --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 1 + src/hydra-queue-runner/build-remote.cc | 12 +++-- src/hydra-queue-runner/build-result.cc | 19 ++++---- src/hydra-queue-runner/builder.cc | 11 +++-- src/hydra-queue-runner/hydra-queue-runner.cc | 11 +++-- src/hydra-queue-runner/queue-monitor.cc | 50 ++++++++++---------- 6 files changed, 59 insertions(+), 45 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 4f2ae3be..313bb855 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -1,5 +1,6 @@ #include #include +#include #include "shared.hh" #include "store-api.hh" diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 152bdd13..3a668ab3 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -263,9 +263,9 @@ void State::buildRemote(ref destStore, auto drv2 = localStore->readDerivation(input.first); for (auto & name : input.second) { if (auto i = get(drv2.outputs, name)) { - auto outPath = i->path(*localStore, drv2.name); - inputs.insert(outPath); - basicDrv.inputSrcs.insert(outPath); + auto outPath = i->path(*localStore, drv2.name, name); + inputs.insert(*outPath); + basicDrv.inputSrcs.insert(*outPath); } } } @@ -434,7 +434,11 @@ void State::buildRemote(ref destStore, auto now1 = std::chrono::steady_clock::now(); - auto outputs = step->drv->outputPaths(*localStore); + StorePathSet outputs; + for (auto & i : step->drv->outputsAndOptPaths(*localStore)) { + if (i.second.second) + outputs.insert(*i.second.second); + } /* Get info about each output path. */ std::map infos; diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc index a22a7115..f69bf0df 100644 --- a/src/hydra-queue-runner/build-result.cc +++ b/src/hydra-queue-runner/build-result.cc @@ -16,10 +16,13 @@ BuildOutput getBuildOutput( BuildOutput res; /* Compute the closure size. */ - auto outputs = drv.outputPaths(*store); + StorePathSet outputs; StorePathSet closure; - for (auto & output : outputs) - store->computeFSClosure(output, closure); + for (auto & i : drv.outputsAndOptPaths(*store)) + if (i.second.second) { + store->computeFSClosure(*i.second.second, closure); + outputs.insert(*i.second.second); + } for (auto & path : closure) { auto info = store->queryPathInfo(path); res.closureSize += info->narSize; @@ -104,13 +107,13 @@ BuildOutput getBuildOutput( /* If no build products were explicitly declared, then add all outputs as a product of type "nix-build". */ if (!explicitProducts) { - for (auto & output : drv.outputs) { + for (auto & [name, output] : drv.outputs) { BuildProduct product; - auto outPath = output.second.path(*store, drv.name); - product.path = store->printStorePath(outPath); + auto outPath = output.path(*store, drv.name, name); + product.path = store->printStorePath(*outPath); product.type = "nix-build"; - product.subtype = output.first == "out" ? "" : output.first; - product.name = outPath.name(); + product.subtype = name == "out" ? "" : name; + product.name = outPath->name(); auto file = narMembers.find(product.path); assert(file != narMembers.end()); diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index c5b44344..7f8830f9 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -274,8 +274,10 @@ State::StepResult State::doBuildStep(nix::ref destStore, assert(stepNr); - for (auto & path : step->drv->outputPaths(*localStore)) - addRoot(path); + for (auto & i : step->drv->outputsAndOptPaths(*localStore)) { + if (i.second.second) + addRoot(*i.second.second); + } /* Register success in the database for all Build objects that have this step as the top-level step. Since the queue @@ -463,8 +465,9 @@ void State::failStep( /* Remember failed paths in the database so that they won't be built again. */ if (result.stepStatus != bsCachedFailure && result.canCache) - for (auto & path : step->drv->outputPaths(*localStore)) - txn.exec_params0("insert into FailedPaths values ($1)", localStore->printStorePath(path)); + for (auto & i : step->drv->outputsAndOptPaths(*localStore)) + if (i.second.second) + txn.exec_params0("insert into FailedPaths values ($1)", localStore->printStorePath(*i.second.second)); txn.commit(); } diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index df890ee9..f50c00e5 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -256,10 +256,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID if (r.affected_rows() == 0) goto restart; - for (auto & output : step->drv->outputs) + for (auto & [name, output] : step->drv->outputs) txn.exec_params0 ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)", - buildId, stepNr, output.first, localStore->printStorePath(output.second.path(*localStore, step->drv->name))); + buildId, stepNr, name, localStore->printStorePath(*output.path(*localStore, step->drv->name, name))); if (status == bsBusy) txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr)); @@ -440,9 +440,10 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build, bool State::checkCachedFailure(Step::ptr step, Connection & conn) { pqxx::work txn(conn); - for (auto & path : step->drv->outputPaths(*localStore)) - if (!txn.exec_params("select 1 from FailedPaths where path = $1", localStore->printStorePath(path)).empty()) - return true; + for (auto & i : step->drv->outputsAndOptPaths(*localStore)) + if (i.second.second) + if (!txn.exec_params("select 1 from FailedPaths where path = $1", localStore->printStorePath(*i.second.second)).empty()) + return true; return false; } diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 257f2f0d..53d00f99 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -176,13 +176,15 @@ bool State::getQueuedBuilds(Connection & conn, if (!res[0].is_null()) propagatedFrom = res[0].as(); if (!propagatedFrom) { - for (auto & output : ex.step->drv->outputPaths(*localStore)) { - auto res = txn.exec_params - ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1", - localStore->printStorePath(output)); - if (!res[0][0].is_null()) { - propagatedFrom = res[0][0].as(); - break; + for (auto & i : ex.step->drv->outputsAndOptPaths(*localStore)) { + if (i.second.second) { + auto res = txn.exec_params + ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1", + localStore->printStorePath(*i.second.second)); + if (!res[0][0].is_null()) { + propagatedFrom = res[0][0].as(); + break; + } } } } @@ -221,8 +223,9 @@ bool State::getQueuedBuilds(Connection & conn, auto drv = localStore->readDerivation(build->drvPath); BuildOutput res = getBuildOutputCached(conn, destStore, drv); - for (auto & path : drv.outputPaths(*localStore)) - addRoot(path); + for (auto & i : drv.outputsAndOptPaths(*localStore)) + if (i.second.second) + addRoot(*i.second.second); { auto mc = startDbUpdate(); @@ -453,10 +456,9 @@ Step::ptr State::createStep(ref destStore, /* Are all outputs valid? */ bool valid = true; - auto outputs = step->drv->outputPaths(*localStore); DerivationOutputs missing; for (auto & i : step->drv->outputs) - if (!destStore->isValidPath(i.second.path(*localStore, step->drv->name))) { + if (!destStore->isValidPath(*i.second.path(*localStore, step->drv->name, i.first))) { valid = false; missing.insert_or_assign(i.first, i.second); } @@ -467,12 +469,12 @@ Step::ptr State::createStep(ref destStore, size_t avail = 0; for (auto & i : missing) { - auto path = i.second.path(*localStore, step->drv->name); - if (/* localStore != destStore && */ localStore->isValidPath(path)) + auto path = i.second.path(*localStore, step->drv->name, i.first); + if (/* localStore != destStore && */ localStore->isValidPath(*path)) avail++; else if (useSubstitutes) { SubstitutablePathInfos infos; - localStore->querySubstitutablePathInfos({{path, {}}}, infos); + localStore->querySubstitutablePathInfos({{*path, {}}}, infos); if (infos.size() == 1) avail++; } @@ -481,37 +483,37 @@ Step::ptr State::createStep(ref destStore, if (missing.size() == avail) { valid = true; for (auto & i : missing) { - auto path = i.second.path(*localStore, step->drv->name); + auto path = i.second.path(*localStore, step->drv->name, i.first); try { time_t startTime = time(0); - if (localStore->isValidPath(path)) + if (localStore->isValidPath(*path)) printInfo("copying output ‘%1%’ of ‘%2%’ from local store", - localStore->printStorePath(path), + localStore->printStorePath(*path), localStore->printStorePath(drvPath)); else { printInfo("substituting output ‘%1%’ of ‘%2%’", - localStore->printStorePath(path), + localStore->printStorePath(*path), localStore->printStorePath(drvPath)); - localStore->ensurePath(path); + localStore->ensurePath(*path); // FIXME: should copy directly from substituter to destStore. } - copyClosure(ref(localStore), destStore, {path}); + copyClosure(ref(localStore), destStore, {*path}); time_t stopTime = time(0); { auto mc = startDbUpdate(); pqxx::work txn(conn); - createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", path); + createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", *path); txn.commit(); } } catch (Error & e) { printError("while copying/substituting output ‘%s’ of ‘%s’: %s", - localStore->printStorePath(path), + localStore->printStorePath(*path), localStore->printStorePath(drvPath), e.what()); valid = false; @@ -617,12 +619,12 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref { pqxx::work txn(conn); - for (auto & output : drv.outputPaths(*localStore)) { + for (auto & [name, output] : drv.outputsAndOptPaths(*localStore)) { auto r = txn.exec_params ("select id, buildStatus, releaseName, closureSize, size from Builds b " "join BuildOutputs o on b.id = o.build " "where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1", - localStore->printStorePath(output)); + localStore->printStorePath(*output.second)); if (r.empty()) continue; BuildID id = r[0][0].as();