ma27/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

Fix build with latest Nix

Browse source 
Recently a few internal APIs have changed[1]. The `outputPaths` function
has been removed and a lot of data structures are modeled with
`std::optional` which broke compilation.

This patch updates the code in `hydra-queue-runner` accordingly to make
sure that Hydra compiles again.

[1] https://github.com/NixOS/nix/pull/3883
This commit is contained in:
Maximilian Bosch 2020-09-26 23:37:39 +02:00
parent 2394140843
commit 9cc76f6d69
No known key found for this signature in database
GPG key ID: 091DBF4D1FC46B8E
6 changed files with 59 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/hydra-eval-jobs/hydra-eval-jobs.cc
View file

@ -1,5 +1,6 @@
#include <map> #include <map>
#include <iostream> #include <iostream>
#include <thread>
#include "shared.hh" #include "shared.hh"
#include "store-api.hh" #include "store-api.hh"

12
src/hydra-queue-runner/build-remote.cc
View file

@ -263,9 +263,9 @@ void State::buildRemote(ref<Store> destStore,
auto drv2 = localStore->readDerivation(input.first); auto drv2 = localStore->readDerivation(input.first);
for (auto & name : input.second) { for (auto & name : input.second) {
if (auto i = get(drv2.outputs, name)) { if (auto i = get(drv2.outputs, name)) {
auto outPath = i->path(*localStore, drv2.name); auto outPath = i->path(*localStore, drv2.name, name);
inputs.insert(outPath); inputs.insert(*outPath);
basicDrv.inputSrcs.insert(outPath); basicDrv.inputSrcs.insert(*outPath);
} }
} }
} }
@ -434,7 +434,11 @@ void State::buildRemote(ref<Store> destStore,
auto now1 = std::chrono::steady_clock::now(); auto now1 = std::chrono::steady_clock::now();
auto outputs = step->drv->outputPaths(*localStore); StorePathSet outputs;
for (auto & i : step->drv->outputsAndOptPaths(*localStore)) {
if (i.second.second)
outputs.insert(*i.second.second);
}
/* Get info about each output path. */ /* Get info about each output path. */
std::map<StorePath, ValidPathInfo> infos; std::map<StorePath, ValidPathInfo> infos;

19
src/hydra-queue-runner/build-result.cc
View file

@ -16,10 +16,13 @@ BuildOutput getBuildOutput(
BuildOutput res; BuildOutput res;
/* Compute the closure size. */ /* Compute the closure size. */
auto outputs = drv.outputPaths(*store); StorePathSet outputs;
StorePathSet closure; StorePathSet closure;
for (auto & output : outputs) for (auto & i : drv.outputsAndOptPaths(*store))
store->computeFSClosure(output, closure); if (i.second.second) {
store->computeFSClosure(*i.second.second, closure);
outputs.insert(*i.second.second);
}
for (auto & path : closure) { for (auto & path : closure) {
auto info = store->queryPathInfo(path); auto info = store->queryPathInfo(path);
res.closureSize += info->narSize; res.closureSize += info->narSize;
@ -104,13 +107,13 @@ BuildOutput getBuildOutput(
/* If no build products were explicitly declared, then add all /* If no build products were explicitly declared, then add all
outputs as a product of type "nix-build". */ outputs as a product of type "nix-build". */
if (!explicitProducts) { if (!explicitProducts) {
for (auto & output : drv.outputs) { for (auto & [name, output] : drv.outputs) {
BuildProduct product; BuildProduct product;
auto outPath = output.second.path(*store, drv.name); auto outPath = output.path(*store, drv.name, name);
product.path = store->printStorePath(outPath); product.path = store->printStorePath(*outPath);
product.type = "nix-build"; product.type = "nix-build";
product.subtype = output.first == "out" ? "" : output.first; product.subtype = name == "out" ? "" : name;
product.name = outPath.name(); product.name = outPath->name();
auto file = narMembers.find(product.path); auto file = narMembers.find(product.path);
assert(file != narMembers.end()); assert(file != narMembers.end());

11
src/hydra-queue-runner/builder.cc
View file

@ -274,8 +274,10 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,
assert(stepNr); assert(stepNr);
for (auto & path : step->drv->outputPaths(*localStore)) for (auto & i : step->drv->outputsAndOptPaths(*localStore)) {
addRoot(path); if (i.second.second)
addRoot(*i.second.second);
}
/* Register success in the database for all Build objects that /* Register success in the database for all Build objects that
have this step as the top-level step. Since the queue have this step as the top-level step. Since the queue
@ -463,8 +465,9 @@ void State::failStep(
/* Remember failed paths in the database so that they /* Remember failed paths in the database so that they
won't be built again. */ won't be built again. */
if (result.stepStatus != bsCachedFailure && result.canCache) if (result.stepStatus != bsCachedFailure && result.canCache)
for (auto & path : step->drv->outputPaths(*localStore)) for (auto & i : step->drv->outputsAndOptPaths(*localStore))
txn.exec_params0("insert into FailedPaths values ($1)", localStore->printStorePath(path)); if (i.second.second)
txn.exec_params0("insert into FailedPaths values ($1)", localStore->printStorePath(*i.second.second));
txn.commit(); txn.commit();
} }

11
src/hydra-queue-runner/hydra-queue-runner.cc
View file

@ -256,10 +256,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID
if (r.affected_rows() == 0) goto restart; if (r.affected_rows() == 0) goto restart;
for (auto & output : step->drv->outputs) for (auto & [name, output] : step->drv->outputs)
txn.exec_params0 txn.exec_params0
("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)", ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)",
buildId, stepNr, output.first, localStore->printStorePath(output.second.path(*localStore, step->drv->name))); buildId, stepNr, name, localStore->printStorePath(*output.path(*localStore, step->drv->name, name)));
if (status == bsBusy) if (status == bsBusy)
txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr)); txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr));
@ -440,9 +440,10 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build,
bool State::checkCachedFailure(Step::ptr step, Connection & conn) bool State::checkCachedFailure(Step::ptr step, Connection & conn)
{ {
pqxx::work txn(conn); pqxx::work txn(conn);
for (auto & path : step->drv->outputPaths(*localStore)) for (auto & i : step->drv->outputsAndOptPaths(*localStore))
if (!txn.exec_params("select 1 from FailedPaths where path = $1", localStore->printStorePath(path)).empty()) if (i.second.second)
return true; if (!txn.exec_params("select 1 from FailedPaths where path = $1", localStore->printStorePath(*i.second.second)).empty())
return true;
return false; return false;
} }

50
src/hydra-queue-runner/queue-monitor.cc
View file

@ -176,13 +176,15 @@ bool State::getQueuedBuilds(Connection & conn,
if (!res[0].is_null()) propagatedFrom = res[0].as<BuildID>(); if (!res[0].is_null()) propagatedFrom = res[0].as<BuildID>();
if (!propagatedFrom) { if (!propagatedFrom) {
for (auto & output : ex.step->drv->outputPaths(*localStore)) { for (auto & i : ex.step->drv->outputsAndOptPaths(*localStore)) {
auto res = txn.exec_params if (i.second.second) {
("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1", auto res = txn.exec_params
localStore->printStorePath(output)); ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1",
if (!res[0][0].is_null()) { localStore->printStorePath(*i.second.second));
propagatedFrom = res[0][0].as<BuildID>(); if (!res[0][0].is_null()) {
break; propagatedFrom = res[0][0].as<BuildID>();
break;
}
} }
} }
} }
@ -221,8 +223,9 @@ bool State::getQueuedBuilds(Connection & conn,
auto drv = localStore->readDerivation(build->drvPath); auto drv = localStore->readDerivation(build->drvPath);
BuildOutput res = getBuildOutputCached(conn, destStore, drv); BuildOutput res = getBuildOutputCached(conn, destStore, drv);
for (auto & path : drv.outputPaths(*localStore)) for (auto & i : drv.outputsAndOptPaths(*localStore))
addRoot(path); if (i.second.second)
addRoot(*i.second.second);
{ {
auto mc = startDbUpdate(); auto mc = startDbUpdate();
@ -453,10 +456,9 @@ Step::ptr State::createStep(ref<Store> destStore,
/* Are all outputs valid? */ /* Are all outputs valid? */
bool valid = true; bool valid = true;
auto outputs = step->drv->outputPaths(*localStore);
DerivationOutputs missing; DerivationOutputs missing;
for (auto & i : step->drv->outputs) for (auto & i : step->drv->outputs)
if (!destStore->isValidPath(i.second.path(*localStore, step->drv->name))) { if (!destStore->isValidPath(*i.second.path(*localStore, step->drv->name, i.first))) {
valid = false; valid = false;
missing.insert_or_assign(i.first, i.second); missing.insert_or_assign(i.first, i.second);
} }
@ -467,12 +469,12 @@ Step::ptr State::createStep(ref<Store> destStore,
size_t avail = 0; size_t avail = 0;
for (auto & i : missing) { for (auto & i : missing) {
auto path = i.second.path(*localStore, step->drv->name); auto path = i.second.path(*localStore, step->drv->name, i.first);
if (/* localStore != destStore && */ localStore->isValidPath(path)) if (/* localStore != destStore && */ localStore->isValidPath(*path))
avail++; avail++;
else if (useSubstitutes) { else if (useSubstitutes) {
SubstitutablePathInfos infos; SubstitutablePathInfos infos;
localStore->querySubstitutablePathInfos({{path, {}}}, infos); localStore->querySubstitutablePathInfos({{*path, {}}}, infos);
if (infos.size() == 1) if (infos.size() == 1)
avail++; avail++;
} }
@ -481,37 +483,37 @@ Step::ptr State::createStep(ref<Store> destStore,
if (missing.size() == avail) { if (missing.size() == avail) {
valid = true; valid = true;
for (auto & i : missing) { for (auto & i : missing) {
auto path = i.second.path(*localStore, step->drv->name); auto path = i.second.path(*localStore, step->drv->name, i.first);
try { try {
time_t startTime = time(0); time_t startTime = time(0);
if (localStore->isValidPath(path)) if (localStore->isValidPath(*path))
printInfo("copying output ‘%1%’ of ‘%2%’ from local store", printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
localStore->printStorePath(path), localStore->printStorePath(*path),
localStore->printStorePath(drvPath)); localStore->printStorePath(drvPath));
else { else {
printInfo("substituting output ‘%1%’ of ‘%2%’", printInfo("substituting output ‘%1%’ of ‘%2%’",
localStore->printStorePath(path), localStore->printStorePath(*path),
localStore->printStorePath(drvPath)); localStore->printStorePath(drvPath));
localStore->ensurePath(path); localStore->ensurePath(*path);
// FIXME: should copy directly from substituter to destStore. // FIXME: should copy directly from substituter to destStore.
} }
copyClosure(ref<Store>(localStore), destStore, {path}); copyClosure(ref<Store>(localStore), destStore, {*path});
time_t stopTime = time(0); time_t stopTime = time(0);
{ {
auto mc = startDbUpdate(); auto mc = startDbUpdate();
pqxx::work txn(conn); pqxx::work txn(conn);
createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", path); createSubstitutionStep(txn, startTime, stopTime, build, drvPath, "out", *path);
txn.commit(); txn.commit();
} }
} catch (Error & e) { } catch (Error & e) {
printError("while copying/substituting output ‘%s’ of ‘%s’: %s", printError("while copying/substituting output ‘%s’ of ‘%s’: %s",
localStore->printStorePath(path), localStore->printStorePath(*path),
localStore->printStorePath(drvPath), localStore->printStorePath(drvPath),
e.what()); e.what());
valid = false; valid = false;
@ -617,12 +619,12 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref<nix::Store>
{ {
pqxx::work txn(conn); pqxx::work txn(conn);
for (auto & output : drv.outputPaths(*localStore)) { for (auto & [name, output] : drv.outputsAndOptPaths(*localStore)) {
auto r = txn.exec_params auto r = txn.exec_params
("select id, buildStatus, releaseName, closureSize, size from Builds b " ("select id, buildStatus, releaseName, closureSize, size from Builds b "
"join BuildOutputs o on b.id = o.build " "join BuildOutputs o on b.id = o.build "
"where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1", "where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1",
localStore->printStorePath(output)); localStore->printStorePath(*output.second));
if (r.empty()) continue; if (r.empty()) continue;
BuildID id = r[0][0].as<BuildID>(); BuildID id = r[0][0].as<BuildID>();